mattermost: Add support for exporting DMs and huddles.

templates/zerver/help/import-from-mattermost.md

@@ -177,11 +177,15 @@ root domain. Replace the last line above with the following, after replacing
 Mattermost's export tool is incomplete and does not support exporting
 the following data:
 
-* private messages and group private messages between users
 * user avatars
 * uploaded files and message attachments.
 
 We expect to add support for importing these data from Mattermost once
 Mattermost's export tool includes them.
 
+Additionally, Mattermost's data exports do not associated private
+messages with a specific Mattermost team.  For that reason, the import
+tool will only import private messages for data export archives
+containing a single Mattermost team.
+
 [upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository

tools/test-backend

@@ -123,7 +123,6 @@ not_yet_fully_covered = {path for target in [
     'zerver/tornado/websocket_client.py',
     # Data import files; relatively low priority
     'zerver/data_import/hipchat*.py',
-    'zerver/data_import/mattermost.py',
     'zerver/data_import/sequencer.py',
     'zerver/data_import/slack.py',
     'zerver/data_import/gitter.py',

zerver/data_import/import_util.py

@@ -222,6 +222,34 @@ def build_stream_subscriptions(
 
     return subscriptions
 
+def build_huddle_subscriptions(
+        get_users: Callable[..., Set[int]],
+        zerver_recipient: List[ZerverFieldsT],
+        zerver_huddle: List[ZerverFieldsT]) -> List[ZerverFieldsT]:
+
+    subscriptions = []  # type: List[ZerverFieldsT]
+
+    huddle_ids = {huddle['id'] for huddle in zerver_huddle}
+
+    recipient_map = {
+        recipient['id']: recipient['type_id']  # recipient_id -> stream_id
+        for recipient in zerver_recipient
+        if recipient['type'] == Recipient.HUDDLE
+        and recipient['type_id'] in huddle_ids
+    }
+
+    for recipient_id, huddle_id in recipient_map.items():
+        user_ids = get_users(huddle_id=huddle_id)
+        for user_id in user_ids:
+            subscription = build_subscription(
+                recipient_id=recipient_id,
+                user_id=user_id,
+                subscription_id=NEXT_ID('subscription'),
+            )
+            subscriptions.append(subscription)
+
+    return subscriptions
+
 def build_personal_subscriptions(zerver_recipient: List[ZerverFieldsT]) -> List[ZerverFieldsT]:
 
     subscriptions = []  # type: List[ZerverFieldsT]
@@ -253,7 +281,8 @@ def build_recipient(type_id: int, recipient_id: int, type: int) -> ZerverFieldsT
     return recipient_dict
 
 def build_recipients(zerver_userprofile: List[ZerverFieldsT],
-                     zerver_stream: List[ZerverFieldsT]) -> List[ZerverFieldsT]:
+                     zerver_stream: List[ZerverFieldsT],
+                     zerver_huddle: List[ZerverFieldsT]=[]) -> List[ZerverFieldsT]:
     '''
     As of this writing, we only use this in the HipChat
     conversion.  The Slack and Gitter conversions do it more
@@ -284,6 +313,16 @@ def build_recipients(zerver_userprofile: List[ZerverFieldsT],
         recipient_dict = model_to_dict(recipient)
         recipients.append(recipient_dict)
 
+    for huddle in zerver_huddle:
+        type_id = huddle['id']
+        type = Recipient.HUDDLE
+        recipient = Recipient(
+            type_id=type_id,
+            id=NEXT_ID('recipient'),
+            type=type,
+        )
+        recipient_dict = model_to_dict(recipient)
+        recipients.append(recipient_dict)
     return recipients
 
 def build_realm(zerver_realm: List[ZerverFieldsT], realm_id: int,

zerver/data_import/mattermost.py

@@ -23,8 +23,9 @@ from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH
 from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \
     build_stream, build_realm, build_message, create_converted_data_files, \
     make_subscriber_map, build_recipients, build_user_profile, \
-    build_stream_subscriptions, build_personal_subscriptions, SubscriberHandler, \
-    build_realm_emoji, make_user_messages
+    build_stream_subscriptions, build_huddle_subscriptions, \
+    build_personal_subscriptions, SubscriberHandler, \
+    build_realm_emoji, make_user_messages, build_huddle
 
 from zerver.data_import.mattermost_user import UserHandler
 from zerver.data_import.sequencer import NEXT_ID, IdMapper
@@ -196,13 +197,43 @@ def convert_channel_data(channel_data: List[ZerverFieldsT],
 
         if channel_users:
             subscriber_handler.set_info(
-                stream_id=stream_id,
                 users=channel_users,
+                stream_id=stream_id,
             )
         streams.append(stream)
-
     return streams
 
+def generate_huddle_name(huddle_members: List[str]) -> str:
+    # Simple hash function to generate a unique hash key for the
+    # members of a huddle.  Needs to be consistent only within the
+    # lifetime of export tool run, as it doesn't appear in the output.
+    import hashlib
+    return hashlib.md5(''.join(sorted(huddle_members)).encode('utf-8')).hexdigest()
+
+def convert_huddle_data(huddle_data: List[ZerverFieldsT],
+                        user_data_map: Dict[str, Dict[str, Any]],
+                        subscriber_handler: SubscriberHandler,
+                        huddle_id_mapper: IdMapper,
+                        user_id_mapper: IdMapper,
+                        realm_id: int,
+                        team_name: str) -> List[ZerverFieldsT]:
+
+    zerver_huddle = []
+    for huddle in huddle_data:
+        if len(huddle["members"]) > 2:
+            huddle_name = generate_huddle_name(huddle["members"])
+            huddle_id = huddle_id_mapper.get(huddle_name)
+            huddle_dict = build_huddle(huddle_id)
+            huddle_user_ids = set()
+            for username in huddle["members"]:
+                huddle_user_ids.add(user_id_mapper.get(username))
+            subscriber_handler.set_info(
+                users=huddle_user_ids,
+                huddle_id=huddle_id,
+            )
+            zerver_huddle.append(huddle_dict)
+    return zerver_huddle
+
 def get_name_to_codepoint_dict() -> Dict[str, str]:
     with open(NAME_TO_CODEPOINT_PATH) as fp:
         return ujson.load(fp)
@@ -291,6 +322,7 @@ def process_raw_message_batch(realm_id: int,
     h = html2text.HTML2Text()
 
     name_to_codepoint = get_name_to_codepoint_dict()
+    pm_members = {}
 
     for raw_message in raw_messages:
         message_id = NEXT_ID('message')
@@ -309,11 +341,21 @@ def process_raw_message_batch(realm_id: int,
 
         date_sent = raw_message['date_sent']
         sender_user_id = raw_message['sender_id']
-        try:
-            recipient_id = get_recipient_id_from_receiver_name(raw_message["receiver_id"], Recipient.STREAM)
-        except KeyError:
-            logging.debug("Could not find recipient_id for a message, skipping.")
-            continue
+        if "channel_name" in raw_message:
+            recipient_id = get_recipient_id_from_receiver_name(raw_message["channel_name"], Recipient.STREAM)
+        elif "huddle_name" in raw_message:
+            recipient_id = get_recipient_id_from_receiver_name(raw_message["huddle_name"], Recipient.HUDDLE)
+        elif "pm_members" in raw_message:
+            members = raw_message["pm_members"]
+            member_ids = {user_id_mapper.get(member) for member in members}
+            pm_members[message_id] = member_ids
+            if sender_user_id == user_id_mapper.get(members[0]):
+                recipient_id = get_recipient_id_from_receiver_name(members[1], Recipient.PERSONAL)
+            else:
+                recipient_id = get_recipient_id_from_receiver_name(members[0], Recipient.PERSONAL)
+        else:
+            raise AssertionError("raw_message without channel_name, huddle_name or pm_members key")
+
         rendered_content = None
 
         topic_name = 'imported from mattermost'
@@ -377,7 +419,8 @@ def process_posts(num_teams: int,
             post_data_list.append(post)
 
     def message_to_dict(post_dict: Dict[str, Any]) -> Dict[str, Any]:
-        sender_id = user_id_mapper.get(post_dict["user"])
+        sender_username = post_dict["user"]
+        sender_id = user_id_mapper.get(sender_username)
         content = post_dict['message']
 
         if masking_content:
@@ -389,13 +432,26 @@ def process_posts(num_teams: int,
         else:
             reactions = []
 
-        return dict(
+        message_dict = dict(
             sender_id=sender_id,
-            receiver_id=post_dict["channel"],
             content=content,
             date_sent=int(post_dict['create_at'] / 1000),
             reactions=reactions
         )
+        if "channel" in post_dict:
+            message_dict["channel_name"] = post_dict["channel"]
+        elif "channel_members" in post_dict:
+            # This case is for handling posts from PMs and huddles, not channels.
+            # PMs and huddles are known as direct_channels in Slack and hence
+            # the name channel_members.
+            channel_members = post_dict["channel_members"]
+            if len(channel_members) > 2:
+                message_dict["huddle_name"] = generate_huddle_name(channel_members)
+            elif len(channel_members) == 2:
+                message_dict["pm_members"] = channel_members
+        else:
+            raise AssertionError("Post without channel or channel_members key.")
+        return message_dict
 
     raw_messages = []
     for post_dict in post_data_list:
@@ -433,45 +489,67 @@ def process_posts(num_teams: int,
 def write_message_data(num_teams: int,
                        team_name: str,
                        realm_id: int,
-                       post_data: List[Dict[str, Any]],
+                       post_data: Dict[str, List[Dict[str, Any]]],
                        zerver_recipient: List[ZerverFieldsT],
                        subscriber_map: Dict[int, Set[int]],
                        output_dir: str,
                        masking_content: bool,
                        stream_id_mapper: IdMapper,
+                       huddle_id_mapper: IdMapper,
                        user_id_mapper: IdMapper,
                        user_handler: UserHandler,
                        username_to_user: Dict[str, Dict[str, Any]],
                        zerver_realmemoji: List[Dict[str, Any]],
                        total_reactions: List[Dict[str, Any]]) -> None:
     stream_id_to_recipient_id = {}
+    huddle_id_to_recipient_id = {}
+    user_id_to_recipient_id = {}
 
     for d in zerver_recipient:
         if d['type'] == Recipient.STREAM:
             stream_id_to_recipient_id[d['type_id']] = d['id']
+        elif d['type'] == Recipient.HUDDLE:
+            huddle_id_to_recipient_id[d['type_id']] = d['id']
+        if d['type'] == Recipient.PERSONAL:
+            user_id_to_recipient_id[d['type_id']] = d['id']
 
     def get_recipient_id_from_receiver_name(receiver_name: str, recipient_type: int) -> int:
         if recipient_type == Recipient.STREAM:
             receiver_id = stream_id_mapper.get(receiver_name)
             recipient_id = stream_id_to_recipient_id[receiver_id]
+        elif recipient_type == Recipient.HUDDLE:
+            receiver_id = huddle_id_mapper.get(receiver_name)
+            recipient_id = huddle_id_to_recipient_id[receiver_id]
+        elif recipient_type == Recipient.PERSONAL:
+            receiver_id = user_id_mapper.get(receiver_name)
+            recipient_id = user_id_to_recipient_id[receiver_id]
+        else:
+            raise AssertionError("Invalid recipient_type")
         return recipient_id
 
-    process_posts(
-        num_teams=num_teams,
-        team_name=team_name,
-        realm_id=realm_id,
-        post_data=post_data,
-        get_recipient_id_from_receiver_name=get_recipient_id_from_receiver_name,
-        subscriber_map=subscriber_map,
-        output_dir=output_dir,
-        is_pm_data=False,
-        masking_content=masking_content,
-        user_id_mapper=user_id_mapper,
-        user_handler=user_handler,
-        username_to_user=username_to_user,
-        zerver_realmemoji=zerver_realmemoji,
-        total_reactions=total_reactions,
-    )
+    if num_teams == 1:
+        post_types = ["channel_post", "direct_post"]
+    else:
+        post_types = ["channel_post"]
+        logging.warning("Skipping importing huddles and PMs since there are multiple teams in the export")
+
+    for post_type in post_types:
+        process_posts(
+            num_teams=num_teams,
+            team_name=team_name,
+            realm_id=realm_id,
+            post_data=post_data[post_type],
+            get_recipient_id_from_receiver_name=get_recipient_id_from_receiver_name,
+            subscriber_map=subscriber_map,
+            output_dir=output_dir,
+            is_pm_data=post_type == "direct_post",
+            masking_content=masking_content,
+            user_id_mapper=user_id_mapper,
+            user_handler=user_handler,
+            username_to_user=username_to_user,
+            zerver_realmemoji=zerver_realmemoji,
+            total_reactions=total_reactions,
+        )
 
 def write_emoticon_data(realm_id: int,
                         custom_emoji_data: List[Dict[str, Any]],
@@ -572,47 +650,50 @@ def check_user_in_team(user: Dict[str, Any], team_name: str) -> bool:
             return True
     return False
 
-def label_mirror_dummy_users(num_teams: int, team_name: str,
-                             mattermost_data: Dict[str, List[Dict[str, Any]]],
+def label_mirror_dummy_users(num_teams: int, team_name: str, mattermost_data: Dict[str, Any],
                              username_to_user: Dict[str, Dict[str, Any]]) -> None:
     # This function might looks like a great place to label admin users. But
     # that won't be fully correct since we are iterating only though posts and
     # it covers only users that has sent atleast one message.
-    for post in mattermost_data["post"]:
-        if "team" not in post:
-            if num_teams > 1:
-                raise AssertionError("Cannot import private messages in export with multiple teams.")
-            elif num_teams == 0:
-                raise AssertionError("Export with 0 teams")
-            else:
-                post_team = team_name
-        else:
-            post_team = post["team"]
-
+    for post in mattermost_data["post"]["channel_post"]:
+        post_team = post["team"]
         if post_team == team_name:
             user = username_to_user[post["user"]]
             if not check_user_in_team(user, team_name):
                 user["is_mirror_dummy"] = True
 
+    if num_teams == 1:
+        for post in mattermost_data["post"]["direct_post"]:
+            assert("team" not in post)
+            user = username_to_user[post["user"]]
+            if not check_user_in_team(user, team_name):
+                user["is_mirror_dummy"] = True
+
 def reset_mirror_dummy_users(username_to_user: Dict[str, Dict[str, Any]]) -> None:
     for username in username_to_user:
         user = username_to_user[username]
         user["is_mirror_dummy"] = False
 
-def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, List[Dict[str, Any]]]:
-    mattermost_data = {}  # type: Dict[str, List[Dict[str, Any]]]
+def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, Any]:
+    mattermost_data = {}  # type: Dict[str, Any]
     mattermost_data["version"] = []
     mattermost_data["team"] = []
     mattermost_data["channel"] = []
     mattermost_data["user"] = []
-    mattermost_data["post"] = []
+    mattermost_data["post"] = {"channel_post": [], "direct_post": []}
     mattermost_data["emoji"] = []
+    mattermost_data["direct_channel"] = []
 
     with open(mattermost_data_file, "r") as fp:
         for line in fp:
             row = ujson.loads(line.rstrip("\n"))
             data_type = row["type"]
-            mattermost_data[data_type].append(row[data_type])
+            if data_type == "post":
+                mattermost_data["post"]["channel_post"].append(row["post"])
+            elif data_type == "direct_post":
+                mattermost_data["post"]["direct_post"].append(row["direct_post"])
+            else:
+                mattermost_data[data_type].append(row[data_type])
     return mattermost_data
 
 def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None:
@@ -635,6 +716,7 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
         subscriber_handler = SubscriberHandler()
         user_id_mapper = IdMapper()
         stream_id_mapper = IdMapper()
+        huddle_id_mapper = IdMapper()
 
         print("Generating data for", team_name)
         realm = make_realm(realm_id, team)
@@ -660,14 +742,27 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
             realm_id=realm_id,
             team_name=team_name,
         )
-
         realm['zerver_stream'] = zerver_stream
 
+        zerver_huddle = []  # type: List[ZerverFieldsT]
+        if len(mattermost_data["team"]) == 1:
+            zerver_huddle = convert_huddle_data(
+                huddle_data=mattermost_data["direct_channel"],
+                user_data_map=username_to_user,
+                subscriber_handler=subscriber_handler,
+                huddle_id_mapper=huddle_id_mapper,
+                user_id_mapper=user_id_mapper,
+                realm_id=realm_id,
+                team_name=team_name,
+            )
+            realm['zerver_huddle'] = zerver_huddle
+
         all_users = user_handler.get_all_users()
 
         zerver_recipient = build_recipients(
             zerver_userprofile=all_users,
             zerver_stream=zerver_stream,
+            zerver_huddle=zerver_huddle,
         )
         realm['zerver_recipient'] = zerver_recipient
 
@@ -677,13 +772,19 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
             zerver_stream=zerver_stream,
         )
 
+        huddle_subscriptions = build_huddle_subscriptions(
+            get_users=subscriber_handler.get_users,
+            zerver_recipient=zerver_recipient,
+            zerver_huddle=zerver_huddle,
+        )
+
         personal_subscriptions = build_personal_subscriptions(
             zerver_recipient=zerver_recipient,
         )
 
         # Mattermost currently supports only exporting messages from channels.
         # Personal messages and huddles are not exported.
-        zerver_subscription = personal_subscriptions + stream_subscriptions
+        zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
         realm['zerver_subscription'] = zerver_subscription
 
         zerver_realmemoji = write_emoticon_data(
@@ -709,6 +810,7 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
             output_dir=realm_output_dir,
             masking_content=masking_content,
             stream_id_mapper=stream_id_mapper,
+            huddle_id_mapper=huddle_id_mapper,
             user_id_mapper=user_id_mapper,
             user_handler=user_handler,
             username_to_user=username_to_user,

zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json

@@ -0,0 +1,22 @@
+{"type":"version","version":1}
+{"type":"team","team":{"name":"gryffindor","display_name":"Gryffindor","type":"O","description":"","allow_open_invite":true}}
+{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-common-room","display_name":"Gryffindor common room","type":"O","header":"","purpose":"A place for talking about Gryffindor common room"}}
+{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-quidditch-team","display_name":"Gryffindor quidditch team","type":"O","header":"","purpose":"A place for talking about Gryffindor quidditch team"}}
+{"type":"channel","channel":{"team":"gryffindor","name":"dumbledores-army","display_name":"Dumbledores army","type":"P","header":"https//:github.com/zulip/zulip","purpose":"A place for talking about Dumbledores army"}}
+{"type":"user","user":{"username":"ginny","email":"ginny@zulip.com","auth_service":"","nickname":"","first_name":"Ginny","last_name":"Weasley","position":"","roles":"system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_user","channels":[{"name":"gryffindor-quidditch-team","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"dumbledores-army","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"ginny,@ginny"}}}
+{"type":"user","user":{"username":"ron","email":"ron@zulip.com","auth_service":"","nickname":"","first_name":"Ron","last_name":"Weasley","position":"","roles":"system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_user","channels":[{"name":"gryffindor-quidditch-team","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"dumbledores-army","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"ron,@ron"}}}
+{"type":"user","user":{"username":"harry","email":"harry@zulip.com","auth_service":"","nickname":"","first_name":"Harry","last_name":"Potter","position":"","roles":"system_admin system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_admin team_user","channels":[{"name":"dumbledores-army","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"harry,@harry"}}}
+{"type":"user","user":{"username":"voldemort","email":"voldemort@zulip.com","auth_service":"","nickname":"","first_name":"Tom","last_name":"Riddle","position":"","roles":"system_admin system_user","locale":"en","teams":null, "notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"harry,@harry"}}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"ron","message":"ron joined the channel.","create_at":1553166512493,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"Hey folks","create_at":1553166519720,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"@ron Welcome mate!","create_at":1553166519726,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"harry","message":"Looks like this channel is empty","create_at":1553166567370,"reactions":[{"user":"ron","create_at":1553166584976,"emoji_name":"rocket"}],"replies":null}}
+{"type":"direct_channel","direct_channel":{"members":["ron","harry"],"favorited_by":null,"header":""}}
+{"type":"direct_channel","direct_channel":{"members":["ron","harry", "ginny"],"favorited_by":null,"header":""}}
+{"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"ron","message":"hey harry","create_at":1566376137676,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"harry","message":"whats up","create_at":1566376318568,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ginny","message":"Who is going to Hogesmead this weekend?","create_at":1566376226493,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"I am going.","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ron","message":"I am going as well","create_at":1566376286363,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"voldemort","message":"Hey Harry.","create_at":1566376318569,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"harry","message":"Ahh. Here we go again.","create_at":1566376318579,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}

zerver/tests/test_mattermost_importer.py

@@ -16,10 +16,11 @@ from zerver.data_import.mattermost_user import UserHandler
 from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \
     create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \
     convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \
-    build_reactions, get_name_to_codepoint_dict, do_convert_data
+    build_reactions, get_name_to_codepoint_dict, do_convert_data, convert_huddle_data, \
+    generate_huddle_name
 from zerver.data_import.sequencer import IdMapper
 from zerver.data_import.import_util import SubscriberHandler
-from zerver.models import Reaction, UserProfile, Message, get_realm, get_user
+from zerver.models import Reaction, UserProfile, Message, get_realm, get_user, Recipient
 
 class MatterMostImporter(ZulipTestCase):
     logger = logging.getLogger()
@@ -29,8 +30,7 @@ class MatterMostImporter(ZulipTestCase):
     def test_mattermost_data_file_to_dict(self) -> None:
         fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
         mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
-
-        self.assertEqual(len(mattermost_data), 6)
+        self.assertEqual(len(mattermost_data), 7)
 
         self.assertEqual(mattermost_data["version"], [1])
 
@@ -45,15 +45,30 @@ class MatterMostImporter(ZulipTestCase):
         self.assertEqual(mattermost_data["user"][1]["username"], "harry")
         self.assertEqual(len(mattermost_data["user"][1]["teams"]), 1)
 
-        self.assertEqual(len(mattermost_data["post"]), 20)
-        self.assertEqual(mattermost_data["post"][0]["team"], "gryffindor")
-        self.assertEqual(mattermost_data["post"][0]["channel"], "dumbledores-army")
-        self.assertEqual(mattermost_data["post"][0]["user"], "harry")
-        self.assertEqual(len(mattermost_data["post"][0]["replies"]), 1)
+        self.assertEqual(len(mattermost_data["post"]["channel_post"]), 20)
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["team"], "gryffindor")
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["channel"], "dumbledores-army")
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["user"], "harry")
+        self.assertEqual(len(mattermost_data["post"]["channel_post"][0]["replies"]), 1)
 
         self.assertEqual(len(mattermost_data["emoji"]), 2)
         self.assertEqual(mattermost_data["emoji"][0]["name"], "peerdium")
 
+        fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures/direct_channel")
+        mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
+
+        self.assertEqual(len(mattermost_data["post"]["channel_post"]), 4)
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["team"], "gryffindor")
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["channel"], "gryffindor-common-room")
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["user"], "ron")
+        self.assertEqual(mattermost_data["post"]["channel_post"][0]["replies"], None)
+
+        self.assertEqual(len(mattermost_data["post"]["direct_post"]), 7)
+        self.assertEqual(mattermost_data["post"]["direct_post"][0]["user"], "ron")
+        self.assertEqual(mattermost_data["post"]["direct_post"][0]["replies"], None)
+        self.assertEqual(mattermost_data["post"]["direct_post"][0]["message"], "hey harry")
+        self.assertEqual(mattermost_data["post"]["direct_post"][0]["channel_members"], ["ron", "harry"])
+
     def test_process_user(self) -> None:
         user_id_mapper = IdMapper()
         fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
@@ -112,6 +127,7 @@ class MatterMostImporter(ZulipTestCase):
         team_name = "gryffindor"
         user_handler = UserHandler()
         convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name)
+        self.assertEqual(len(user_handler.get_all_users()), 2)
         self.assertTrue(user_id_mapper.has("harry"))
         self.assertTrue(user_id_mapper.has("ron"))
         self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
@@ -235,6 +251,43 @@ class MatterMostImporter(ZulipTestCase):
         self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id})
         self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
 
+    def test_convert_huddle_data(self) -> None:
+        fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures/direct_channel")
+        mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
+        username_to_user = create_username_to_user_mapping(mattermost_data["user"])
+        reset_mirror_dummy_users(username_to_user)
+
+        user_handler = UserHandler()
+        subscriber_handler = SubscriberHandler()
+        huddle_id_mapper = IdMapper()
+        user_id_mapper = IdMapper()
+        team_name = "gryffindor"
+
+        convert_user_data(
+            user_handler=user_handler,
+            user_id_mapper=user_id_mapper,
+            user_data_map=username_to_user,
+            realm_id=3,
+            team_name=team_name,
+        )
+
+        zerver_huddle = convert_huddle_data(
+            huddle_data=mattermost_data["direct_channel"],
+            user_data_map=username_to_user,
+            subscriber_handler=subscriber_handler,
+            huddle_id_mapper=huddle_id_mapper,
+            user_id_mapper=user_id_mapper,
+            realm_id=3,
+            team_name=team_name,
+        )
+
+        self.assertEqual(len(zerver_huddle), 1)
+        huddle_members = mattermost_data["direct_channel"][1]["members"]
+        huddle_name = generate_huddle_name(huddle_members)
+
+        self.assertTrue(huddle_id_mapper.has(huddle_name))
+        self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
+
     def test_write_emoticon_data(self) -> None:
         fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
         mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
@@ -512,6 +565,97 @@ class MatterMostImporter(ZulipTestCase):
         for message in messages:
             self.assertIsNotNone(message.rendered_content)
 
+    def test_do_convert_data_with_direct_messages(self) -> None:
+        mattermost_data_dir = self.fixture_file_name("direct_channel", "mattermost_fixtures")
+        output_dir = self.make_import_output_dir("mattermost")
+
+        do_convert_data(
+            mattermost_data_dir=mattermost_data_dir,
+            output_dir=output_dir,
+            masking_content=False
+        )
+
+        harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor")
+        self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'avatars')), True)
+        self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'emoji')), True)
+        self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'attachment.json')), True)
+
+        realm = self.read_file(harry_team_output_dir, 'realm.json')
+
+        self.assertEqual('Organization imported from Mattermost!',
+                         realm['zerver_realm'][0]['description'])
+
+        exported_user_ids = self.get_set(realm['zerver_userprofile'], 'id')
+        exported_user_full_names = self.get_set(realm['zerver_userprofile'], 'full_name')
+        self.assertEqual(set(['Harry Potter', 'Ron Weasley', 'Ginny Weasley', 'Tom Riddle']), exported_user_full_names)
+
+        exported_user_emails = self.get_set(realm['zerver_userprofile'], 'email')
+        self.assertEqual(set(['harry@zulip.com', 'ron@zulip.com', 'ginny@zulip.com', 'voldemort@zulip.com']), exported_user_emails)
+
+        self.assertEqual(len(realm['zerver_stream']), 3)
+        exported_stream_names = self.get_set(realm['zerver_stream'], 'name')
+        self.assertEqual(exported_stream_names, set(['Gryffindor common room', 'Gryffindor quidditch team', 'Dumbledores army']))
+        self.assertEqual(self.get_set(realm['zerver_stream'], 'realm'), set([realm['zerver_realm'][0]['id']]))
+        self.assertEqual(self.get_set(realm['zerver_stream'], 'deactivated'), set([False]))
+
+        self.assertEqual(len(realm['zerver_defaultstream']), 0)
+
+        exported_recipient_ids = self.get_set(realm['zerver_recipient'], 'id')
+        self.assertEqual(len(exported_recipient_ids), 8)
+        exported_recipient_types = self.get_set(realm['zerver_recipient'], 'type')
+        self.assertEqual(exported_recipient_types, set([1, 2, 3]))
+        exported_recipient_type_ids = self.get_set(realm['zerver_recipient'], 'type_id')
+        self.assertEqual(len(exported_recipient_type_ids), 4)
+
+        exported_subscription_userprofile = self.get_set(realm['zerver_subscription'], 'user_profile')
+        self.assertEqual(len(exported_subscription_userprofile), 4)
+        exported_subscription_recipients = self.get_set(realm['zerver_subscription'], 'recipient')
+        self.assertEqual(len(exported_subscription_recipients), 8)
+
+        messages = self.read_file(harry_team_output_dir, 'messages-000001.json')
+
+        exported_messages_id = self.get_set(messages['zerver_message'], 'id')
+        self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids)
+        self.assertIn(messages['zerver_message'][0]['recipient'], exported_recipient_ids)
+        self.assertIn(messages['zerver_message'][0]['content'], 'ron joined the channel.\n\n')
+
+        exported_usermessage_userprofiles = self.get_set(messages['zerver_usermessage'], 'user_profile')
+        self.assertEqual(len(exported_usermessage_userprofiles), 3)
+        exported_usermessage_messages = self.get_set(messages['zerver_usermessage'], 'message')
+        self.assertEqual(exported_usermessage_messages, exported_messages_id)
+
+        do_import_realm(
+            import_dir=harry_team_output_dir,
+            subdomain='gryffindor'
+        )
+        realm = get_realm('gryffindor')
+
+        messages = Message.objects.filter(sender__realm=realm)
+        for message in messages:
+            self.assertIsNotNone(message.rendered_content)
+        self.assertEqual(len(messages), 11)
+
+        stream_messages = messages.filter(recipient__type=Recipient.STREAM).order_by("pub_date")
+        stream_recipients = stream_messages.values_list("recipient", flat=True)
+        self.assertEqual(len(stream_messages), 4)
+        self.assertEqual(len(set(stream_recipients)), 2)
+        self.assertEqual(stream_messages[0].sender.email, "ron@zulip.com")
+        self.assertEqual(stream_messages[0].content, "ron joined the channel.\n\n")
+
+        huddle_messages = messages.filter(recipient__type=Recipient.HUDDLE).order_by("pub_date")
+        huddle_recipients = huddle_messages.values_list("recipient", flat=True)
+        self.assertEqual(len(huddle_messages), 3)
+        self.assertEqual(len(set(huddle_recipients)), 1)
+        self.assertEqual(huddle_messages[0].sender.email, "ginny@zulip.com")
+        self.assertEqual(huddle_messages[0].content, "Who is going to Hogesmead this weekend?\n\n")
+
+        personal_messages = messages.filter(recipient__type=Recipient.PERSONAL).order_by("pub_date")
+        personal_recipients = personal_messages.values_list("recipient", flat=True)
+        self.assertEqual(len(personal_messages), 4)
+        self.assertEqual(len(set(personal_recipients)), 3)
+        self.assertEqual(personal_messages[0].sender.email, "ron@zulip.com")
+        self.assertEqual(personal_messages[0].content, "hey harry\n\n")
+
     def test_do_convert_data_with_masking(self) -> None:
         mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures")
         output_dir = self.make_import_output_dir("mattermost")