From 044fe547d347137558be4245189cc2c0cfcff1a8 Mon Sep 17 00:00:00 2001 From: Priyansh Garg Date: Tue, 27 Jul 2021 02:34:11 +0530 Subject: [PATCH] data_import: Add huddle import support for Rocket.Chat. --- zerver/data_import/rocketchat.py | 110 +++++++++++--- .../rocketchat_message.bson | Bin 12426 -> 14766 bytes .../rocketchat_fixtures/rocketchat_room.bson | Bin 6571 -> 7408 bytes zerver/tests/test_rocketchat_importer.py | 136 +++++++++++++++--- 4 files changed, 206 insertions(+), 40 deletions(-) diff --git a/zerver/data_import/rocketchat.py b/zerver/data_import/rocketchat.py index a141cd8593..ad39d09bed 100644 --- a/zerver/data_import/rocketchat.py +++ b/zerver/data_import/rocketchat.py @@ -10,6 +10,8 @@ from django.forms.models import model_to_dict from zerver.data_import.import_util import ( SubscriberHandler, ZerverFieldsT, + build_huddle, + build_huddle_subscriptions, build_message, build_personal_subscriptions, build_realm, @@ -174,7 +176,7 @@ def convert_channel_data( return streams -def convert_subscription_data( +def convert_stream_subscription_data( user_id_to_user_map: Dict[str, Dict[str, Any]], dsc_id_to_dsc_map: Dict[str, Dict[str, Any]], zerver_stream: List[ZerverFieldsT], @@ -211,6 +213,31 @@ def convert_subscription_data( subscriber_handler.set_info(users=users, stream_id=stream["id"]) +def convert_huddle_data( + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]], + huddle_id_mapper: IdMapper, + user_id_mapper: IdMapper, + subscriber_handler: SubscriberHandler, +) -> List[ZerverFieldsT]: + zerver_huddle: List[ZerverFieldsT] = [] + + for rc_huddle_id in huddle_id_to_huddle_map: + huddle_id = huddle_id_mapper.get(rc_huddle_id) + huddle = build_huddle(huddle_id) + zerver_huddle.append(huddle) + + huddle_dict = huddle_id_to_huddle_map[rc_huddle_id] + huddle_user_ids = set() + for rc_user_id in huddle_dict["uids"]: + huddle_user_ids.add(user_id_mapper.get(rc_user_id)) + subscriber_handler.set_info( + users=huddle_user_ids, + huddle_id=huddle_id, + ) + + return zerver_huddle + + def build_reactions( total_reactions: List[ZerverFieldsT], reactions: List[Dict[str, Any]], @@ -334,8 +361,11 @@ def process_messages( user_id_to_recipient_id: Dict[int, int], stream_id_mapper: IdMapper, stream_id_to_recipient_id: Dict[int, int], - direct_id_to_direct_map: Dict[str, Dict[str, Any]], + huddle_id_mapper: IdMapper, + huddle_id_to_recipient_id: Dict[int, int], dsc_id_to_dsc_map: Dict[str, Dict[str, Any]], + direct_id_to_direct_map: Dict[str, Dict[str, Any]], + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]], total_reactions: List[ZerverFieldsT], output_dir: str, ) -> None: @@ -373,15 +403,20 @@ def process_messages( # Add recipient_id and topic to message_dict if is_pm_data: - direct_channel_id = message["rid"] - rc_member_ids = direct_id_to_direct_map[direct_channel_id]["uids"] - if rc_sender_id == rc_member_ids[0]: - zulip_member_id = user_id_mapper.get(rc_member_ids[1]) - message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id] + # Message is in a PM or a huddle. + rc_channel_id = message["rid"] + if rc_channel_id in huddle_id_to_huddle_map: + huddle_id = huddle_id_mapper.get(rc_channel_id) + message_dict["recipient_id"] = huddle_id_to_recipient_id[huddle_id] else: - zulip_member_id = user_id_mapper.get(rc_member_ids[0]) - message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id] - # PMs don't have topics, but topic_name field is required in `build_message`. + rc_member_ids = direct_id_to_direct_map[rc_channel_id]["uids"] + if rc_sender_id == rc_member_ids[0]: + zulip_member_id = user_id_mapper.get(rc_member_ids[1]) + message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id] + else: + zulip_member_id = user_id_mapper.get(rc_member_ids[0]) + message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id] + # PMs and huddles don't have topics, but topic_name field is required in `build_message`. message_dict["topic_name"] = "" elif message["rid"] in dsc_id_to_dsc_map: # Message is in a discussion @@ -438,16 +473,19 @@ def process_messages( def separate_channel_and_private_messages( messages: List[Dict[str, Any]], direct_id_to_direct_map: Dict[str, Dict[str, Any]], + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]], channel_messages: List[Dict[str, Any]], private_messages: List[Dict[str, Any]], ) -> None: - direct_channels_list = direct_id_to_direct_map.keys() + private_channels_list = list(direct_id_to_direct_map.keys()) + list( + huddle_id_to_huddle_map.keys() + ) for message in messages: if not message.get("rid"): # Message does not belong to any channel (might be # related to livechat), so ignore all such messages. continue - if message["rid"] in direct_channels_list: + if message["rid"] in private_channels_list: private_messages.append(message) else: channel_messages.append(message) @@ -456,12 +494,15 @@ def separate_channel_and_private_messages( def map_receiver_id_to_recipient_id( zerver_recipient: List[ZerverFieldsT], stream_id_to_recipient_id: Dict[int, int], + huddle_id_to_recipient_id: Dict[int, int], user_id_to_recipient_id: Dict[int, int], ) -> None: - # receiver_id represents stream_id/user_id + # receiver_id represents stream_id/huddle_id/user_id for recipient in zerver_recipient: if recipient["type"] == Recipient.STREAM: stream_id_to_recipient_id[recipient["type_id"]] = recipient["id"] + elif recipient["type"] == Recipient.HUDDLE: + huddle_id_to_recipient_id[recipient["type_id"]] = recipient["id"] elif recipient["type"] == Recipient.PERSONAL: user_id_to_recipient_id[recipient["type_id"]] = recipient["id"] @@ -472,12 +513,16 @@ def categorize_channels_and_map_with_id( team_id_to_team_map: Dict[str, Dict[str, Any]], dsc_id_to_dsc_map: Dict[str, Dict[str, Any]], direct_id_to_direct_map: Dict[str, Dict[str, Any]], + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]], ) -> None: for channel in channel_data: if channel.get("prid"): dsc_id_to_dsc_map[channel["_id"]] = channel elif channel["t"] == "d": - direct_id_to_direct_map[channel["_id"]] = channel + if len(channel["uids"]) > 2: + huddle_id_to_huddle_map[channel["_id"]] = channel + else: + direct_id_to_direct_map[channel["_id"]] = channel else: room_id_to_room_map[channel["_id"]] = channel if channel.get("teamMain") is True: @@ -553,6 +598,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() + huddle_id_mapper = IdMapper() process_users( user_id_to_user_map=user_id_to_user_map, @@ -566,6 +612,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], @@ -573,6 +620,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( @@ -583,8 +631,8 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: ) realm["zerver_stream"] = zerver_stream - # Add subscription data to subscriber handler - convert_subscription_data( + # Add stream subscription data to `subscriber_handler` + convert_stream_subscription_data( user_id_to_user_map=user_id_to_user_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, zerver_stream=zerver_stream, @@ -593,11 +641,20 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: subscriber_handler=subscriber_handler, ) + zerver_huddle = convert_huddle_data( + huddle_id_to_huddle_map=huddle_id_to_huddle_map, + huddle_id_mapper=huddle_id_mapper, + user_id_mapper=user_id_mapper, + subscriber_handler=subscriber_handler, + ) + realm["zerver_huddle"] = zerver_huddle + all_users = user_handler.get_all_users() zerver_recipient = build_recipients( zerver_userprofile=all_users, zerver_stream=zerver_stream, + zerver_huddle=zerver_huddle, ) realm["zerver_recipient"] = zerver_recipient @@ -607,11 +664,17 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: zerver_stream=zerver_stream, ) + huddle_subscriptions = build_huddle_subscriptions( + get_users=subscriber_handler.get_users, + zerver_recipient=zerver_recipient, + zerver_huddle=zerver_huddle, + ) + personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) - zerver_subscription = personal_subscriptions + stream_subscriptions + zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions realm["zerver_subscription"] = zerver_subscription subscriber_map = make_subscriber_map( @@ -619,11 +682,13 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: ) stream_id_to_recipient_id: Dict[int, int] = {} + huddle_id_to_recipient_id: Dict[int, int] = {} user_id_to_recipient_id: Dict[int, int] = {} map_receiver_id_to_recipient_id( zerver_recipient=zerver_recipient, stream_id_to_recipient_id=stream_id_to_recipient_id, + huddle_id_to_recipient_id=huddle_id_to_recipient_id, user_id_to_recipient_id=user_id_to_recipient_id, ) @@ -633,6 +698,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: separate_channel_and_private_messages( messages=rocketchat_data["message"], direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, channel_messages=channel_messages, private_messages=private_messages, ) @@ -651,8 +717,11 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: user_id_to_recipient_id=user_id_to_recipient_id, stream_id_mapper=stream_id_mapper, stream_id_to_recipient_id=stream_id_to_recipient_id, - direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_mapper=huddle_id_mapper, + huddle_id_to_recipient_id=huddle_id_to_recipient_id, dsc_id_to_dsc_map=dsc_id_to_dsc_map, + direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, total_reactions=total_reactions, output_dir=output_dir, ) @@ -668,8 +737,11 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: user_id_to_recipient_id=user_id_to_recipient_id, stream_id_mapper=stream_id_mapper, stream_id_to_recipient_id=stream_id_to_recipient_id, - direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_mapper=huddle_id_mapper, + huddle_id_to_recipient_id=huddle_id_to_recipient_id, dsc_id_to_dsc_map=dsc_id_to_dsc_map, + direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, total_reactions=total_reactions, output_dir=output_dir, ) diff --git a/zerver/tests/fixtures/rocketchat_fixtures/rocketchat_message.bson b/zerver/tests/fixtures/rocketchat_fixtures/rocketchat_message.bson index 81f228e1c91bc076be47c8b9ab5ca6241b2f34bb..97739d28a00404291110009c813698b60617bed7 100644 GIT binary patch delta 1028 zcmb_a&ubGw6rLtV6R1=JQW2>W5TwOOtoeC!o1}@^q|mOVn~Ic8c9Lwe+1>1JHtvGt zY%d~n@*nUho&{0QB6{bAwcWc|B>eR}gx)b63u3x8Ak0qvwp|Mbr8a08ofWy+1} zQnKqoW1f5yQEy6#fp`#$R9$YN-`d zbE<23qg!c-CE(Ndoz5^tp`f($1*^f^3p}3yzOK;(lLVrOY^WhslTpMqRh%DgITQHP zR44G9hWNl$0%<0$s=BVqjFRirRnk!yas~kFrH0n$`9d;nF_lijD)txi|D+Zq5QLg- nM@F<(-&VFA96KilWH&lsb(i!JR;wKm9+)W-0MC&XRbf;0dOP_tXs zQGZ}mwzvALqzfYZF}HAA?9n!C@P=EM`Zj56c1ydWH4)Kz6B*4V8qz$Btei`nP-}J& zA=uFn<>})f=k62i+BbGnyNFVu$xoS?4dKNtABN7F8#%85Oc`ylVc47Y9 zz5g|wYjRVZ-ed#*c6cvF2CxG;JD#8ZH6=VAXU^mVc1>2KLb-l4Zp>a1(lYXz9eao^ z5TG18#-5(aj^EDxOkR8x3miMB>ors*-OcK-Wrmkcvjn0Kya6nb0*XC?xMQ%vzNrf= zGjp~g7D3@v&n5YNjHrV{fK=2-*WG~lkl@7>cb~>0k#$e*zl$kKfo)N^&g)`0@M@9LSnDIaN?WY{qs$fhtA@2Iic^;u7D~ z;^M^gR0ihB2Apb>+1P{>IZKKeCP9@jwkxa1e*D&(f7mSpCoPj2M1pL~W> zY;uZ#(B!}TaypY?s^c?L7=(bT-78I#*OoER85b8^9^Zr;eQ$i~(T iahpHC None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) @@ -218,6 +226,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], @@ -225,6 +234,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( @@ -272,7 +282,7 @@ class RocketChatImporter(ZulipTestCase): self.assertEqual(zerver_stream[5]["stream_post_policy"], 1) self.assertEqual(zerver_stream[5]["realm"], realm_id) - def test_convert_subscription_data(self) -> None: + def test_convert_stream_subscription_data(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) @@ -298,6 +308,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], @@ -305,6 +316,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( @@ -314,7 +326,7 @@ class RocketChatImporter(ZulipTestCase): realm_id=realm_id, ) - convert_subscription_data( + convert_stream_subscription_data( user_id_to_user_map=user_id_to_user_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, zerver_stream=zerver_stream, @@ -362,7 +374,7 @@ class RocketChatImporter(ZulipTestCase): realm_id=realm_id, ) - convert_subscription_data( + convert_stream_subscription_data( user_id_to_user_map=user_id_to_user_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, zerver_stream=zerver_stream, @@ -374,7 +386,7 @@ class RocketChatImporter(ZulipTestCase): self.assert_length(subscriber_handler.get_users(stream_id=zerver_stream[6]["id"]), 0) self.assertTrue(zerver_stream[6]["deactivated"]) - def test_map_receiver_id_to_recipient_id(self) -> None: + def test_convert_huddle_data(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) @@ -382,8 +394,9 @@ class RocketChatImporter(ZulipTestCase): domain_name = "zulip.com" user_handler = UserHandler() + subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() - stream_id_mapper = IdMapper() + huddle_id_mapper = IdMapper() user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) @@ -399,6 +412,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], @@ -406,6 +420,60 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, + ) + + zerver_huddle = convert_huddle_data( + huddle_id_to_huddle_map=huddle_id_to_huddle_map, + huddle_id_mapper=huddle_id_mapper, + user_id_mapper=user_id_mapper, + subscriber_handler=subscriber_handler, + ) + + self.assert_length(zerver_huddle, 1) + + rc_huddle_id = rocketchat_data["room"][12]["_id"] + self.assertTrue(huddle_id_mapper.has(rc_huddle_id)) + + huddle_id = huddle_id_mapper.get(rc_huddle_id) + self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id), {3, 4, 5}) + + def test_map_receiver_id_to_recipient_id(self) -> None: + fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") + rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) + + realm_id = 3 + domain_name = "zulip.com" + + user_handler = UserHandler() + subscriber_handler = SubscriberHandler() + user_id_mapper = IdMapper() + stream_id_mapper = IdMapper() + huddle_id_mapper = IdMapper() + + user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) + + process_users( + user_id_to_user_map=user_id_to_user_map, + realm_id=realm_id, + domain_name=domain_name, + user_handler=user_handler, + user_id_mapper=user_id_mapper, + ) + + room_id_to_room_map: Dict[str, Dict[str, Any]] = {} + team_id_to_team_map: Dict[str, Dict[str, Any]] = {} + dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} + direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} + + categorize_channels_and_map_with_id( + channel_data=rocketchat_data["room"], + room_id_to_room_map=room_id_to_room_map, + team_id_to_team_map=team_id_to_team_map, + dsc_id_to_dsc_map=dsc_id_to_dsc_map, + direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( @@ -415,26 +483,37 @@ class RocketChatImporter(ZulipTestCase): realm_id=realm_id, ) + zerver_huddle = convert_huddle_data( + huddle_id_to_huddle_map=huddle_id_to_huddle_map, + huddle_id_mapper=huddle_id_mapper, + user_id_mapper=user_id_mapper, + subscriber_handler=subscriber_handler, + ) + all_users = user_handler.get_all_users() zerver_recipient = build_recipients( zerver_userprofile=all_users, zerver_stream=zerver_stream, + zerver_huddle=zerver_huddle, ) stream_id_to_recipient_id: Dict[int, int] = {} user_id_to_recipient_id: Dict[int, int] = {} + huddle_id_to_recipient_id: Dict[int, int] = {} map_receiver_id_to_recipient_id( zerver_recipient=zerver_recipient, stream_id_to_recipient_id=stream_id_to_recipient_id, user_id_to_recipient_id=user_id_to_recipient_id, + huddle_id_to_recipient_id=huddle_id_to_recipient_id, ) # 6 for streams and 6 for users. - self.assert_length(zerver_recipient, 12) + self.assert_length(zerver_recipient, 13) self.assert_length(stream_id_to_recipient_id, 6) self.assert_length(user_id_to_recipient_id, 6) + self.assert_length(huddle_id_to_recipient_id, 1) # First user recipients are built, followed by stream recipients in `build_recipients`. self.assertEqual( @@ -451,6 +530,10 @@ class RocketChatImporter(ZulipTestCase): stream_id_to_recipient_id[zerver_recipient[7]["type_id"]], zerver_recipient[7]["id"] ) + self.assertEqual( + huddle_id_to_recipient_id[zerver_recipient[12]["type_id"]], zerver_recipient[12]["id"] + ) + def test_separate_channel_and_private_messages(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) @@ -459,6 +542,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} + huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], @@ -466,6 +550,7 @@ class RocketChatImporter(ZulipTestCase): team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) channel_messages: List[Dict[str, Any]] = [] @@ -474,13 +559,14 @@ class RocketChatImporter(ZulipTestCase): separate_channel_and_private_messages( messages=rocketchat_data["message"], direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, channel_messages=channel_messages, private_messages=private_messages, ) - self.assert_length(rocketchat_data["message"], 44) - self.assert_length(channel_messages, 41) - self.assert_length(private_messages, 3) + self.assert_length(rocketchat_data["message"], 52) + self.assert_length(channel_messages, 47) + self.assert_length(private_messages, 5) self.assertIn(rocketchat_data["message"][0], channel_messages) self.assertIn(rocketchat_data["message"][1], channel_messages) @@ -488,7 +574,7 @@ class RocketChatImporter(ZulipTestCase): self.assertIn(rocketchat_data["message"][11], private_messages) self.assertIn(rocketchat_data["message"][12], private_messages) - self.assertIn(rocketchat_data["message"][13], private_messages) + self.assertIn(rocketchat_data["message"][50], private_messages) # Huddle message # Add a message with no `rid` rocketchat_data["message"].append( @@ -520,13 +606,14 @@ class RocketChatImporter(ZulipTestCase): separate_channel_and_private_messages( messages=rocketchat_data["message"], direct_id_to_direct_map=direct_id_to_direct_map, + huddle_id_to_huddle_map=huddle_id_to_huddle_map, channel_messages=channel_messages, private_messages=private_messages, ) # No new message added to channel or private messages - self.assert_length(channel_messages, 41) - self.assert_length(private_messages, 3) + self.assert_length(channel_messages, 47) + self.assert_length(private_messages, 5) def test_build_reactions(self) -> None: total_reactions: List[ZerverFieldsT] = [] @@ -648,16 +735,16 @@ class RocketChatImporter(ZulipTestCase): self.assert_length(realm["zerver_defaultstream"], 0) exported_recipient_ids = self.get_set(realm["zerver_recipient"], "id") - self.assert_length(exported_recipient_ids, 12) + self.assert_length(exported_recipient_ids, 13) exported_recipient_types = self.get_set(realm["zerver_recipient"], "type") - self.assertEqual(exported_recipient_types, {1, 2}) + self.assertEqual(exported_recipient_types, {1, 2, 3}) exported_subscription_userprofile = self.get_set( realm["zerver_subscription"], "user_profile" ) self.assert_length(exported_subscription_userprofile, 6) exported_subscription_recipients = self.get_set(realm["zerver_subscription"], "recipient") - self.assert_length(exported_subscription_recipients, 12) + self.assert_length(exported_subscription_recipients, 13) messages = self.read_file(output_dir, "messages-000001.json") @@ -698,15 +785,22 @@ class RocketChatImporter(ZulipTestCase): self.assertIsNotNone(message.rendered_content) # After removing user_joined, added_user, discussion_created, etc. # messages. (Total messages were 44.) - self.assert_length(messages, 24) + self.assert_length(messages, 27) stream_messages = messages.filter(recipient__type=Recipient.STREAM).order_by("date_sent") stream_recipients = stream_messages.values_list("recipient", flat=True) - self.assert_length(stream_messages, 21) + self.assert_length(stream_messages, 22) self.assert_length(set(stream_recipients), 5) self.assertEqual(stream_messages[0].sender.email, "priyansh3133@email.com") self.assertEqual(stream_messages[0].content, "Hey everyone, how's it going??") + huddle_messages = messages.filter(recipient__type=Recipient.HUDDLE).order_by("date_sent") + huddle_recipients = huddle_messages.values_list("recipient", flat=True) + self.assert_length(huddle_messages, 2) + self.assert_length(set(huddle_recipients), 1) + self.assertEqual(huddle_messages[0].sender.email, "hermionegranger@email.com") + self.assertEqual(huddle_messages[0].content, "Hey people!") + personal_messages = messages.filter(recipient__type=Recipient.PERSONAL).order_by( "date_sent" )