Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

data_import: Add huddle import support for Rocket.Chat.

This commit is contained in:
Priyansh Garg 2021-07-27 02:34:11 +05:30 committed by Tim Abbott
parent e68b1989b3
commit 044fe547d3
4 changed files with 206 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
zerver/data_import/rocketchat.py
View File

@ -10,6 +10,8 @@ from django.forms.models import model_to_dict
from zerver.data_import.import_util import (
SubscriberHandler,
ZerverFieldsT,
build_huddle,
build_huddle_subscriptions,
build_message,
build_personal_subscriptions,
build_realm,
@ -174,7 +176,7 @@ def convert_channel_data(
return streams
def convert_subscription_data(
def convert_stream_subscription_data(
user_id_to_user_map: Dict[str, Dict[str, Any]],
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
zerver_stream: List[ZerverFieldsT],
@ -211,6 +213,31 @@ def convert_subscription_data(
subscriber_handler.set_info(users=users, stream_id=stream["id"])
def convert_huddle_data(
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
huddle_id_mapper: IdMapper,
user_id_mapper: IdMapper,
subscriber_handler: SubscriberHandler,
) -> List[ZerverFieldsT]:
zerver_huddle: List[ZerverFieldsT] = []
for rc_huddle_id in huddle_id_to_huddle_map:
huddle_id = huddle_id_mapper.get(rc_huddle_id)
huddle = build_huddle(huddle_id)
zerver_huddle.append(huddle)
huddle_dict = huddle_id_to_huddle_map[rc_huddle_id]
huddle_user_ids = set()
for rc_user_id in huddle_dict["uids"]:
huddle_user_ids.add(user_id_mapper.get(rc_user_id))
subscriber_handler.set_info(
users=huddle_user_ids,
huddle_id=huddle_id,
)
return zerver_huddle
def build_reactions(
total_reactions: List[ZerverFieldsT],
reactions: List[Dict[str, Any]],
@ -334,8 +361,11 @@ def process_messages(
user_id_to_recipient_id: Dict[int, int],
stream_id_mapper: IdMapper,
stream_id_to_recipient_id: Dict[int, int],
direct_id_to_direct_map: Dict[str, Dict[str, Any]],
huddle_id_mapper: IdMapper,
huddle_id_to_recipient_id: Dict[int, int],
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
direct_id_to_direct_map: Dict[str, Dict[str, Any]],
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
total_reactions: List[ZerverFieldsT],
output_dir: str,
) -> None:
@ -373,15 +403,20 @@ def process_messages(
# Add recipient_id and topic to message_dict
if is_pm_data:
direct_channel_id = message["rid"]
rc_member_ids = direct_id_to_direct_map[direct_channel_id]["uids"]
if rc_sender_id == rc_member_ids[0]:
zulip_member_id = user_id_mapper.get(rc_member_ids[1])
message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id]
# Message is in a PM or a huddle.
rc_channel_id = message["rid"]
if rc_channel_id in huddle_id_to_huddle_map:
huddle_id = huddle_id_mapper.get(rc_channel_id)
message_dict["recipient_id"] = huddle_id_to_recipient_id[huddle_id]
else:
zulip_member_id = user_id_mapper.get(rc_member_ids[0])
message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id]
# PMs don't have topics, but topic_name field is required in `build_message`.
rc_member_ids = direct_id_to_direct_map[rc_channel_id]["uids"]
if rc_sender_id == rc_member_ids[0]:
zulip_member_id = user_id_mapper.get(rc_member_ids[1])
message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id]
else:
zulip_member_id = user_id_mapper.get(rc_member_ids[0])
message_dict["recipient_id"] = user_id_to_recipient_id[zulip_member_id]
# PMs and huddles don't have topics, but topic_name field is required in `build_message`.
message_dict["topic_name"] = ""
elif message["rid"] in dsc_id_to_dsc_map:
# Message is in a discussion
@ -438,16 +473,19 @@ def process_messages(
def separate_channel_and_private_messages(
messages: List[Dict[str, Any]],
direct_id_to_direct_map: Dict[str, Dict[str, Any]],
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
channel_messages: List[Dict[str, Any]],
private_messages: List[Dict[str, Any]],
) -> None:
direct_channels_list = direct_id_to_direct_map.keys()
private_channels_list = list(direct_id_to_direct_map.keys()) + list(
huddle_id_to_huddle_map.keys()
)
for message in messages:
if not message.get("rid"):
# Message does not belong to any channel (might be
# related to livechat), so ignore all such messages.
continue
if message["rid"] in direct_channels_list:
if message["rid"] in private_channels_list:
private_messages.append(message)
else:
channel_messages.append(message)
@ -456,12 +494,15 @@ def separate_channel_and_private_messages(
def map_receiver_id_to_recipient_id(
zerver_recipient: List[ZerverFieldsT],
stream_id_to_recipient_id: Dict[int, int],
huddle_id_to_recipient_id: Dict[int, int],
user_id_to_recipient_id: Dict[int, int],
) -> None:
# receiver_id represents stream_id/user_id
# receiver_id represents stream_id/huddle_id/user_id
for recipient in zerver_recipient:
if recipient["type"] == Recipient.STREAM:
stream_id_to_recipient_id[recipient["type_id"]] = recipient["id"]
elif recipient["type"] == Recipient.HUDDLE:
huddle_id_to_recipient_id[recipient["type_id"]] = recipient["id"]
elif recipient["type"] == Recipient.PERSONAL:
user_id_to_recipient_id[recipient["type_id"]] = recipient["id"]
@ -472,12 +513,16 @@ def categorize_channels_and_map_with_id(
team_id_to_team_map: Dict[str, Dict[str, Any]],
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
direct_id_to_direct_map: Dict[str, Dict[str, Any]],
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
) -> None:
for channel in channel_data:
if channel.get("prid"):
dsc_id_to_dsc_map[channel["_id"]] = channel
elif channel["t"] == "d":
direct_id_to_direct_map[channel["_id"]] = channel
if len(channel["uids"]) > 2:
huddle_id_to_huddle_map[channel["_id"]] = channel
else:
direct_id_to_direct_map[channel["_id"]] = channel
else:
room_id_to_room_map[channel["_id"]] = channel
if channel.get("teamMain") is True:
@ -553,6 +598,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
@ -566,6 +612,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
@ -573,6 +620,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
@ -583,8 +631,8 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
)
realm["zerver_stream"] = zerver_stream
# Add subscription data to subscriber handler
convert_subscription_data(
# Add stream subscription data to `subscriber_handler`
convert_stream_subscription_data(
user_id_to_user_map=user_id_to_user_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
zerver_stream=zerver_stream,
@ -593,11 +641,20 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
subscriber_handler=subscriber_handler,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
realm["zerver_huddle"] = zerver_huddle
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
realm["zerver_recipient"] = zerver_recipient
@ -607,11 +664,17 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
zerver_stream=zerver_stream,
)
huddle_subscriptions = build_huddle_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_huddle=zerver_huddle,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient,
)
zerver_subscription = personal_subscriptions + stream_subscriptions
zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
realm["zerver_subscription"] = zerver_subscription
subscriber_map = make_subscriber_map(
@ -619,11 +682,13 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
)
stream_id_to_recipient_id: Dict[int, int] = {}
huddle_id_to_recipient_id: Dict[int, int] = {}
user_id_to_recipient_id: Dict[int, int] = {}
map_receiver_id_to_recipient_id(
zerver_recipient=zerver_recipient,
stream_id_to_recipient_id=stream_id_to_recipient_id,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
user_id_to_recipient_id=user_id_to_recipient_id,
)
@ -633,6 +698,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
separate_channel_and_private_messages(
messages=rocketchat_data["message"],
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
channel_messages=channel_messages,
private_messages=private_messages,
)
@ -651,8 +717,11 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
user_id_to_recipient_id=user_id_to_recipient_id,
stream_id_mapper=stream_id_mapper,
stream_id_to_recipient_id=stream_id_to_recipient_id,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_mapper=huddle_id_mapper,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
total_reactions=total_reactions,
output_dir=output_dir,
)
@ -668,8 +737,11 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
user_id_to_recipient_id=user_id_to_recipient_id,
stream_id_mapper=stream_id_mapper,
stream_id_to_recipient_id=stream_id_to_recipient_id,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_mapper=huddle_id_mapper,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
total_reactions=total_reactions,
output_dir=output_dir,
)

BIN
zerver/tests/fixtures/rocketchat_fixtures/rocketchat_message.bson vendored
View File

Binary file not shown.

BIN
zerver/tests/fixtures/rocketchat_fixtures/rocketchat_room.bson vendored
View File

Binary file not shown.

136
zerver/tests/test_rocketchat_importer.py
View File

@ -9,7 +9,8 @@ from zerver.data_import.rocketchat import (
build_reactions,
categorize_channels_and_map_with_id,
convert_channel_data,
convert_subscription_data,
convert_huddle_data,
convert_stream_subscription_data,
do_convert_data,
map_receiver_id_to_recipient_id,
map_user_id_to_user,
@ -36,11 +37,11 @@ class RocketChatImporter(ZulipTestCase):
self.assertEqual(rocketchat_data["user"][2]["username"], "harry.potter")
self.assert_length(rocketchat_data["user"][2]["__rooms"], 10)
self.assert_length(rocketchat_data["room"], 12)
self.assert_length(rocketchat_data["room"], 13)
self.assertEqual(rocketchat_data["room"][0]["_id"], "GENERAL")
self.assertEqual(rocketchat_data["room"][0]["name"], "general")
self.assert_length(rocketchat_data["message"], 44)
self.assert_length(rocketchat_data["message"], 52)
self.assertEqual(rocketchat_data["message"][1]["msg"], "Hey everyone, how's it going??")
self.assertEqual(rocketchat_data["message"][1]["rid"], "GENERAL")
self.assertEqual(rocketchat_data["message"][1]["u"]["username"], "priyansh3133")
@ -175,6 +176,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
@ -182,14 +184,16 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
self.assert_length(rocketchat_data["room"], 12)
self.assert_length(rocketchat_data["room"], 13)
# Teams are a subset of rooms.
self.assert_length(room_id_to_room_map, 6)
self.assert_length(team_id_to_team_map, 1)
self.assert_length(dsc_id_to_dsc_map, 4)
self.assert_length(direct_id_to_direct_map, 2)
self.assert_length(huddle_id_to_huddle_map, 1)
room_id = rocketchat_data["room"][0]["_id"]
self.assertIn(room_id, room_id_to_room_map)
@ -207,6 +211,10 @@ class RocketChatImporter(ZulipTestCase):
self.assertIn(direct_id, direct_id_to_direct_map)
self.assertEqual(direct_id_to_direct_map[direct_id], rocketchat_data["room"][4])
huddle_id = rocketchat_data["room"][12]["_id"]
self.assertIn(huddle_id, huddle_id_to_huddle_map)
self.assertEqual(huddle_id_to_huddle_map[huddle_id], rocketchat_data["room"][12])
def test_convert_channel_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
@ -218,6 +226,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
@ -225,6 +234,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
@ -272,7 +282,7 @@ class RocketChatImporter(ZulipTestCase):
self.assertEqual(zerver_stream[5]["stream_post_policy"], 1)
self.assertEqual(zerver_stream[5]["realm"], realm_id)
def test_convert_subscription_data(self) -> None:
def test_convert_stream_subscription_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
@ -298,6 +308,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
@ -305,6 +316,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
@ -314,7 +326,7 @@ class RocketChatImporter(ZulipTestCase):
realm_id=realm_id,
)
convert_subscription_data(
convert_stream_subscription_data(
user_id_to_user_map=user_id_to_user_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
zerver_stream=zerver_stream,
@ -362,7 +374,7 @@ class RocketChatImporter(ZulipTestCase):
realm_id=realm_id,
)
convert_subscription_data(
convert_stream_subscription_data(
user_id_to_user_map=user_id_to_user_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
zerver_stream=zerver_stream,
@ -374,7 +386,7 @@ class RocketChatImporter(ZulipTestCase):
self.assert_length(subscriber_handler.get_users(stream_id=zerver_stream[6]["id"]), 0)
self.assertTrue(zerver_stream[6]["deactivated"])
def test_map_receiver_id_to_recipient_id(self) -> None:
def test_convert_huddle_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
@ -382,8 +394,9 @@ class RocketChatImporter(ZulipTestCase):
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
@ -399,6 +412,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
@ -406,6 +420,60 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
self.assert_length(zerver_huddle, 1)
rc_huddle_id = rocketchat_data["room"][12]["_id"]
self.assertTrue(huddle_id_mapper.has(rc_huddle_id))
huddle_id = huddle_id_mapper.get(rc_huddle_id)
self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id), {3, 4, 5})
def test_map_receiver_id_to_recipient_id(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
@ -415,26 +483,37 @@ class RocketChatImporter(ZulipTestCase):
realm_id=realm_id,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
stream_id_to_recipient_id: Dict[int, int] = {}
user_id_to_recipient_id: Dict[int, int] = {}
huddle_id_to_recipient_id: Dict[int, int] = {}
map_receiver_id_to_recipient_id(
zerver_recipient=zerver_recipient,
stream_id_to_recipient_id=stream_id_to_recipient_id,
user_id_to_recipient_id=user_id_to_recipient_id,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
)
# 6 for streams and 6 for users.
self.assert_length(zerver_recipient, 12)
self.assert_length(zerver_recipient, 13)
self.assert_length(stream_id_to_recipient_id, 6)
self.assert_length(user_id_to_recipient_id, 6)
self.assert_length(huddle_id_to_recipient_id, 1)
# First user recipients are built, followed by stream recipients in `build_recipients`.
self.assertEqual(
@ -451,6 +530,10 @@ class RocketChatImporter(ZulipTestCase):
stream_id_to_recipient_id[zerver_recipient[7]["type_id"]], zerver_recipient[7]["id"]
)
self.assertEqual(
huddle_id_to_recipient_id[zerver_recipient[12]["type_id"]], zerver_recipient[12]["id"]
)
def test_separate_channel_and_private_messages(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
@ -459,6 +542,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
@ -466,6 +550,7 @@ class RocketChatImporter(ZulipTestCase):
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
channel_messages: List[Dict[str, Any]] = []
@ -474,13 +559,14 @@ class RocketChatImporter(ZulipTestCase):
separate_channel_and_private_messages(
messages=rocketchat_data["message"],
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
channel_messages=channel_messages,
private_messages=private_messages,
)
self.assert_length(rocketchat_data["message"], 44)
self.assert_length(channel_messages, 41)
self.assert_length(private_messages, 3)
self.assert_length(rocketchat_data["message"], 52)
self.assert_length(channel_messages, 47)
self.assert_length(private_messages, 5)
self.assertIn(rocketchat_data["message"][0], channel_messages)
self.assertIn(rocketchat_data["message"][1], channel_messages)
@ -488,7 +574,7 @@ class RocketChatImporter(ZulipTestCase):
self.assertIn(rocketchat_data["message"][11], private_messages)
self.assertIn(rocketchat_data["message"][12], private_messages)
self.assertIn(rocketchat_data["message"][13], private_messages)
self.assertIn(rocketchat_data["message"][50], private_messages) # Huddle message
# Add a message with no `rid`
rocketchat_data["message"].append(
@ -520,13 +606,14 @@ class RocketChatImporter(ZulipTestCase):
separate_channel_and_private_messages(
messages=rocketchat_data["message"],
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
channel_messages=channel_messages,
private_messages=private_messages,
)
# No new message added to channel or private messages
self.assert_length(channel_messages, 41)
self.assert_length(private_messages, 3)
self.assert_length(channel_messages, 47)
self.assert_length(private_messages, 5)
def test_build_reactions(self) -> None:
total_reactions: List[ZerverFieldsT] = []
@ -648,16 +735,16 @@ class RocketChatImporter(ZulipTestCase):
self.assert_length(realm["zerver_defaultstream"], 0)
exported_recipient_ids = self.get_set(realm["zerver_recipient"], "id")
self.assert_length(exported_recipient_ids, 12)
self.assert_length(exported_recipient_ids, 13)
exported_recipient_types = self.get_set(realm["zerver_recipient"], "type")
self.assertEqual(exported_recipient_types, {1, 2})
self.assertEqual(exported_recipient_types, {1, 2, 3})
exported_subscription_userprofile = self.get_set(
realm["zerver_subscription"], "user_profile"
)
self.assert_length(exported_subscription_userprofile, 6)
exported_subscription_recipients = self.get_set(realm["zerver_subscription"], "recipient")
self.assert_length(exported_subscription_recipients, 12)
self.assert_length(exported_subscription_recipients, 13)
messages = self.read_file(output_dir, "messages-000001.json")
@ -698,15 +785,22 @@ class RocketChatImporter(ZulipTestCase):
self.assertIsNotNone(message.rendered_content)
# After removing user_joined, added_user, discussion_created, etc.
# messages. (Total messages were 44.)
self.assert_length(messages, 24)
self.assert_length(messages, 27)
stream_messages = messages.filter(recipient__type=Recipient.STREAM).order_by("date_sent")
stream_recipients = stream_messages.values_list("recipient", flat=True)
self.assert_length(stream_messages, 21)
self.assert_length(stream_messages, 22)
self.assert_length(set(stream_recipients), 5)
self.assertEqual(stream_messages[0].sender.email, "priyansh3133@email.com")
self.assertEqual(stream_messages[0].content, "Hey everyone, how's it going??")
huddle_messages = messages.filter(recipient__type=Recipient.HUDDLE).order_by("date_sent")
huddle_recipients = huddle_messages.values_list("recipient", flat=True)
self.assert_length(huddle_messages, 2)
self.assert_length(set(huddle_recipients), 1)
self.assertEqual(huddle_messages[0].sender.email, "hermionegranger@email.com")
self.assertEqual(huddle_messages[0].content, "Hey people!")
personal_messages = messages.filter(recipient__type=Recipient.PERSONAL).order_by(
"date_sent"
)