From 10c73ae577c37d33103b6c86e47e9ff8a6428e6b Mon Sep 17 00:00:00 2001 From: Rhea Parekh Date: Sun, 25 Feb 2018 11:38:28 +0530 Subject: [PATCH] slack importer: Add function to precompute all the messages. The messages were first being read and passed to the helper functions channel wise. This function makes a list of all the messages in the all the channels beforehand which would be used to pass in the helper functions. --- zerver/lib/slack_data_to_zulip_data.py | 23 +++++++++++++++++++++-- zerver/tests/test_slack_importer.py | 10 ++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/zerver/lib/slack_data_to_zulip_data.py b/zerver/lib/slack_data_to_zulip_data.py index 1791798e13..da1b7b1152 100755 --- a/zerver/lib/slack_data_to_zulip_data.py +++ b/zerver/lib/slack_data_to_zulip_data.py @@ -479,15 +479,17 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel message_json = {} zerver_message = [] # type: List[ZerverFieldsT] zerver_usermessage = [] # type: List[ZerverFieldsT] + all_messages = get_all_messages(slack_data_dir, added_channels) logging.info('######### IMPORTING MESSAGES STARTED #########\n') # To pre-compute the total number of messages and usermessages + total_messages = 0 total_usermessages = 0 for channel in added_channels.keys(): tm, tum = get_total_messages_and_usermessages(slack_data_dir, channel, realm['zerver_subscription'], - added_recipient) + added_recipient, all_messages) total_messages += tm total_usermessages += tum message_id_list = allocate_ids(Message, total_messages) @@ -502,6 +504,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel id_list = [message_id, usermessage_id, message_id_list, usermessage_id_list] zm, zum = channel_message_to_zerver_message(constants, channel, users, added_users, added_recipient, + all_messages, realm['zerver_subscription'], id_list) zerver_message += zm @@ -513,9 +516,24 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel return message_json +def get_all_messages(slack_data_dir: str, added_channels: AddedChannelsT) -> List[ZerverFieldsT]: + all_messages = [] # type: List[ZerverFieldsT] + for channel_name in added_channels.keys(): + channel_dir = os.path.join(slack_data_dir, channel_name) + json_names = os.listdir(channel_dir) + for json_name in json_names: + message_dir = os.path.join(channel_dir, json_name) + messages = get_data_file(message_dir) + for message in messages: + # To give every message the channel information + message['channel_name'] = channel_name + all_messages += messages + return all_messages + def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str, zerver_subscription: List[ZerverFieldsT], - added_recipient: AddedRecipientsT) -> Tuple[int, int]: + added_recipient: AddedRecipientsT, + all_messages: List[ZerverFieldsT]) -> Tuple[int, int]: """ Returns: 1. message_id, which is total number of messages @@ -543,6 +561,7 @@ def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str, def channel_message_to_zerver_message(constants: List[Any], channel: str, users: List[ZerverFieldsT], added_users: AddedUsersT, added_recipient: AddedRecipientsT, + all_messages: List[ZerverFieldsT], zerver_subscription: List[ZerverFieldsT], ids: List[Any]) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT]]: diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 35877ec95e..659b587367 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -365,11 +365,13 @@ class SlackImporter(ZulipTestCase): added_recipient = {'random': 2} zerver_subscription = [{'recipient': 2}, {'recipient': 4}, {'recipient': 2}] + all_messages = [] # type: List[Dict[str, Any]] total_messages, total_usermessages = get_total_messages_and_usermessages('./path', 'random', zerver_subscription, - added_recipient) + added_recipient, + all_messages) # subtype: channel_join, channel_leave are filtered out self.assertEqual(total_messages, 4) self.assertEqual(total_usermessages, 8) @@ -451,9 +453,11 @@ class SlackImporter(ZulipTestCase): zerver_usermessage = [] # type: List[Dict[str, Any]] zerver_subscription = [] # type: List[Dict[str, Any]] + all_messages = [] # type: List[Dict[str,Any]] zerver_message, zerver_usermessage = channel_message_to_zerver_message(constants, channel_name, user_data, added_users, added_recipient, + all_messages, zerver_subscription, ids) # functioning already tested in helper function self.assertEqual(zerver_usermessage, []) @@ -484,9 +488,11 @@ class SlackImporter(ZulipTestCase): @mock.patch("zerver.lib.slack_data_to_zulip_data.channel_message_to_zerver_message") @mock.patch("zerver.lib.slack_data_to_zulip_data.allocate_ids") + @mock.patch("zerver.lib.slack_data_to_zulip_data.get_all_messages") @mock.patch("zerver.lib.slack_data_to_zulip_data.get_total_messages_and_usermessages", return_value=[1, 2]) def test_convert_slack_workspace_messages(self, mock_get_total_messages_and_usermessages: mock.Mock, - mock_allocate_ids: mock.Mock, mock_message: mock.Mock) -> None: + mock_get_all_messages: mock.Mock, mock_allocate_ids: mock.Mock, + mock_message: mock.Mock) -> None: added_channels = {'random': 1, 'general': 2} zerver_message1 = [{'id': 1}]