Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

slack importer: Add function to precompute all the messages. 

The messages were first being read and passed to the helper
functions channel wise.
This function makes a list of all the messages in the all the channels
beforehand which would be used to pass in the helper functions.
This commit is contained in:
Rhea Parekh 2018-02-25 11:38:28 +05:30 committed by Tim Abbott
parent 59b8f85c63
commit 10c73ae577
2 changed files with 29 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
zerver/lib/slack_data_to_zulip_data.py
View File

@ -479,15 +479,17 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
message_json = {}
zerver_message = [] # type: List[ZerverFieldsT]
zerver_usermessage = [] # type: List[ZerverFieldsT]
all_messages = get_all_messages(slack_data_dir, added_channels)
logging.info('######### IMPORTING MESSAGES STARTED #########\n')
# To pre-compute the total number of messages and usermessages
total_messages = 0
total_usermessages = 0
for channel in added_channels.keys():
tm, tum = get_total_messages_and_usermessages(slack_data_dir, channel,
realm['zerver_subscription'],
added_recipient)
added_recipient, all_messages)
total_messages += tm
total_usermessages += tum
message_id_list = allocate_ids(Message, total_messages)
@ -502,6 +504,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
id_list = [message_id, usermessage_id, message_id_list, usermessage_id_list]
zm, zum = channel_message_to_zerver_message(constants, channel, users,
added_users, added_recipient,
all_messages,
realm['zerver_subscription'],
id_list)
zerver_message += zm
@ -513,9 +516,24 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
return message_json
def get_all_messages(slack_data_dir: str, added_channels: AddedChannelsT) -> List[ZerverFieldsT]:
all_messages = [] # type: List[ZerverFieldsT]
for channel_name in added_channels.keys():
channel_dir = os.path.join(slack_data_dir, channel_name)
json_names = os.listdir(channel_dir)
for json_name in json_names:
message_dir = os.path.join(channel_dir, json_name)
messages = get_data_file(message_dir)
for message in messages:
# To give every message the channel information
message['channel_name'] = channel_name
all_messages += messages
return all_messages
def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str,
zerver_subscription: List[ZerverFieldsT],
added_recipient: AddedRecipientsT) -> Tuple[int, int]:
added_recipient: AddedRecipientsT,
all_messages: List[ZerverFieldsT]) -> Tuple[int, int]:
"""
Returns:
1. message_id, which is total number of messages
@ -543,6 +561,7 @@ def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str,
def channel_message_to_zerver_message(constants: List[Any], channel: str,
users: List[ZerverFieldsT], added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
all_messages: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
ids: List[Any]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT]]:

10
zerver/tests/test_slack_importer.py
View File

@ -365,11 +365,13 @@ class SlackImporter(ZulipTestCase):
added_recipient = {'random': 2}
zerver_subscription = [{'recipient': 2}, {'recipient': 4}, {'recipient': 2}]
all_messages = [] # type: List[Dict[str, Any]]
total_messages, total_usermessages = get_total_messages_and_usermessages('./path',
'random',
zerver_subscription,
added_recipient)
added_recipient,
all_messages)
# subtype: channel_join, channel_leave are filtered out
self.assertEqual(total_messages, 4)
self.assertEqual(total_usermessages, 8)
@ -451,9 +453,11 @@ class SlackImporter(ZulipTestCase):
zerver_usermessage = [] # type: List[Dict[str, Any]]
zerver_subscription = [] # type: List[Dict[str, Any]]
all_messages = [] # type: List[Dict[str,Any]]
zerver_message, zerver_usermessage = channel_message_to_zerver_message(constants, channel_name,
user_data, added_users,
added_recipient,
all_messages,
zerver_subscription, ids)
# functioning already tested in helper function
self.assertEqual(zerver_usermessage, [])
@ -484,9 +488,11 @@ class SlackImporter(ZulipTestCase):
@mock.patch("zerver.lib.slack_data_to_zulip_data.channel_message_to_zerver_message")
@mock.patch("zerver.lib.slack_data_to_zulip_data.allocate_ids")
@mock.patch("zerver.lib.slack_data_to_zulip_data.get_all_messages")
@mock.patch("zerver.lib.slack_data_to_zulip_data.get_total_messages_and_usermessages", return_value=[1, 2])
def test_convert_slack_workspace_messages(self, mock_get_total_messages_and_usermessages: mock.Mock,
mock_allocate_ids: mock.Mock, mock_message: mock.Mock) -> None:
mock_get_all_messages: mock.Mock, mock_allocate_ids: mock.Mock,
mock_message: mock.Mock) -> None:
added_channels = {'random': 1, 'general': 2}
zerver_message1 = [{'id': 1}]