From 00f822a26a9fd746b3b983296b4eb8897945dd9d Mon Sep 17 00:00:00 2001 From: Steve Howell Date: Sun, 28 Oct 2018 17:32:15 +0000 Subject: [PATCH] conversion: Generate attachment_ids with helpers. --- zerver/data_import/hipchat_attachment.py | 3 --- zerver/data_import/import_util.py | 4 +++- zerver/data_import/slack.py | 19 ++++++++----------- zerver/tests/test_slack_importer.py | 4 ++-- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/zerver/data_import/hipchat_attachment.py b/zerver/data_import/hipchat_attachment.py index 3d9893d62b..2b8dd0b192 100644 --- a/zerver/data_import/hipchat_attachment.py +++ b/zerver/data_import/hipchat_attachment.py @@ -9,8 +9,6 @@ from zerver.data_import.import_util import ( from typing import Any, Dict, List, Optional -from zerver.data_import.sequencer import NEXT_ID - class AttachmentHandler: def __init__(self) -> None: self.info_dict = dict() # type: Dict[str, Dict[str, Any]] @@ -75,7 +73,6 @@ class AttachmentHandler: build_attachment( realm_id=realm_id, message_ids=info['message_ids'], - attachment_id=NEXT_ID('attachment'), user_id=info['sender_id'], fileinfo=dict( created=info['mtime'], # minor lie diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py index 3402ac25f9..8cc97a9598 100644 --- a/zerver/data_import/import_util.py +++ b/zerver/data_import/import_util.py @@ -348,13 +348,15 @@ def build_message(subject: str, pub_date: float, message_id: int, content: str, return zulip_message_dict -def build_attachment(realm_id: int, message_ids: Set[int], attachment_id: int, +def build_attachment(realm_id: int, message_ids: Set[int], user_id: int, fileinfo: ZerverFieldsT, s3_path: str, zerver_attachment: List[ZerverFieldsT]) -> None: """ This function should be passed a 'fileinfo' dictionary, which contains information about 'size', 'created' (created time) and ['name'] (filename). """ + attachment_id = NEXT_ID('attachment') + attachment = Attachment( id=attachment_id, size=fileinfo['size'], diff --git a/zerver/data_import/slack.py b/zerver/data_import/slack.py index 8ff90b0ec6..9fefbd7020 100755 --- a/zerver/data_import/slack.py +++ b/zerver/data_import/slack.py @@ -459,8 +459,8 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel total_attachments = [] # type: List[ZerverFieldsT] total_uploads = [] # type: List[ZerverFieldsT] - message_id = reaction_id = attachment_id = 0 - id_list = (message_id, reaction_id, attachment_id) + message_id = reaction_id = 0 + id_list = (message_id, reaction_id) # The messages are stored in batches low_index = 0 @@ -521,13 +521,13 @@ def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT], zerver_realmemoji: List[ZerverFieldsT], subscriber_map: Dict[int, Set[int]], added_channels: AddedChannelsT, - id_list: Tuple[int, int, int], + id_list: Tuple[int, int], domain_name: str) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], List[ZerverFieldsT], List[ZerverFieldsT], List[ZerverFieldsT], - Tuple[int, int, int]]: + Tuple[int, int]]: """ Returns: 1. zerver_message, which is a list of the messages @@ -535,9 +535,9 @@ def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT], 3. zerver_attachment, which is a list of the attachments 4. uploads_list, which is a list of uploads to be mapped in uploads records.json 5. reaction_list, which is a list of all user reactions - 6. id_list, which is a tuple of max ids of messages, reactions and attachments + 6. id_list, which is a tuple of max ids of messages, reactions """ - message_id_count, reaction_id_count, attachment_id_count = id_list + message_id_count, reaction_id_count = id_list zerver_message = [] zerver_usermessage = [] # type: List[ZerverFieldsT] uploads_list = [] # type: List[ZerverFieldsT] @@ -617,10 +617,8 @@ def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT], build_uploads(added_users[user], realm_id, file_user_email, fileinfo, s3_path, uploads_list) - attachment_id = attachment_id_count - build_attachment(realm_id, {message_id}, attachment_id, added_users[user], + build_attachment(realm_id, {message_id}, added_users[user], fileinfo, s3_path, zerver_attachment) - attachment_id_count += 1 # For attachments with link not from slack # Example: Google drive integration else: @@ -650,8 +648,7 @@ def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT], message_id_count += 1 - id_list = (message_id_count, - reaction_id_count, attachment_id_count) + id_list = (message_id_count, reaction_id_count) return zerver_message, zerver_usermessage, zerver_attachment, uploads_list, \ reaction_list, id_list diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 6c3f20fc20..18fac4243a 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -444,7 +444,7 @@ class SlackImporter(ZulipTestCase): reaction, id_list = channel_message_to_zerver_message( 1, user_data, added_users, added_recipient, all_messages, [], subscriber_map, - added_channels, (0, 0, 0), 'domain') + added_channels, (0, 0), 'domain') # functioning already tested in helper function self.assertEqual(zerver_usermessage, []) # subtype: channel_join is filtered @@ -452,7 +452,7 @@ class SlackImporter(ZulipTestCase): self.assertEqual(uploads, []) self.assertEqual(attachment, []) - self.assertEqual(id_list, (5, 1, 0)) + self.assertEqual(id_list, (5, 1)) # Test reactions self.assertEqual(reaction[0]['user_profile'], 24)