slack importer: Fetch and save the avatars.

This gets the avatar of size 512 px and saves it in the
user's avatar directory with both the extensions
'.png' and '.original'.

zerver/lib/slack_data_to_zulip_data.py

@@ -18,6 +18,7 @@ from zerver.models import UserProfile, Realm, Stream, UserMessage, \
 from zerver.forms import check_subdomain_available
 from zerver.lib.slack_message_conversion import convert_to_zulip_markdown, \
     get_user_full_name
+from zerver.lib.avatar_hash import user_avatar_path_from_ids
 
 # stubs
 ZerverFieldsT = Dict[str, Any]
@@ -660,6 +661,12 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str,
                                                     added_users, added_recipient, added_channels,
                                                     realm)
 
+    avatar_folder = os.path.join(output_dir, 'avatars')
+    avatar_realm_folder = os.path.join(avatar_folder, str(REALM_ID))
+
+    os.makedirs(avatar_realm_folder, exist_ok=True)
+    avatar_records = process_avatars(avatar_list, avatar_folder, REALM_ID)
+
     zerver_attachment = []  # type: List[ZerverFieldsT]
     attachment = {"zerver_attachment": zerver_attachment}
 
@@ -668,7 +675,7 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str,
     # IO message.json
     create_converted_data_files(message_json, output_dir, '/messages-000001.json', False)
     # IO avatar records
-    create_converted_data_files([], output_dir, '/avatars/records.json', True)
+    create_converted_data_files(avatar_records, output_dir, '/avatars/records.json', False)
     # IO uploads TODO
     create_converted_data_files([], output_dir, '/uploads/records.json', True)
     # IO attachments
@@ -681,6 +688,45 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str,
     logging.info('######### DATA CONVERSION FINISHED #########\n')
     logging.info("Zulip data dump created at %s" % (output_dir))
 
+def process_avatars(avatar_list: List[ZerverFieldsT], avatar_dir: str,
+                    realm_id: int) -> List[ZerverFieldsT]:
+    """
+    This function gets the avatar of size 512 px and saves it in the
+    user's avatar directory with both the extensions
+    '.png' and '.original'
+    """
+    logging.info('######### GETTING AVATARS #########\n')
+    avatar_original_list = []
+    for avatar in avatar_list:
+        avatar_hash = user_avatar_path_from_ids(avatar['user_profile_id'], realm_id)
+        slack_avatar_url = avatar['path']
+        avatar_original = dict(avatar)
+
+        image_path = ('%s/%s.png' % (avatar_dir, avatar_hash))
+        original_image_path = ('%s/%s.original' % (avatar_dir, avatar_hash))
+
+        # Fetch the avatars from the url
+        get_avatar(slack_avatar_url, image_path, original_image_path)
+        image_size = os.stat(image_path).st_size
+
+        avatar['path'] = image_path
+        avatar['s3_path'] = image_path
+        avatar['size'] = image_size
+
+        avatar_original['path'] = original_image_path
+        avatar_original['s3_path'] = original_image_path
+        avatar_original['size'] = image_size
+        avatar_original_list.append(avatar_original)
+    logging.info('######### GETTING AVATARS FINISHED #########\n')
+    return avatar_list + avatar_original_list
+
+def get_avatar(slack_avatar_url: str, image_path: str, original_image_path: str) -> None:
+    # get avatar of size 512
+    response = requests.get(slack_avatar_url + '-512', stream=True)
+    with open(image_path, 'wb') as image_file:
+        shutil.copyfileobj(response.raw, image_file)
+    shutil.copy(image_path, original_image_path)
+
 def get_data_file(path: str) -> Any:
     data = json.load(open(path))
     return data

zerver/tests/test_slack_importer.py

@@ -23,10 +23,14 @@ from zerver.lib.slack_data_to_zulip_data import (
     channel_message_to_zerver_message,
     convert_slack_workspace_messages,
     do_convert_data,
+    process_avatars,
 )
 from zerver.lib.export import (
     do_import_realm,
 )
+from zerver.lib.avatar_hash import (
+    user_avatar_path_from_ids,
+)
 from zerver.lib.test_classes import (
     ZulipTestCase,
 )
@@ -95,7 +99,9 @@ class SlackImporter(ZulipTestCase):
         self.assertEqual(test_zerver_realm_dict['name'], realm_subdomain)
         self.assertEqual(test_zerver_realm_dict['date_created'], time)
 
-    def test_user_avatars(self) -> None:
+    @mock.patch("zerver.lib.slack_data_to_zulip_data.get_avatar")
+    @mock.patch("os.stat")
+    def test_user_avatars(self, mock_get_avatar: mock.Mock, mock_os: mock.Mock) -> None:
         avatar_url = "https://ca.slack-edge.com/{}-{}-{}".format('T5YFFM2QY', 'U6006P1CN',
                                                                  'gd41c3c33cbe')
         self.assertEqual(build_avatar_url('U6006P1CN', 'T5YFFM2QY', 'gd41c3c33cbe'), avatar_url)
@@ -107,6 +113,14 @@ class SlackImporter(ZulipTestCase):
         self.assertEqual(test_avatar_list[0]['s3_path'], '')
         self.assertEqual(test_avatar_list[0]['user_profile_id'], 1)
 
+        avatar_list = process_avatars(test_avatar_list, './avatar_dir', 3)
+        avatar_hash = user_avatar_path_from_ids(1, 3)
+        image_path = ('%s/%s.png' % ('./avatar_dir', avatar_hash))
+        original_image_path = ('%s/%s.original' % ('./avatar_dir', avatar_hash))
+
+        self.assertEqual(avatar_list[0]['path'], image_path)
+        self.assertEqual(avatar_list[1]['path'], original_image_path)
+
     def test_get_admin(self) -> None:
         user_data = [{'is_admin': True, 'is_owner': False, 'is_primary_owner': False},
                      {'is_admin': True, 'is_owner': True, 'is_primary_owner': False},