import os import shutil from io import BytesIO from typing import Any, Dict, Iterator, List, Set, Tuple from unittest import mock from unittest.mock import ANY from urllib.parse import parse_qs, urlparse import orjson import responses from django.conf import settings from django.utils.timezone import now as timezone_now from requests.models import PreparedRequest from zerver.data_import.import_util import ( ZerverFieldsT, build_defaultstream, build_recipient, build_subscription, build_usermessages, build_zerver_realm, ) from zerver.data_import.sequencer import NEXT_ID from zerver.data_import.slack import ( AddedChannelsT, AddedMPIMsT, DMMembersT, SlackBotEmail, channel_message_to_zerver_message, channels_to_zerver_stream, convert_slack_workspace_messages, do_convert_data, fetch_shared_channel_users, get_admin, get_guest, get_message_sending_user, get_owner, get_slack_api_data, get_subscription, get_user_timezone, process_message_files, slack_workspace_to_realm, users_to_zerver_userprofile, ) from zerver.lib.import_realm import do_import_realm from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import read_test_image_file from zerver.lib.topic import EXPORT_TOPIC_NAME from zerver.models import Realm, RealmAuditLog, Recipient, UserProfile, get_realm def remove_folder(path: str) -> None: if os.path.exists(path): shutil.rmtree(path) def request_callback(request: PreparedRequest) -> Tuple[int, Dict[str, str], bytes]: valid_endpoint = False endpoints = [ "https://slack.com/api/users.list", "https://slack.com/api/users.info", "https://slack.com/api/team.info", ] for endpoint in endpoints: if request.url and endpoint in request.url: valid_endpoint = True break if not valid_endpoint: return (404, {}, b"") if request.headers.get("Authorization") != "Bearer xoxp-valid-token": return (200, {}, orjson.dumps({"ok": False, "error": "invalid_auth"})) if request.url == "https://slack.com/api/users.list": return (200, {}, orjson.dumps({"ok": True, "members": "user_data"})) query_from_url = str(urlparse(request.url).query) qs = parse_qs(query_from_url) if request.url and "https://slack.com/api/users.info" in request.url: user2team_dict = { "U061A3E0G": "T6LARQE2Z", "U061A8H1G": "T7KJRQE8Y", "U8X25EBAB": "T5YFFM2QY", } try: user_id = qs["user"][0] team_id = user2team_dict[user_id] except KeyError: return (200, {}, orjson.dumps({"ok": False, "error": "user_not_found"})) return (200, {}, orjson.dumps({"ok": True, "user": {"id": user_id, "team_id": team_id}})) # Else, https://slack.com/api/team.info team_not_found: Tuple[int, Dict[str, str], bytes] = ( 200, {}, orjson.dumps({"ok": False, "error": "team_not_found"}), ) try: team_id = qs["team"][0] except KeyError: return team_not_found team_dict = { "T6LARQE2Z": "foreignteam1", "T7KJRQE8Y": "foreignteam2", } try: team_domain = team_dict[team_id] except KeyError: return team_not_found return (200, {}, orjson.dumps({"ok": True, "team": {"id": team_id, "domain": team_domain}})) class SlackImporter(ZulipTestCase): @responses.activate def test_get_slack_api_data(self) -> None: token = "xoxp-valid-token" # Users list slack_user_list_url = "https://slack.com/api/users.list" responses.add_callback(responses.GET, slack_user_list_url, callback=request_callback) self.assertEqual( get_slack_api_data(slack_user_list_url, "members", token=token), "user_data" ) # Users info slack_users_info_url = "https://slack.com/api/users.info" user_id = "U8X25EBAB" responses.add_callback(responses.GET, slack_users_info_url, callback=request_callback) self.assertEqual( get_slack_api_data(slack_users_info_url, "user", token=token, user=user_id), {"id": user_id, "team_id": "T5YFFM2QY"}, ) # Should error if the required user argument is not specified with self.assertRaises(Exception) as invalid: get_slack_api_data(slack_users_info_url, "user", token=token) self.assertEqual(invalid.exception.args, ("Error accessing Slack API: user_not_found",)) # Should error if the required user is not found with self.assertRaises(Exception) as invalid: get_slack_api_data(slack_users_info_url, "user", token=token, user="idontexist") self.assertEqual(invalid.exception.args, ("Error accessing Slack API: user_not_found",)) # Team info slack_team_info_url = "https://slack.com/api/team.info" responses.add_callback(responses.GET, slack_team_info_url, callback=request_callback) with self.assertRaises(Exception) as invalid: get_slack_api_data(slack_team_info_url, "team", token=token, team="wedontexist") self.assertEqual(invalid.exception.args, ("Error accessing Slack API: team_not_found",)) # Should error if the required user argument is not specified with self.assertRaises(Exception) as invalid: get_slack_api_data(slack_team_info_url, "team", token=token) self.assertEqual(invalid.exception.args, ("Error accessing Slack API: team_not_found",)) token = "xoxp-invalid-token" with self.assertRaises(Exception) as invalid: get_slack_api_data(slack_user_list_url, "members", token=token) self.assertEqual(invalid.exception.args, ("Error accessing Slack API: invalid_auth",)) with self.assertRaises(Exception) as invalid: get_slack_api_data(slack_user_list_url, "members") self.assertEqual(invalid.exception.args, ("Slack token missing in kwargs",)) token = "xoxp-status404" wrong_url = "https://slack.com/api/wrong" responses.add_callback(responses.GET, wrong_url, callback=request_callback) with self.assertRaises(Exception) as invalid: get_slack_api_data(wrong_url, "members", token=token) self.assertEqual(invalid.exception.args, ("HTTP error accessing the Slack API.",)) def test_build_zerver_realm(self) -> None: realm_id = 2 realm_subdomain = "test-realm" time = float(timezone_now().timestamp()) test_realm: List[Dict[str, Any]] = build_zerver_realm( realm_id, realm_subdomain, time, "Slack" ) test_zerver_realm_dict = test_realm[0] self.assertEqual(test_zerver_realm_dict["id"], realm_id) self.assertEqual(test_zerver_realm_dict["string_id"], realm_subdomain) self.assertEqual(test_zerver_realm_dict["name"], realm_subdomain) self.assertEqual(test_zerver_realm_dict["date_created"], time) def test_get_owner(self) -> None: user_data = [ {"is_owner": False, "is_primary_owner": False}, {"is_owner": True, "is_primary_owner": False}, {"is_owner": False, "is_primary_owner": True}, {"is_owner": True, "is_primary_owner": True}, ] self.assertEqual(get_owner(user_data[0]), False) self.assertEqual(get_owner(user_data[1]), True) self.assertEqual(get_owner(user_data[2]), True) self.assertEqual(get_owner(user_data[3]), True) def test_get_admin(self) -> None: user_data = [{"is_admin": True}, {"is_admin": False}] self.assertEqual(get_admin(user_data[0]), True) self.assertEqual(get_admin(user_data[1]), False) def test_get_guest(self) -> None: user_data = [ {"is_restricted": False, "is_ultra_restricted": False}, {"is_restricted": True, "is_ultra_restricted": False}, {"is_restricted": False, "is_ultra_restricted": True}, {"is_restricted": True, "is_ultra_restricted": True}, ] self.assertEqual(get_guest(user_data[0]), False) self.assertEqual(get_guest(user_data[1]), True) self.assertEqual(get_guest(user_data[2]), True) self.assertEqual(get_guest(user_data[3]), True) def test_get_timezone(self) -> None: user_chicago_timezone = {"tz": "America/Chicago"} user_timezone_none = {"tz": None} user_no_timezone: Dict[str, Any] = {} self.assertEqual(get_user_timezone(user_chicago_timezone), "America/Chicago") self.assertEqual(get_user_timezone(user_timezone_none), "America/New_York") self.assertEqual(get_user_timezone(user_no_timezone), "America/New_York") @mock.patch("zerver.data_import.slack.get_data_file") @mock.patch("zerver.data_import.slack.get_messages_iterator") @responses.activate def test_fetch_shared_channel_users( self, messages_mock: mock.Mock, data_file_mock: mock.Mock ) -> None: users = [{"id": "U061A1R2R"}, {"id": "U061A5N1G"}, {"id": "U064KUGRJ"}] data_file_mock.side_effect = [ [ {"name": "general", "members": ["U061A1R2R", "U061A5N1G"]}, {"name": "sharedchannel", "members": ["U061A1R2R", "U061A3E0G"]}, ], [], ] messages_mock.return_value = [ {"user": "U061A1R2R"}, {"user": "U061A5N1G"}, {"user": "U061A8H1G"}, ] # Users info slack_users_info_url = "https://slack.com/api/users.info" responses.add_callback(responses.GET, slack_users_info_url, callback=request_callback) # Team info slack_team_info_url = "https://slack.com/api/team.info" responses.add_callback(responses.GET, slack_team_info_url, callback=request_callback) slack_data_dir = self.fixture_file_name("", type="slack_fixtures") fetch_shared_channel_users(users, slack_data_dir, "xoxp-valid-token") # Normal users self.assert_length(users, 5) self.assertEqual(users[0]["id"], "U061A1R2R") self.assertEqual(users[0]["is_mirror_dummy"], False) self.assertFalse("team_domain" in users[0]) self.assertEqual(users[1]["id"], "U061A5N1G") self.assertEqual(users[2]["id"], "U064KUGRJ") # Shared channel users # We need to do this because the outcome order of `users` list is # not deterministic. fourth_fifth = [users[3], users[4]] fourth_fifth.sort(key=lambda x: x["id"]) self.assertEqual(fourth_fifth[0]["id"], "U061A3E0G") self.assertEqual(fourth_fifth[0]["team_domain"], "foreignteam1") self.assertEqual(fourth_fifth[0]["is_mirror_dummy"], True) self.assertEqual(fourth_fifth[1]["id"], "U061A8H1G") self.assertEqual(fourth_fifth[1]["team_domain"], "foreignteam2") self.assertEqual(fourth_fifth[1]["is_mirror_dummy"], True) @mock.patch("zerver.data_import.slack.get_data_file") def test_users_to_zerver_userprofile(self, mock_get_data_file: mock.Mock) -> None: custom_profile_field_user1 = { "Xf06054BBB": {"value": "random1"}, "Xf023DSCdd": {"value": "employee"}, } custom_profile_field_user2 = { "Xf06054BBB": {"value": "random2"}, "Xf023DSCdd": {"value": "employer"}, } user_data = [ { "id": "U08RGD1RD", "team_id": "T5YFFM2QY", "name": "john", "deleted": False, "is_mirror_dummy": False, "real_name": "John Doe", "profile": { "image_32": "", "email": "jon@gmail.com", "avatar_hash": "hash", "phone": "+1-123-456-77-868", "fields": custom_profile_field_user1, }, }, { "id": "U0CBK5KAT", "team_id": "T5YFFM2QY", "is_admin": True, "is_bot": False, "is_owner": True, "is_primary_owner": True, "name": "Jane", "real_name": "Jane Doe", "deleted": False, "is_mirror_dummy": False, "profile": { "image_32": "https://secure.gravatar.com/avatar/random.png", "fields": custom_profile_field_user2, "email": "jane@foo.com", "avatar_hash": "hash", }, }, { "id": "U09TYF5Sk", "team_id": "T5YFFM2QY", "name": "Bot", "real_name": "Bot", "is_bot": True, "deleted": False, "is_mirror_dummy": False, "profile": { "image_32": "https://secure.gravatar.com/avatar/random1.png", "skype": "test_skype_name", "email": "bot1@zulipchat.com", "avatar_hash": "hash", }, }, { "id": "UHSG7OPQN", "team_id": "T6LARQE2Z", "name": "matt.perry", "color": "9d8eee", "is_bot": False, "is_app_user": False, "is_mirror_dummy": True, "team_domain": "foreignteam", "profile": { "image_32": "https://secure.gravatar.com/avatar/random6.png", "avatar_hash": "hash", "first_name": "Matt", "last_name": "Perry", "real_name": "Matt Perry", "display_name": "matt.perry", "team": "T6LARQE2Z", }, }, { "id": "U8VAHEVUY", "team_id": "T5YFFM2QY", "name": "steviejacob34", "real_name": "Steve Jacob", "is_admin": False, "is_owner": False, "is_primary_owner": False, "is_restricted": True, "is_ultra_restricted": False, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "steviejacob34@yahoo.com", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random6.png", }, }, { "id": "U8X25EBAB", "team_id": "T5YFFM2QY", "name": "pratikweb_0", "real_name": "Pratik", "is_admin": False, "is_owner": False, "is_primary_owner": False, "is_restricted": True, "is_ultra_restricted": True, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "pratik@mit.edu", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random.png", }, }, { "id": "U015J7JSE", "team_id": "T5YFFM2QY", "name": "georgesm27", "real_name": "George", "is_admin": True, "is_owner": True, "is_primary_owner": False, "is_restricted": False, "is_ultra_restricted": False, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "george@yahoo.com", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random5.png", }, }, { "id": "U1RDFEC80", "team_id": "T5YFFM2QY", "name": "daniel.smith", "real_name": "Daniel Smith", "is_admin": True, "is_owner": False, "is_primary_owner": False, "is_restricted": False, "is_ultra_restricted": False, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "daniel@gmail.com", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random7.png", }, }, ] mock_get_data_file.return_value = user_data # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first # and hence has zulip_id = 1 test_slack_user_id_to_zulip_user_id = { "U08RGD1RD": 1, "U0CBK5KAT": 0, "U09TYF5Sk": 2, "UHSG7OPQN": 3, "U8VAHEVUY": 4, "U8X25EBAB": 5, "U015J7JSE": 6, "U1RDFEC80": 7, } slack_data_dir = "./random_path" timestamp = int(timezone_now().timestamp()) mock_get_data_file.return_value = user_data with self.assertLogs(level="INFO"): ( zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, customprofilefield, customprofilefield_value, ) = users_to_zerver_userprofile(slack_data_dir, user_data, 1, timestamp, "test_domain") # Test custom profile fields self.assertEqual(customprofilefield[0]["field_type"], 1) self.assertEqual(customprofilefield[3]["name"], "skype") cpf_name = {cpf["name"] for cpf in customprofilefield} self.assertIn("phone", cpf_name) self.assertIn("skype", cpf_name) cpf_name.remove("phone") cpf_name.remove("skype") for name in cpf_name: self.assertTrue(name.startswith("Slack custom field ")) self.assert_length(customprofilefield_value, 6) self.assertEqual(customprofilefield_value[0]["field"], 0) self.assertEqual(customprofilefield_value[0]["user_profile"], 1) self.assertEqual(customprofilefield_value[3]["user_profile"], 0) self.assertEqual(customprofilefield_value[5]["value"], "test_skype_name") # test that the primary owner should always be imported first self.assertDictEqual(slack_user_id_to_zulip_user_id, test_slack_user_id_to_zulip_user_id) self.assert_length(avatar_list, 8) self.assert_length(zerver_userprofile, 8) self.assertEqual(zerver_userprofile[0]["is_staff"], False) self.assertEqual(zerver_userprofile[0]["is_bot"], False) self.assertEqual(zerver_userprofile[0]["is_active"], True) self.assertEqual(zerver_userprofile[0]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[0]["role"], UserProfile.ROLE_MEMBER) self.assertEqual(zerver_userprofile[0]["enable_desktop_notifications"], True) self.assertEqual(zerver_userprofile[0]["email"], "jon@gmail.com") self.assertEqual(zerver_userprofile[0]["full_name"], "John Doe") self.assertEqual( zerver_userprofile[1]["id"], test_slack_user_id_to_zulip_user_id["U0CBK5KAT"] ) self.assertEqual(zerver_userprofile[1]["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(zerver_userprofile[1]["is_staff"], False) self.assertEqual(zerver_userprofile[1]["is_active"], True) self.assertEqual(zerver_userprofile[0]["is_mirror_dummy"], False) self.assertEqual( zerver_userprofile[2]["id"], test_slack_user_id_to_zulip_user_id["U09TYF5Sk"] ) self.assertEqual(zerver_userprofile[2]["is_bot"], True) self.assertEqual(zerver_userprofile[2]["is_active"], True) self.assertEqual(zerver_userprofile[2]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[2]["email"], "bot1@zulipchat.com") self.assertEqual(zerver_userprofile[2]["bot_type"], 1) self.assertEqual(zerver_userprofile[2]["avatar_source"], "U") self.assertEqual( zerver_userprofile[3]["id"], test_slack_user_id_to_zulip_user_id["UHSG7OPQN"] ) self.assertEqual(zerver_userprofile[3]["role"], UserProfile.ROLE_MEMBER) self.assertEqual(zerver_userprofile[3]["is_staff"], False) self.assertEqual(zerver_userprofile[3]["is_active"], False) self.assertEqual(zerver_userprofile[3]["email"], "matt.perry@foreignteam.slack.com") self.assertEqual(zerver_userprofile[3]["realm"], 1) self.assertEqual(zerver_userprofile[3]["full_name"], "Matt Perry") self.assertEqual(zerver_userprofile[3]["is_mirror_dummy"], True) self.assertEqual(zerver_userprofile[3]["can_forge_sender"], False) self.assertEqual( zerver_userprofile[4]["id"], test_slack_user_id_to_zulip_user_id["U8VAHEVUY"] ) self.assertEqual(zerver_userprofile[4]["role"], UserProfile.ROLE_GUEST) self.assertEqual(zerver_userprofile[4]["is_staff"], False) self.assertEqual(zerver_userprofile[4]["is_active"], True) self.assertEqual(zerver_userprofile[4]["is_mirror_dummy"], False) self.assertEqual( zerver_userprofile[5]["id"], test_slack_user_id_to_zulip_user_id["U8X25EBAB"] ) self.assertEqual(zerver_userprofile[5]["role"], UserProfile.ROLE_GUEST) self.assertEqual(zerver_userprofile[5]["is_staff"], False) self.assertEqual(zerver_userprofile[5]["is_active"], True) self.assertEqual(zerver_userprofile[5]["is_mirror_dummy"], False) self.assertEqual( zerver_userprofile[6]["id"], test_slack_user_id_to_zulip_user_id["U015J7JSE"] ) self.assertEqual(zerver_userprofile[6]["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(zerver_userprofile[6]["is_staff"], False) self.assertEqual(zerver_userprofile[6]["is_active"], True) self.assertEqual(zerver_userprofile[6]["is_mirror_dummy"], False) self.assertEqual( zerver_userprofile[7]["id"], test_slack_user_id_to_zulip_user_id["U1RDFEC80"] ) self.assertEqual(zerver_userprofile[7]["role"], UserProfile.ROLE_REALM_ADMINISTRATOR) self.assertEqual(zerver_userprofile[7]["is_staff"], False) self.assertEqual(zerver_userprofile[7]["is_active"], True) self.assertEqual(zerver_userprofile[7]["is_mirror_dummy"], False) def test_build_defaultstream(self) -> None: realm_id = 1 stream_id = 1 default_channel_general = build_defaultstream(realm_id, stream_id, 1) test_default_channel = {"stream": 1, "realm": 1, "id": 1} self.assertDictEqual(test_default_channel, default_channel_general) default_channel_general = build_defaultstream(realm_id, stream_id, 1) test_default_channel = {"stream": 1, "realm": 1, "id": 1} self.assertDictEqual(test_default_channel, default_channel_general) def test_build_pm_recipient_sub_from_user(self) -> None: zulip_user_id = 3 recipient_id = 5 subscription_id = 7 sub = build_subscription(recipient_id, zulip_user_id, subscription_id) recipient = build_recipient(zulip_user_id, recipient_id, Recipient.PERSONAL) self.assertEqual(recipient["id"], sub["recipient"]) self.assertEqual(recipient["type_id"], sub["user_profile"]) self.assertEqual(recipient["type"], Recipient.PERSONAL) self.assertEqual(recipient["type_id"], 3) self.assertEqual(sub["recipient"], 5) self.assertEqual(sub["id"], 7) self.assertEqual(sub["active"], True) def test_build_subscription(self) -> None: channel_members = ["U061A1R2R", "U061A3E0G", "U061A5N1G", "U064KUGRJ"] slack_user_id_to_zulip_user_id = { "U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5, } subscription_id_count = 0 recipient_id = 12 zerver_subscription: List[Dict[str, Any]] = [] final_subscription_id = get_subscription( channel_members, zerver_subscription, recipient_id, slack_user_id_to_zulip_user_id, subscription_id_count, ) # sanity checks self.assertEqual(final_subscription_id, 4) self.assertEqual(zerver_subscription[0]["recipient"], 12) self.assertEqual(zerver_subscription[0]["id"], 0) self.assertEqual( zerver_subscription[0]["user_profile"], slack_user_id_to_zulip_user_id[channel_members[0]], ) self.assertEqual( zerver_subscription[2]["user_profile"], slack_user_id_to_zulip_user_id[channel_members[2]], ) self.assertEqual(zerver_subscription[3]["id"], 3) self.assertEqual(zerver_subscription[1]["recipient"], zerver_subscription[3]["recipient"]) self.assertEqual(zerver_subscription[1]["pin_to_top"], False) def test_channels_to_zerver_stream(self) -> None: slack_user_id_to_zulip_user_id = { "U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5, } zerver_userprofile = [{"id": 1}, {"id": 8}, {"id": 7}, {"id": 5}] realm_id = 3 with self.assertLogs(level="INFO"): ( realm, added_channels, added_mpims, dm_members, slack_recipient_name_to_zulip_recipient_id, ) = channels_to_zerver_stream( self.fixture_file_name("", "slack_fixtures"), realm_id, {"zerver_userpresence": []}, slack_user_id_to_zulip_user_id, zerver_userprofile, ) test_added_channels = { "sharedchannel": ("C061A0HJG", 3), "general": ("C061A0YJG", 1), "general1": ("C061A0YJP", 2), "random": ("C061A0WJG", 0), } test_added_mpims = { "mpdm-user9--user2--user10-1": ("G9HBG2A5D", 0), "mpdm-user6--user7--user4-1": ("G6H1Z0ZPS", 1), "mpdm-user4--user1--user5-1": ("G6N944JPL", 2), } test_dm_members = { "DJ47BL849": ("U061A1R2R", "U061A5N1G"), "DHX1UP7EG": ("U061A5N1G", "U064KUGRJ"), "DK8HSJDHS": ("U061A1R2R", "U064KUGRJ"), "DRS3PSLDK": ("U064KUGRJ", "U064KUGRJ"), } slack_recipient_names = ( set(slack_user_id_to_zulip_user_id.keys()) | set(test_added_channels.keys()) | set(test_added_mpims.keys()) ) self.assertDictEqual(test_added_channels, added_channels) # zerver defaultstream already tested in helper functions. # Note that the `random` stream is archived and thus should # not be created as a DefaultStream. self.assertEqual(realm["zerver_defaultstream"], [{"id": 0, "realm": 3, "stream": 1}]) self.assertDictEqual(test_added_mpims, added_mpims) self.assertDictEqual(test_dm_members, dm_members) # We can't do an assertDictEqual since during the construction of personal # recipients, slack_user_id_to_zulip_user_id are iterated in different order in Python 3.5 and 3.6. self.assertEqual( set(slack_recipient_name_to_zulip_recipient_id.keys()), slack_recipient_names ) self.assertEqual(set(slack_recipient_name_to_zulip_recipient_id.values()), set(range(11))) # functioning of zerver subscriptions are already tested in the helper functions # This is to check the concatenation of the output lists from the helper functions # subscriptions for stream zerver_subscription = realm["zerver_subscription"] zerver_recipient = realm["zerver_recipient"] zerver_stream = realm["zerver_stream"] self.assertEqual(self.get_set(zerver_subscription, "recipient"), set(range(11))) self.assertEqual(self.get_set(zerver_subscription, "user_profile"), {1, 5, 7, 8}) self.assertEqual( self.get_set(zerver_recipient, "id"), self.get_set(zerver_subscription, "recipient") ) self.assertEqual(self.get_set(zerver_recipient, "type_id"), {0, 1, 2, 3, 5, 7, 8}) self.assertEqual(self.get_set(zerver_recipient, "type"), {1, 2, 3}) # stream mapping self.assertEqual(zerver_stream[0]["name"], "random") self.assertEqual(zerver_stream[0]["deactivated"], True) self.assertEqual(zerver_stream[0]["description"], "no purpose") self.assertEqual(zerver_stream[0]["invite_only"], False) self.assertEqual(zerver_stream[0]["history_public_to_subscribers"], True) self.assertEqual(zerver_stream[0]["realm"], realm_id) self.assertEqual(zerver_stream[2]["id"], test_added_channels[zerver_stream[2]["name"]][1]) self.assertEqual(self.get_set(realm["zerver_huddle"], "id"), {0, 1, 2}) self.assertEqual(realm["zerver_userpresence"], []) @mock.patch( "zerver.data_import.slack.users_to_zerver_userprofile", return_value=[[], [], {}, [], []] ) @mock.patch( "zerver.data_import.slack.channels_to_zerver_stream", return_value=[{"zerver_stream": []}, {}, {}, {}, {}], ) def test_slack_workspace_to_realm( self, mock_channels_to_zerver_stream: mock.Mock, mock_users_to_zerver_userprofile: mock.Mock ) -> None: realm_id = 1 user_list: List[Dict[str, Any]] = [] ( realm, slack_user_id_to_zulip_user_id, slack_recipient_name_to_zulip_recipient_id, added_channels, added_mpims, dm_members, avatar_list, em, ) = slack_workspace_to_realm( "testdomain", realm_id, user_list, "test-realm", "./random_path", {} ) test_zerver_realmdomain = [ {"realm": realm_id, "allow_subdomains": False, "domain": "testdomain", "id": realm_id} ] # Functioning already tests in helper functions self.assertEqual(slack_user_id_to_zulip_user_id, {}) self.assertEqual(added_channels, {}) self.assertEqual(added_mpims, {}) self.assertEqual(slack_recipient_name_to_zulip_recipient_id, {}) self.assertEqual(avatar_list, []) mock_channels_to_zerver_stream.assert_called_once_with("./random_path", 1, ANY, {}, []) passed_realm = mock_channels_to_zerver_stream.call_args_list[0][0][2] zerver_realmdomain = passed_realm["zerver_realmdomain"] self.assertListEqual(zerver_realmdomain, test_zerver_realmdomain) self.assertEqual( passed_realm["zerver_realm"][0]["description"], "Organization imported from Slack!" ) self.assertEqual(passed_realm["zerver_userpresence"], []) self.assert_length(passed_realm.keys(), 15) self.assertEqual(realm["zerver_stream"], []) self.assertEqual(realm["zerver_userprofile"], []) self.assertEqual(realm["zerver_realmemoji"], []) self.assertEqual(realm["zerver_customprofilefield"], []) self.assertEqual(realm["zerver_customprofilefieldvalue"], []) self.assert_length(realm.keys(), 5) def test_get_message_sending_user(self) -> None: message_with_file = {"subtype": "file", "type": "message", "file": {"user": "U064KUGRJ"}} message_without_file = {"subtype": "file", "type": "message", "user": "U064KUGRJ"} user_file = get_message_sending_user(message_with_file) self.assertEqual(user_file, "U064KUGRJ") user_without_file = get_message_sending_user(message_without_file) self.assertEqual(user_without_file, "U064KUGRJ") def test_build_zerver_message(self) -> None: zerver_usermessage: List[Dict[str, Any]] = [] # recipient_id -> set of user_ids subscriber_map = { 2: {3, 7, 15, 16}, # these we care about 4: {12}, 6: {19, 21}, } recipient_id = 2 mentioned_user_ids = [7] message_id = 9 um_id = NEXT_ID("user_message") build_usermessages( zerver_usermessage=zerver_usermessage, subscriber_map=subscriber_map, recipient_id=recipient_id, mentioned_user_ids=mentioned_user_ids, message_id=message_id, is_private=False, ) self.assertEqual(zerver_usermessage[0]["id"], um_id + 1) self.assertEqual(zerver_usermessage[0]["message"], message_id) self.assertEqual(zerver_usermessage[0]["flags_mask"], 1) self.assertEqual(zerver_usermessage[1]["id"], um_id + 2) self.assertEqual(zerver_usermessage[1]["message"], message_id) self.assertEqual(zerver_usermessage[1]["user_profile"], 7) self.assertEqual(zerver_usermessage[1]["flags_mask"], 9) # mentioned self.assertEqual(zerver_usermessage[2]["id"], um_id + 3) self.assertEqual(zerver_usermessage[2]["message"], message_id) self.assertEqual(zerver_usermessage[3]["id"], um_id + 4) self.assertEqual(zerver_usermessage[3]["message"], message_id) @mock.patch("zerver.data_import.slack.build_usermessages", return_value=(2, 4)) def test_channel_message_to_zerver_message(self, mock_build_usermessage: mock.Mock) -> None: user_data = [ {"id": "U066MTL5U", "name": "john doe", "deleted": False, "real_name": "John"}, {"id": "U061A5N1G", "name": "jane doe", "deleted": False, "real_name": "Jane"}, {"id": "U061A1R2R", "name": "jon", "deleted": False, "real_name": "Jon"}, ] slack_user_id_to_zulip_user_id = {"U066MTL5U": 5, "U061A5N1G": 24, "U061A1R2R": 43} reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}] all_messages: List[Dict[str, Any]] = [ { "text": "<@U066MTL5U> has joined the channel", "subtype": "channel_join", "user": "U066MTL5U", "ts": "1434139102.000002", "channel_name": "random", }, { "text": "<@U061A5N1G>: hey!", "user": "U061A1R2R", "ts": "1437868294.000006", "has_image": True, "channel_name": "random", }, { "text": "random", "user": "U061A5N1G", "reactions": reactions, "ts": "1439868294.000006", "channel_name": "random", }, { "text": "without a user", "user": None, # this message will be ignored as it has no user "ts": "1239868294.000006", "channel_name": "general", }, { "text": "", "user": "U061A1R2R", "ts": "1463868370.000008", "channel_name": "general", }, { "text": "added bot", "user": "U061A5N1G", "subtype": "bot_add", "ts": "1433868549.000010", "channel_name": "general", }, # This message will be ignored since it has no user and file is None. # See #9217 for the situation; likely file uploads on archived channels { "upload": False, "file": None, "text": "A file was shared", "channel_name": "general", "type": "message", "ts": "1433868549.000011", "subtype": "file_share", }, { "text": "random test", "user": "U061A1R2R", "ts": "1433868669.000012", "channel_name": "general", }, { "text": "Hello everyone", "user": "U061A1R2R", "type": "message", "ts": "1433868669.000015", "mpim_name": "mpdm-user9--user2--user10-1", }, { "text": "Who is watching the World Cup", "user": "U061A5N1G", "type": "message", "ts": "1433868949.000015", "mpim_name": "mpdm-user6--user7--user4-1", }, { "client_msg_id": "998d9229-35aa-424f-8d87-99e00df27dc9", "type": "message", "text": "Who is coming for camping this weekend?", "user": "U061A1R2R", "ts": "1553607595.000700", "pm_name": "DHX1UP7EG", }, { "client_msg_id": "998d9229-35aa-424f-8d87-99e00df27dc9", "type": "message", "text": "<@U061A5N1G>: Are you in Kochi?", "user": "U066MTL5U", "ts": "1553607595.000700", "pm_name": "DJ47BL849", }, ] slack_recipient_name_to_zulip_recipient_id = { "random": 2, "general": 1, "mpdm-user9--user2--user10-1": 5, "mpdm-user6--user7--user4-1": 6, "U066MTL5U": 7, "U061A5N1G": 8, "U061A1R2R": 8, } dm_members = { "DJ47BL849": ("U066MTL5U", "U061A5N1G"), "DHX1UP7EG": ("U061A5N1G", "U061A1R2R"), } zerver_usermessage: List[Dict[str, Any]] = [] subscriber_map: Dict[int, Set[int]] = {} added_channels: Dict[str, Tuple[str, int]] = {"random": ("c5", 1), "general": ("c6", 2)} ( zerver_message, zerver_usermessage, attachment, uploads, reaction, ) = channel_message_to_zerver_message( 1, user_data, slack_user_id_to_zulip_user_id, slack_recipient_name_to_zulip_recipient_id, all_messages, [], subscriber_map, added_channels, dm_members, "domain", set(), ) # functioning already tested in helper function self.assertEqual(zerver_usermessage, []) # subtype: channel_join is filtered self.assert_length(zerver_message, 9) self.assertEqual(uploads, []) self.assertEqual(attachment, []) # Test reactions self.assertEqual(reaction[0]["user_profile"], 24) self.assertEqual(reaction[0]["emoji_name"], reactions[0]["name"]) # Message conversion already tested in tests.test_slack_message_conversion self.assertEqual(zerver_message[0]["content"], "@**Jane**: hey!") self.assertEqual(zerver_message[0]["has_link"], False) self.assertEqual(zerver_message[2]["content"], "http://journals.plos.org/plosone/article") self.assertEqual(zerver_message[2]["has_link"], True) self.assertEqual(zerver_message[5]["has_link"], False) self.assertEqual(zerver_message[7]["has_link"], False) self.assertEqual(zerver_message[3][EXPORT_TOPIC_NAME], "imported from Slack") self.assertEqual(zerver_message[3]["content"], "/me added bot") self.assertEqual( zerver_message[4]["recipient"], slack_recipient_name_to_zulip_recipient_id["general"] ) self.assertEqual(zerver_message[2][EXPORT_TOPIC_NAME], "imported from Slack") self.assertEqual( zerver_message[1]["recipient"], slack_recipient_name_to_zulip_recipient_id["random"] ) self.assertEqual( zerver_message[5]["recipient"], slack_recipient_name_to_zulip_recipient_id["mpdm-user9--user2--user10-1"], ) self.assertEqual( zerver_message[6]["recipient"], slack_recipient_name_to_zulip_recipient_id["mpdm-user6--user7--user4-1"], ) self.assertEqual( zerver_message[7]["recipient"], slack_recipient_name_to_zulip_recipient_id["U061A5N1G"] ) self.assertEqual( zerver_message[7]["recipient"], slack_recipient_name_to_zulip_recipient_id["U061A5N1G"] ) self.assertEqual(zerver_message[3]["id"], zerver_message[0]["id"] + 3) self.assertEqual(zerver_message[4]["id"], zerver_message[0]["id"] + 4) self.assertEqual(zerver_message[5]["id"], zerver_message[0]["id"] + 5) self.assertEqual(zerver_message[7]["id"], zerver_message[0]["id"] + 7) self.assertIsNone(zerver_message[3]["rendered_content"]) self.assertEqual(zerver_message[0]["has_image"], False) self.assertEqual(zerver_message[0]["date_sent"], float(all_messages[1]["ts"])) self.assertEqual(zerver_message[2]["rendered_content_version"], 1) self.assertEqual(zerver_message[0]["sender"], 43) self.assertEqual(zerver_message[3]["sender"], 24) self.assertEqual(zerver_message[5]["sender"], 43) self.assertEqual(zerver_message[6]["sender"], 24) self.assertEqual(zerver_message[7]["sender"], 43) self.assertEqual(zerver_message[8]["sender"], 5) @mock.patch("zerver.data_import.slack.channel_message_to_zerver_message") @mock.patch("zerver.data_import.slack.get_messages_iterator") def test_convert_slack_workspace_messages( self, mock_get_messages_iterator: mock.Mock, mock_message: mock.Mock ) -> None: output_dir = os.path.join(settings.TEST_WORKER_DIR, "test-slack-import") os.makedirs(output_dir, exist_ok=True) added_channels: Dict[str, Tuple[str, int]] = {"random": ("c5", 1), "general": ("c6", 2)} time = float(timezone_now().timestamp()) zerver_message = [{"id": 1, "ts": time}, {"id": 5, "ts": time}] def fake_get_messages_iter( slack_data_dir: str, added_channels: AddedChannelsT, added_mpims: AddedMPIMsT, dm_members: DMMembersT, ) -> Iterator[ZerverFieldsT]: import copy return iter(copy.deepcopy(zerver_message)) realm: Dict[str, Any] = {"zerver_subscription": []} user_list: List[Dict[str, Any]] = [] reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}] attachments: List[Dict[str, Any]] = [] uploads: List[Dict[str, Any]] = [] zerver_usermessage = [{"id": 3}, {"id": 5}, {"id": 6}, {"id": 9}] mock_get_messages_iterator.side_effect = fake_get_messages_iter mock_message.side_effect = [ [zerver_message[:1], zerver_usermessage[:2], attachments, uploads, reactions[:1]], [zerver_message[1:2], zerver_usermessage[2:5], attachments, uploads, reactions[1:1]], ] with self.assertLogs(level="INFO"): # Hacky: We should include a zerver_userprofile, not the empty [] test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages( "./random_path", user_list, 2, {}, {}, added_channels, {}, {}, realm, [], [], "domain", output_dir=output_dir, chunk_size=1, ) messages_file_1 = os.path.join(output_dir, "messages-000001.json") self.assertTrue(os.path.exists(messages_file_1)) messages_file_2 = os.path.join(output_dir, "messages-000002.json") self.assertTrue(os.path.exists(messages_file_2)) with open(messages_file_1, "rb") as f: message_json = orjson.loads(f.read()) self.assertEqual(message_json["zerver_message"], zerver_message[:1]) self.assertEqual(message_json["zerver_usermessage"], zerver_usermessage[:2]) with open(messages_file_2, "rb") as f: message_json = orjson.loads(f.read()) self.assertEqual(message_json["zerver_message"], zerver_message[1:2]) self.assertEqual(message_json["zerver_usermessage"], zerver_usermessage[2:5]) self.assertEqual(test_reactions, reactions) @mock.patch("zerver.data_import.slack.requests.get") @mock.patch("zerver.data_import.slack.process_uploads", return_value=[]) @mock.patch("zerver.data_import.slack.build_attachment", return_value=[]) @mock.patch("zerver.data_import.slack.build_avatar_url") @mock.patch("zerver.data_import.slack.build_avatar") @mock.patch("zerver.data_import.slack.get_slack_api_data") def test_slack_import_to_existing_database( self, mock_get_slack_api_data: mock.Mock, mock_build_avatar_url: mock.Mock, mock_build_avatar: mock.Mock, mock_process_uploads: mock.Mock, mock_attachment: mock.Mock, mock_requests_get: mock.Mock, ) -> None: test_slack_dir = os.path.join( settings.DEPLOY_ROOT, "zerver", "tests", "fixtures", "slack_fixtures" ) test_slack_zip_file = os.path.join(test_slack_dir, "test_slack_importer.zip") test_slack_unzipped_file = os.path.join(test_slack_dir, "test_slack_importer") test_realm_subdomain = "test-slack-import" output_dir = os.path.join(settings.DEPLOY_ROOT, "var", "test-slack-importer-data") token = "xoxp-valid-token" # If the test fails, the 'output_dir' would not be deleted and hence it would give an # error when we run the tests next time, as 'do_convert_data' expects an empty 'output_dir' # hence we remove it before running 'do_convert_data' self.rm_tree(output_dir) # Also the unzipped data file should be removed if the test fails at 'do_convert_data' self.rm_tree(test_slack_unzipped_file) user_data_fixture = orjson.loads(self.fixture_data("user_data.json", type="slack_fixtures")) team_info_fixture = orjson.loads(self.fixture_data("team_info.json", type="slack_fixtures")) mock_get_slack_api_data.side_effect = [ user_data_fixture["members"], {}, team_info_fixture["team"], ] mock_requests_get.return_value.raw = BytesIO(read_test_image_file("img.png")) with self.assertLogs(level="INFO"): do_convert_data(test_slack_zip_file, output_dir, token) self.assertTrue(os.path.exists(output_dir)) self.assertTrue(os.path.exists(output_dir + "/realm.json")) realm_icons_path = os.path.join(output_dir, "realm_icons") realm_icon_records_path = os.path.join(realm_icons_path, "records.json") self.assertTrue(os.path.exists(realm_icon_records_path)) with open(realm_icon_records_path, "rb") as f: records = orjson.loads(f.read()) self.assert_length(records, 2) self.assertEqual(records[0]["path"], "0/icon.original") self.assertTrue(os.path.exists(os.path.join(realm_icons_path, records[0]["path"]))) self.assertEqual(records[1]["path"], "0/icon.png") self.assertTrue(os.path.exists(os.path.join(realm_icons_path, records[1]["path"]))) # test import of the converted slack data into an existing database with self.settings(BILLING_ENABLED=False), self.assertLogs(level="INFO"): do_import_realm(output_dir, test_realm_subdomain) realm = get_realm(test_realm_subdomain) self.assertTrue(realm.name, test_realm_subdomain) self.assertEqual(realm.icon_source, Realm.ICON_UPLOADED) # test RealmAuditLog realmauditlog = RealmAuditLog.objects.filter(realm=realm) realmauditlog_event_type = {log.event_type for log in realmauditlog} self.assertEqual( realmauditlog_event_type, { RealmAuditLog.SUBSCRIPTION_CREATED, RealmAuditLog.REALM_PLAN_TYPE_CHANGED, RealmAuditLog.REALM_CREATED, }, ) Realm.objects.filter(name=test_realm_subdomain).delete() remove_folder(output_dir) # remove tar file created in 'do_convert_data' function os.remove(output_dir + ".tar.gz") self.assertFalse(os.path.exists(output_dir)) def test_message_files(self) -> None: alice_id = 7 alice = dict( id=alice_id, profile=dict( email="alice@example.com", ), ) files = [ dict( url_private="files.slack.com/apple.png", title="Apple", name="apple.png", mimetype="image/png", timestamp=9999, created=8888, size=3000000, ), dict( url_private="example.com/banana.zip", title="banana", ), ] message = dict( user=alice_id, files=files, ) domain_name = "example.com" realm_id = 5 message_id = 99 slack_user_id = "alice" users = [alice] slack_user_id_to_zulip_user_id = { "alice": alice_id, } zerver_attachment: List[Dict[str, Any]] = [] uploads_list: List[Dict[str, Any]] = [] info = process_message_files( message=message, domain_name=domain_name, realm_id=realm_id, message_id=message_id, slack_user_id=slack_user_id, users=users, slack_user_id_to_zulip_user_id=slack_user_id_to_zulip_user_id, zerver_attachment=zerver_attachment, uploads_list=uploads_list, ) self.assert_length(zerver_attachment, 1) self.assert_length(uploads_list, 1) image_path = zerver_attachment[0]["path_id"] expected_content = f"[Apple](/user_uploads/{image_path})\n[banana](example.com/banana.zip)" self.assertEqual(info["content"], expected_content) self.assertTrue(info["has_link"]) self.assertTrue(info["has_image"]) self.assertEqual(uploads_list[0]["s3_path"], image_path) self.assertEqual(uploads_list[0]["realm_id"], realm_id) self.assertEqual(uploads_list[0]["user_profile_email"], "alice@example.com") def test_bot_duplicates(self) -> None: self.assertEqual( SlackBotEmail.get_email( {"real_name_normalized": "Real Bot", "bot_id": "foo"}, "example.com" ), "real-bot@example.com", ) # SlackBotEmail keeps state -- doing it again appends a "2", "3", etc self.assertEqual( SlackBotEmail.get_email( {"real_name_normalized": "Real Bot", "bot_id": "bar"}, "example.com" ), "real-bot-2@example.com", ) self.assertEqual( SlackBotEmail.get_email( {"real_name_normalized": "Real Bot", "bot_id": "baz"}, "example.com" ), "real-bot-3@example.com", ) # But caches based on the bot_id self.assertEqual( SlackBotEmail.get_email( {"real_name_normalized": "Real Bot", "bot_id": "foo"}, "example.com" ), "real-bot@example.com", ) self.assertEqual( SlackBotEmail.get_email({"first_name": "Other Name", "bot_id": "other"}, "example.com"), "othername-bot@example.com", )