Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

mattermost: Add support for exporting DMs and huddles.

This commit is contained in:
Vishnu KS 2019-09-25 19:16:05 +05:30 committed by Tim Abbott
parent ae5bc92602
commit 1585ad7bf4
6 changed files with 370 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
templates/zerver/help/import-from-mattermost.md
View File

@ -177,11 +177,15 @@ root domain. Replace the last line above with the following, after replacing
Mattermost's export tool is incomplete and does not support exporting Mattermost's export tool is incomplete and does not support exporting
the following data: the following data:
* private messages and group private messages between users
* user avatars * user avatars
* uploaded files and message attachments. * uploaded files and message attachments.
We expect to add support for importing these data from Mattermost once We expect to add support for importing these data from Mattermost once
Mattermost's export tool includes them. Mattermost's export tool includes them.
Additionally, Mattermost's data exports do not associated private
messages with a specific Mattermost team. For that reason, the import
tool will only import private messages for data export archives
containing a single Mattermost team.
[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository [upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository

1
tools/test-backend
View File

@ -123,7 +123,6 @@ not_yet_fully_covered = {path for target in [
'zerver/tornado/websocket_client.py', 'zerver/tornado/websocket_client.py',
# Data import files; relatively low priority # Data import files; relatively low priority
'zerver/data_import/hipchat*.py', 'zerver/data_import/hipchat*.py',
'zerver/data_import/mattermost.py',
'zerver/data_import/sequencer.py', 'zerver/data_import/sequencer.py',
'zerver/data_import/slack.py', 'zerver/data_import/slack.py',
'zerver/data_import/gitter.py', 'zerver/data_import/gitter.py',

41
zerver/data_import/import_util.py
View File

@ -222,6 +222,34 @@ def build_stream_subscriptions(
return subscriptions return subscriptions
def build_huddle_subscriptions(
get_users: Callable[..., Set[int]],
zerver_recipient: List[ZerverFieldsT],
zerver_huddle: List[ZerverFieldsT]) -> List[ZerverFieldsT]:
subscriptions = [] # type: List[ZerverFieldsT]
huddle_ids = {huddle['id'] for huddle in zerver_huddle}
recipient_map = {
recipient['id']: recipient['type_id'] # recipient_id -> stream_id
for recipient in zerver_recipient
if recipient['type'] == Recipient.HUDDLE
and recipient['type_id'] in huddle_ids
}
for recipient_id, huddle_id in recipient_map.items():
user_ids = get_users(huddle_id=huddle_id)
for user_id in user_ids:
subscription = build_subscription(
recipient_id=recipient_id,
user_id=user_id,
subscription_id=NEXT_ID('subscription'),
)
subscriptions.append(subscription)
return subscriptions
def build_personal_subscriptions(zerver_recipient: List[ZerverFieldsT]) -> List[ZerverFieldsT]: def build_personal_subscriptions(zerver_recipient: List[ZerverFieldsT]) -> List[ZerverFieldsT]:
subscriptions = [] # type: List[ZerverFieldsT] subscriptions = [] # type: List[ZerverFieldsT]
@ -253,7 +281,8 @@ def build_recipient(type_id: int, recipient_id: int, type: int) -> ZerverFieldsT
return recipient_dict return recipient_dict
def build_recipients(zerver_userprofile: List[ZerverFieldsT], def build_recipients(zerver_userprofile: List[ZerverFieldsT],
zerver_stream: List[ZerverFieldsT]) -> List[ZerverFieldsT]: zerver_stream: List[ZerverFieldsT],
zerver_huddle: List[ZerverFieldsT]=[]) -> List[ZerverFieldsT]:
''' '''
As of this writing, we only use this in the HipChat As of this writing, we only use this in the HipChat
conversion. The Slack and Gitter conversions do it more conversion. The Slack and Gitter conversions do it more
@ -284,6 +313,16 @@ def build_recipients(zerver_userprofile: List[ZerverFieldsT],
recipient_dict = model_to_dict(recipient) recipient_dict = model_to_dict(recipient)
recipients.append(recipient_dict) recipients.append(recipient_dict)
for huddle in zerver_huddle:
type_id = huddle['id']
type = Recipient.HUDDLE
recipient = Recipient(
type_id=type_id,
id=NEXT_ID('recipient'),
type=type,
)
recipient_dict = model_to_dict(recipient)
recipients.append(recipient_dict)
return recipients return recipients
def build_realm(zerver_realm: List[ZerverFieldsT], realm_id: int, def build_realm(zerver_realm: List[ZerverFieldsT], realm_id: int,

198
zerver/data_import/mattermost.py
View File

@ -23,8 +23,9 @@ from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH
from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \
build_stream, build_realm, build_message, create_converted_data_files, \ build_stream, build_realm, build_message, create_converted_data_files, \
make_subscriber_map, build_recipients, build_user_profile, \ make_subscriber_map, build_recipients, build_user_profile, \
build_stream_subscriptions, build_personal_subscriptions, SubscriberHandler, \ build_stream_subscriptions, build_huddle_subscriptions, \
build_realm_emoji, make_user_messages build_personal_subscriptions, SubscriberHandler, \
build_realm_emoji, make_user_messages, build_huddle
from zerver.data_import.mattermost_user import UserHandler from zerver.data_import.mattermost_user import UserHandler
from zerver.data_import.sequencer import NEXT_ID, IdMapper from zerver.data_import.sequencer import NEXT_ID, IdMapper
@ -196,13 +197,43 @@ def convert_channel_data(channel_data: List[ZerverFieldsT],
if channel_users: if channel_users:
subscriber_handler.set_info( subscriber_handler.set_info(
stream_id=stream_id,
users=channel_users, users=channel_users,
stream_id=stream_id,
) )
streams.append(stream) streams.append(stream)
return streams return streams
def generate_huddle_name(huddle_members: List[str]) -> str:
# Simple hash function to generate a unique hash key for the
# members of a huddle. Needs to be consistent only within the
# lifetime of export tool run, as it doesn't appear in the output.
import hashlib
return hashlib.md5(''.join(sorted(huddle_members)).encode('utf-8')).hexdigest()
def convert_huddle_data(huddle_data: List[ZerverFieldsT],
user_data_map: Dict[str, Dict[str, Any]],
subscriber_handler: SubscriberHandler,
huddle_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
team_name: str) -> List[ZerverFieldsT]:
zerver_huddle = []
for huddle in huddle_data:
if len(huddle["members"]) > 2:
huddle_name = generate_huddle_name(huddle["members"])
huddle_id = huddle_id_mapper.get(huddle_name)
huddle_dict = build_huddle(huddle_id)
huddle_user_ids = set()
for username in huddle["members"]:
huddle_user_ids.add(user_id_mapper.get(username))
subscriber_handler.set_info(
users=huddle_user_ids,
huddle_id=huddle_id,
)
zerver_huddle.append(huddle_dict)
return zerver_huddle
def get_name_to_codepoint_dict() -> Dict[str, str]: def get_name_to_codepoint_dict() -> Dict[str, str]:
with open(NAME_TO_CODEPOINT_PATH) as fp: with open(NAME_TO_CODEPOINT_PATH) as fp:
return ujson.load(fp) return ujson.load(fp)
@ -291,6 +322,7 @@ def process_raw_message_batch(realm_id: int,
h = html2text.HTML2Text() h = html2text.HTML2Text()
name_to_codepoint = get_name_to_codepoint_dict() name_to_codepoint = get_name_to_codepoint_dict()
pm_members = {}
for raw_message in raw_messages: for raw_message in raw_messages:
message_id = NEXT_ID('message') message_id = NEXT_ID('message')
@ -309,11 +341,21 @@ def process_raw_message_batch(realm_id: int,
date_sent = raw_message['date_sent'] date_sent = raw_message['date_sent']
sender_user_id = raw_message['sender_id'] sender_user_id = raw_message['sender_id']
try: if "channel_name" in raw_message:
recipient_id = get_recipient_id_from_receiver_name(raw_message["receiver_id"], Recipient.STREAM) recipient_id = get_recipient_id_from_receiver_name(raw_message["channel_name"], Recipient.STREAM)
except KeyError: elif "huddle_name" in raw_message:
logging.debug("Could not find recipient_id for a message, skipping.") recipient_id = get_recipient_id_from_receiver_name(raw_message["huddle_name"], Recipient.HUDDLE)
continue elif "pm_members" in raw_message:
members = raw_message["pm_members"]
member_ids = {user_id_mapper.get(member) for member in members}
pm_members[message_id] = member_ids
if sender_user_id == user_id_mapper.get(members[0]):
recipient_id = get_recipient_id_from_receiver_name(members[1], Recipient.PERSONAL)
else:
recipient_id = get_recipient_id_from_receiver_name(members[0], Recipient.PERSONAL)
else:
raise AssertionError("raw_message without channel_name, huddle_name or pm_members key")
rendered_content = None rendered_content = None
topic_name = 'imported from mattermost' topic_name = 'imported from mattermost'
@ -377,7 +419,8 @@ def process_posts(num_teams: int,
post_data_list.append(post) post_data_list.append(post)
def message_to_dict(post_dict: Dict[str, Any]) -> Dict[str, Any]: def message_to_dict(post_dict: Dict[str, Any]) -> Dict[str, Any]:
sender_id = user_id_mapper.get(post_dict["user"]) sender_username = post_dict["user"]
sender_id = user_id_mapper.get(sender_username)
content = post_dict['message'] content = post_dict['message']
if masking_content: if masking_content:
@ -389,13 +432,26 @@ def process_posts(num_teams: int,
else: else:
reactions = [] reactions = []
return dict( message_dict = dict(
sender_id=sender_id, sender_id=sender_id,
receiver_id=post_dict["channel"],
content=content, content=content,
date_sent=int(post_dict['create_at'] / 1000), date_sent=int(post_dict['create_at'] / 1000),
reactions=reactions reactions=reactions
) )
if "channel" in post_dict:
message_dict["channel_name"] = post_dict["channel"]
elif "channel_members" in post_dict:
# This case is for handling posts from PMs and huddles, not channels.
# PMs and huddles are known as direct_channels in Slack and hence
# the name channel_members.
channel_members = post_dict["channel_members"]
if len(channel_members) > 2:
message_dict["huddle_name"] = generate_huddle_name(channel_members)
elif len(channel_members) == 2:
message_dict["pm_members"] = channel_members
else:
raise AssertionError("Post without channel or channel_members key.")
return message_dict
raw_messages = [] raw_messages = []
for post_dict in post_data_list: for post_dict in post_data_list:
@ -433,45 +489,67 @@ def process_posts(num_teams: int,
def write_message_data(num_teams: int, def write_message_data(num_teams: int,
team_name: str, team_name: str,
realm_id: int, realm_id: int,
post_data: List[Dict[str, Any]], post_data: Dict[str, List[Dict[str, Any]]],
zerver_recipient: List[ZerverFieldsT], zerver_recipient: List[ZerverFieldsT],
subscriber_map: Dict[int, Set[int]], subscriber_map: Dict[int, Set[int]],
output_dir: str, output_dir: str,
masking_content: bool, masking_content: bool,
stream_id_mapper: IdMapper, stream_id_mapper: IdMapper,
huddle_id_mapper: IdMapper,
user_id_mapper: IdMapper, user_id_mapper: IdMapper,
user_handler: UserHandler, user_handler: UserHandler,
username_to_user: Dict[str, Dict[str, Any]], username_to_user: Dict[str, Dict[str, Any]],
zerver_realmemoji: List[Dict[str, Any]], zerver_realmemoji: List[Dict[str, Any]],
total_reactions: List[Dict[str, Any]]) -> None: total_reactions: List[Dict[str, Any]]) -> None:
stream_id_to_recipient_id = {} stream_id_to_recipient_id = {}
huddle_id_to_recipient_id = {}
user_id_to_recipient_id = {}
for d in zerver_recipient: for d in zerver_recipient:
if d['type'] == Recipient.STREAM: if d['type'] == Recipient.STREAM:
stream_id_to_recipient_id[d['type_id']] = d['id'] stream_id_to_recipient_id[d['type_id']] = d['id']
elif d['type'] == Recipient.HUDDLE:
huddle_id_to_recipient_id[d['type_id']] = d['id']
if d['type'] == Recipient.PERSONAL:
user_id_to_recipient_id[d['type_id']] = d['id']
def get_recipient_id_from_receiver_name(receiver_name: str, recipient_type: int) -> int: def get_recipient_id_from_receiver_name(receiver_name: str, recipient_type: int) -> int:
if recipient_type == Recipient.STREAM: if recipient_type == Recipient.STREAM:
receiver_id = stream_id_mapper.get(receiver_name) receiver_id = stream_id_mapper.get(receiver_name)
recipient_id = stream_id_to_recipient_id[receiver_id] recipient_id = stream_id_to_recipient_id[receiver_id]
elif recipient_type == Recipient.HUDDLE:
receiver_id = huddle_id_mapper.get(receiver_name)
recipient_id = huddle_id_to_recipient_id[receiver_id]
elif recipient_type == Recipient.PERSONAL:
receiver_id = user_id_mapper.get(receiver_name)
recipient_id = user_id_to_recipient_id[receiver_id]
else:
raise AssertionError("Invalid recipient_type")
return recipient_id return recipient_id
process_posts( if num_teams == 1:
num_teams=num_teams, post_types = ["channel_post", "direct_post"]
team_name=team_name, else:
realm_id=realm_id, post_types = ["channel_post"]
post_data=post_data, logging.warning("Skipping importing huddles and PMs since there are multiple teams in the export")
get_recipient_id_from_receiver_name=get_recipient_id_from_receiver_name,
subscriber_map=subscriber_map, for post_type in post_types:
output_dir=output_dir, process_posts(
is_pm_data=False, num_teams=num_teams,
masking_content=masking_content, team_name=team_name,
user_id_mapper=user_id_mapper, realm_id=realm_id,
user_handler=user_handler, post_data=post_data[post_type],
username_to_user=username_to_user, get_recipient_id_from_receiver_name=get_recipient_id_from_receiver_name,
zerver_realmemoji=zerver_realmemoji, subscriber_map=subscriber_map,
total_reactions=total_reactions, output_dir=output_dir,
) is_pm_data=post_type == "direct_post",
masking_content=masking_content,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
username_to_user=username_to_user,
zerver_realmemoji=zerver_realmemoji,
total_reactions=total_reactions,
)
def write_emoticon_data(realm_id: int, def write_emoticon_data(realm_id: int,
custom_emoji_data: List[Dict[str, Any]], custom_emoji_data: List[Dict[str, Any]],
@ -572,47 +650,50 @@ def check_user_in_team(user: Dict[str, Any], team_name: str) -> bool:
return True return True
return False return False
def label_mirror_dummy_users(num_teams: int, team_name: str, def label_mirror_dummy_users(num_teams: int, team_name: str, mattermost_data: Dict[str, Any],
mattermost_data: Dict[str, List[Dict[str, Any]]],
username_to_user: Dict[str, Dict[str, Any]]) -> None: username_to_user: Dict[str, Dict[str, Any]]) -> None:
# This function might looks like a great place to label admin users. But # This function might looks like a great place to label admin users. But
# that won't be fully correct since we are iterating only though posts and # that won't be fully correct since we are iterating only though posts and
# it covers only users that has sent atleast one message. # it covers only users that has sent atleast one message.
for post in mattermost_data["post"]: for post in mattermost_data["post"]["channel_post"]:
if "team" not in post: post_team = post["team"]
if num_teams > 1:
raise AssertionError("Cannot import private messages in export with multiple teams.")
elif num_teams == 0:
raise AssertionError("Export with 0 teams")
else:
post_team = team_name
else:
post_team = post["team"]
if post_team == team_name: if post_team == team_name:
user = username_to_user[post["user"]] user = username_to_user[post["user"]]
if not check_user_in_team(user, team_name): if not check_user_in_team(user, team_name):
user["is_mirror_dummy"] = True user["is_mirror_dummy"] = True
if num_teams == 1:
for post in mattermost_data["post"]["direct_post"]:
assert("team" not in post)
user = username_to_user[post["user"]]
if not check_user_in_team(user, team_name):
user["is_mirror_dummy"] = True
def reset_mirror_dummy_users(username_to_user: Dict[str, Dict[str, Any]]) -> None: def reset_mirror_dummy_users(username_to_user: Dict[str, Dict[str, Any]]) -> None:
for username in username_to_user: for username in username_to_user:
user = username_to_user[username] user = username_to_user[username]
user["is_mirror_dummy"] = False user["is_mirror_dummy"] = False
def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, List[Dict[str, Any]]]: def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, Any]:
mattermost_data = {} # type: Dict[str, List[Dict[str, Any]]] mattermost_data = {} # type: Dict[str, Any]
mattermost_data["version"] = [] mattermost_data["version"] = []
mattermost_data["team"] = [] mattermost_data["team"] = []
mattermost_data["channel"] = [] mattermost_data["channel"] = []
mattermost_data["user"] = [] mattermost_data["user"] = []
mattermost_data["post"] = [] mattermost_data["post"] = {"channel_post": [], "direct_post": []}
mattermost_data["emoji"] = [] mattermost_data["emoji"] = []
mattermost_data["direct_channel"] = []
with open(mattermost_data_file, "r") as fp: with open(mattermost_data_file, "r") as fp:
for line in fp: for line in fp:
row = ujson.loads(line.rstrip("\n")) row = ujson.loads(line.rstrip("\n"))
data_type = row["type"] data_type = row["type"]
mattermost_data[data_type].append(row[data_type]) if data_type == "post":
mattermost_data["post"]["channel_post"].append(row["post"])
elif data_type == "direct_post":
mattermost_data["post"]["direct_post"].append(row["direct_post"])
else:
mattermost_data[data_type].append(row[data_type])
return mattermost_data return mattermost_data
def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None: def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None:
@ -635,6 +716,7 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
subscriber_handler = SubscriberHandler() subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper() user_id_mapper = IdMapper()
stream_id_mapper = IdMapper() stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
print("Generating data for", team_name) print("Generating data for", team_name)
realm = make_realm(realm_id, team) realm = make_realm(realm_id, team)
@ -660,14 +742,27 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
realm_id=realm_id, realm_id=realm_id,
team_name=team_name, team_name=team_name,
) )
realm['zerver_stream'] = zerver_stream realm['zerver_stream'] = zerver_stream
zerver_huddle = [] # type: List[ZerverFieldsT]
if len(mattermost_data["team"]) == 1:
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm['zerver_huddle'] = zerver_huddle
all_users = user_handler.get_all_users() all_users = user_handler.get_all_users()
zerver_recipient = build_recipients( zerver_recipient = build_recipients(
zerver_userprofile=all_users, zerver_userprofile=all_users,
zerver_stream=zerver_stream, zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
) )
realm['zerver_recipient'] = zerver_recipient realm['zerver_recipient'] = zerver_recipient
@ -677,13 +772,19 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
zerver_stream=zerver_stream, zerver_stream=zerver_stream,
) )
huddle_subscriptions = build_huddle_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_huddle=zerver_huddle,
)
personal_subscriptions = build_personal_subscriptions( personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient, zerver_recipient=zerver_recipient,
) )
# Mattermost currently supports only exporting messages from channels. # Mattermost currently supports only exporting messages from channels.
# Personal messages and huddles are not exported. # Personal messages and huddles are not exported.
zerver_subscription = personal_subscriptions + stream_subscriptions zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
realm['zerver_subscription'] = zerver_subscription realm['zerver_subscription'] = zerver_subscription
zerver_realmemoji = write_emoticon_data( zerver_realmemoji = write_emoticon_data(
@ -709,6 +810,7 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
output_dir=realm_output_dir, output_dir=realm_output_dir,
masking_content=masking_content, masking_content=masking_content,
stream_id_mapper=stream_id_mapper, stream_id_mapper=stream_id_mapper,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper, user_id_mapper=user_id_mapper,
user_handler=user_handler, user_handler=user_handler,
username_to_user=username_to_user, username_to_user=username_to_user,

22
zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json vendored Normal file
View File

@ -0,0 +1,22 @@
{"type":"version","version":1}
{"type":"team","team":{"name":"gryffindor","display_name":"Gryffindor","type":"O","description":"","allow_open_invite":true}}
{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-common-room","display_name":"Gryffindor common room","type":"O","header":"","purpose":"A place for talking about Gryffindor common room"}}
{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-quidditch-team","display_name":"Gryffindor quidditch team","type":"O","header":"","purpose":"A place for talking about Gryffindor quidditch team"}}
{"type":"channel","channel":{"team":"gryffindor","name":"dumbledores-army","display_name":"Dumbledores army","type":"P","header":"https//:github.com/zulip/zulip","purpose":"A place for talking about Dumbledores army"}}
{"type":"user","user":{"username":"ginny","email":"ginny@zulip.com","auth_service":"","nickname":"","first_name":"Ginny","last_name":"Weasley","position":"","roles":"system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_user","channels":[{"name":"gryffindor-quidditch-team","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"dumbledores-army","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"ginny,@ginny"}}}
{"type":"user","user":{"username":"ron","email":"ron@zulip.com","auth_service":"","nickname":"","first_name":"Ron","last_name":"Weasley","position":"","roles":"system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_user","channels":[{"name":"gryffindor-quidditch-team","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"dumbledores-army","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"ron,@ron"}}}
{"type":"user","user":{"username":"harry","email":"harry@zulip.com","auth_service":"","nickname":"","first_name":"Harry","last_name":"Potter","position":"","roles":"system_admin system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_admin team_user","channels":[{"name":"dumbledores-army","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"harry,@harry"}}}
{"type":"user","user":{"username":"voldemort","email":"voldemort@zulip.com","auth_service":"","nickname":"","first_name":"Tom","last_name":"Riddle","position":"","roles":"system_admin system_user","locale":"en","teams":null, "notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"harry,@harry"}}}
{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"ron","message":"ron joined the channel.","create_at":1553166512493,"reactions":null,"replies":null}}
{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"Hey folks","create_at":1553166519720,"reactions":null,"replies":null}}
{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"@ron Welcome mate!","create_at":1553166519726,"reactions":null,"replies":null}}
{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"harry","message":"Looks like this channel is empty","create_at":1553166567370,"reactions":[{"user":"ron","create_at":1553166584976,"emoji_name":"rocket"}],"replies":null}}
{"type":"direct_channel","direct_channel":{"members":["ron","harry"],"favorited_by":null,"header":""}}
{"type":"direct_channel","direct_channel":{"members":["ron","harry", "ginny"],"favorited_by":null,"header":""}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"ron","message":"hey harry","create_at":1566376137676,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"harry","message":"whats up","create_at":1566376318568,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ginny","message":"Who is going to Hogesmead this weekend?","create_at":1566376226493,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"I am going.","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ron","message":"I am going as well","create_at":1566376286363,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"voldemort","message":"Hey Harry.","create_at":1566376318569,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"harry","message":"Ahh. Here we go again.","create_at":1566376318579,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}

162
zerver/tests/test_mattermost_importer.py
View File

@ -16,10 +16,11 @@ from zerver.data_import.mattermost_user import UserHandler
from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \ from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \
create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \ create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \
convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \ convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \
build_reactions, get_name_to_codepoint_dict, do_convert_data build_reactions, get_name_to_codepoint_dict, do_convert_data, convert_huddle_data, \
generate_huddle_name
from zerver.data_import.sequencer import IdMapper from zerver.data_import.sequencer import IdMapper
from zerver.data_import.import_util import SubscriberHandler from zerver.data_import.import_util import SubscriberHandler
from zerver.models import Reaction, UserProfile, Message, get_realm, get_user from zerver.models import Reaction, UserProfile, Message, get_realm, get_user, Recipient
class MatterMostImporter(ZulipTestCase): class MatterMostImporter(ZulipTestCase):
logger = logging.getLogger() logger = logging.getLogger()
@ -29,8 +30,7 @@ class MatterMostImporter(ZulipTestCase):
def test_mattermost_data_file_to_dict(self) -> None: def test_mattermost_data_file_to_dict(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name) mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
self.assertEqual(len(mattermost_data), 7)
self.assertEqual(len(mattermost_data), 6)
self.assertEqual(mattermost_data["version"], [1]) self.assertEqual(mattermost_data["version"], [1])
@ -45,15 +45,30 @@ class MatterMostImporter(ZulipTestCase):
self.assertEqual(mattermost_data["user"][1]["username"], "harry") self.assertEqual(mattermost_data["user"][1]["username"], "harry")
self.assertEqual(len(mattermost_data["user"][1]["teams"]), 1) self.assertEqual(len(mattermost_data["user"][1]["teams"]), 1)
self.assertEqual(len(mattermost_data["post"]), 20) self.assertEqual(len(mattermost_data["post"]["channel_post"]), 20)
self.assertEqual(mattermost_data["post"][0]["team"], "gryffindor") self.assertEqual(mattermost_data["post"]["channel_post"][0]["team"], "gryffindor")
self.assertEqual(mattermost_data["post"][0]["channel"], "dumbledores-army") self.assertEqual(mattermost_data["post"]["channel_post"][0]["channel"], "dumbledores-army")
self.assertEqual(mattermost_data["post"][0]["user"], "harry") self.assertEqual(mattermost_data["post"]["channel_post"][0]["user"], "harry")
self.assertEqual(len(mattermost_data["post"][0]["replies"]), 1) self.assertEqual(len(mattermost_data["post"]["channel_post"][0]["replies"]), 1)
self.assertEqual(len(mattermost_data["emoji"]), 2) self.assertEqual(len(mattermost_data["emoji"]), 2)
self.assertEqual(mattermost_data["emoji"][0]["name"], "peerdium") self.assertEqual(mattermost_data["emoji"][0]["name"], "peerdium")
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
self.assertEqual(len(mattermost_data["post"]["channel_post"]), 4)
self.assertEqual(mattermost_data["post"]["channel_post"][0]["team"], "gryffindor")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["channel"], "gryffindor-common-room")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["user"], "ron")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["replies"], None)
self.assertEqual(len(mattermost_data["post"]["direct_post"]), 7)
self.assertEqual(mattermost_data["post"]["direct_post"][0]["user"], "ron")
self.assertEqual(mattermost_data["post"]["direct_post"][0]["replies"], None)
self.assertEqual(mattermost_data["post"]["direct_post"][0]["message"], "hey harry")
self.assertEqual(mattermost_data["post"]["direct_post"][0]["channel_members"], ["ron", "harry"])
def test_process_user(self) -> None: def test_process_user(self) -> None:
user_id_mapper = IdMapper() user_id_mapper = IdMapper()
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
@ -112,6 +127,7 @@ class MatterMostImporter(ZulipTestCase):
team_name = "gryffindor" team_name = "gryffindor"
user_handler = UserHandler() user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 2)
self.assertTrue(user_id_mapper.has("harry")) self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron")) self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter") self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
@ -235,6 +251,43 @@ class MatterMostImporter(ZulipTestCase):
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def test_convert_huddle_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
huddle_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_huddle), 1)
huddle_members = mattermost_data["direct_channel"][1]["members"]
huddle_name = generate_huddle_name(huddle_members)
self.assertTrue(huddle_id_mapper.has(huddle_name))
self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def test_write_emoticon_data(self) -> None: def test_write_emoticon_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name) mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
@ -512,6 +565,97 @@ class MatterMostImporter(ZulipTestCase):
for message in messages: for message in messages:
self.assertIsNotNone(message.rendered_content) self.assertIsNotNone(message.rendered_content)
def test_do_convert_data_with_direct_messages(self) -> None:
mattermost_data_dir = self.fixture_file_name("direct_channel", "mattermost_fixtures")
output_dir = self.make_import_output_dir("mattermost")
do_convert_data(
mattermost_data_dir=mattermost_data_dir,
output_dir=output_dir,
masking_content=False
)
harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor")
self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'avatars')), True)
self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'emoji')), True)
self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'attachment.json')), True)
realm = self.read_file(harry_team_output_dir, 'realm.json')
self.assertEqual('Organization imported from Mattermost!',
realm['zerver_realm'][0]['description'])
exported_user_ids = self.get_set(realm['zerver_userprofile'], 'id')
exported_user_full_names = self.get_set(realm['zerver_userprofile'], 'full_name')
self.assertEqual(set(['Harry Potter', 'Ron Weasley', 'Ginny Weasley', 'Tom Riddle']), exported_user_full_names)
exported_user_emails = self.get_set(realm['zerver_userprofile'], 'email')
self.assertEqual(set(['harry@zulip.com', 'ron@zulip.com', 'ginny@zulip.com', 'voldemort@zulip.com']), exported_user_emails)
self.assertEqual(len(realm['zerver_stream']), 3)
exported_stream_names = self.get_set(realm['zerver_stream'], 'name')
self.assertEqual(exported_stream_names, set(['Gryffindor common room', 'Gryffindor quidditch team', 'Dumbledores army']))
self.assertEqual(self.get_set(realm['zerver_stream'], 'realm'), set([realm['zerver_realm'][0]['id']]))
self.assertEqual(self.get_set(realm['zerver_stream'], 'deactivated'), set([False]))
self.assertEqual(len(realm['zerver_defaultstream']), 0)
exported_recipient_ids = self.get_set(realm['zerver_recipient'], 'id')
self.assertEqual(len(exported_recipient_ids), 8)
exported_recipient_types = self.get_set(realm['zerver_recipient'], 'type')
self.assertEqual(exported_recipient_types, set([1, 2, 3]))
exported_recipient_type_ids = self.get_set(realm['zerver_recipient'], 'type_id')
self.assertEqual(len(exported_recipient_type_ids), 4)
exported_subscription_userprofile = self.get_set(realm['zerver_subscription'], 'user_profile')
self.assertEqual(len(exported_subscription_userprofile), 4)
exported_subscription_recipients = self.get_set(realm['zerver_subscription'], 'recipient')
self.assertEqual(len(exported_subscription_recipients), 8)
messages = self.read_file(harry_team_output_dir, 'messages-000001.json')
exported_messages_id = self.get_set(messages['zerver_message'], 'id')
self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids)
self.assertIn(messages['zerver_message'][0]['recipient'], exported_recipient_ids)
self.assertIn(messages['zerver_message'][0]['content'], 'ron joined the channel.\n\n')
exported_usermessage_userprofiles = self.get_set(messages['zerver_usermessage'], 'user_profile')
self.assertEqual(len(exported_usermessage_userprofiles), 3)
exported_usermessage_messages = self.get_set(messages['zerver_usermessage'], 'message')
self.assertEqual(exported_usermessage_messages, exported_messages_id)
do_import_realm(
import_dir=harry_team_output_dir,
subdomain='gryffindor'
)
realm = get_realm('gryffindor')
messages = Message.objects.filter(sender__realm=realm)
for message in messages:
self.assertIsNotNone(message.rendered_content)
self.assertEqual(len(messages), 11)
stream_messages = messages.filter(recipient__type=Recipient.STREAM).order_by("pub_date")
stream_recipients = stream_messages.values_list("recipient", flat=True)
self.assertEqual(len(stream_messages), 4)
self.assertEqual(len(set(stream_recipients)), 2)
self.assertEqual(stream_messages[0].sender.email, "ron@zulip.com")
self.assertEqual(stream_messages[0].content, "ron joined the channel.\n\n")
huddle_messages = messages.filter(recipient__type=Recipient.HUDDLE).order_by("pub_date")
huddle_recipients = huddle_messages.values_list("recipient", flat=True)
self.assertEqual(len(huddle_messages), 3)
self.assertEqual(len(set(huddle_recipients)), 1)
self.assertEqual(huddle_messages[0].sender.email, "ginny@zulip.com")
self.assertEqual(huddle_messages[0].content, "Who is going to Hogesmead this weekend?\n\n")
personal_messages = messages.filter(recipient__type=Recipient.PERSONAL).order_by("pub_date")
personal_recipients = personal_messages.values_list("recipient", flat=True)
self.assertEqual(len(personal_messages), 4)
self.assertEqual(len(set(personal_recipients)), 3)
self.assertEqual(personal_messages[0].sender.email, "ron@zulip.com")
self.assertEqual(personal_messages[0].content, "hey harry\n\n")
def test_do_convert_data_with_masking(self) -> None: def test_do_convert_data_with_masking(self) -> None:
mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures") mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures")
output_dir = self.make_import_output_dir("mattermost") output_dir = self.make_import_output_dir("mattermost")