diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py index 3953f58697..630ba4fdc2 100644 --- a/zerver/data_import/import_util.py +++ b/zerver/data_import/import_util.py @@ -17,6 +17,7 @@ from django.utils.timezone import now as timezone_now from zerver.data_import.sequencer import NEXT_ID from zerver.lib.avatar_hash import user_avatar_base_path_from_ids +from zerver.lib.message import normalize_body_for_import from zerver.lib.mime_types import guess_extension from zerver.lib.partial import partial from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS as STREAM_COLORS @@ -499,6 +500,8 @@ def build_message( has_link: bool = False, has_attachment: bool = True, ) -> ZerverFieldsT: + # check and remove NULL Bytes if any. + content = normalize_body_for_import(content) zulip_message = Message( rendered_content_version=1, # this is Zulip specific id=message_id, diff --git a/zerver/lib/message.py b/zerver/lib/message.py index 570e9b39d5..07cb28f6a4 100644 --- a/zerver/lib/message.py +++ b/zerver/lib/message.py @@ -196,6 +196,12 @@ def normalize_body(body: str) -> str: return truncate_content(body, settings.MAX_MESSAGE_LENGTH, "\n[message truncated]") +def normalize_body_for_import(body: str) -> str: + if "\x00" in body: + body = re.sub(r"\x00", "", body) + return truncate_content(body, settings.MAX_MESSAGE_LENGTH, "\n[message truncated]") + + def truncate_topic(topic_name: str) -> str: return truncate_content(topic_name, MAX_TOPIC_NAME_LENGTH, "...") diff --git a/zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json b/zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json index be785e3875..73762c80f1 100644 --- a/zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json +++ b/zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json @@ -17,7 +17,7 @@ {"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"ron","message":"hey harry","create_at":1566376137676,"flagged_by":null,"reactions":null,"replies":null,"attachments":[{"path":"20210622/teams/noteam/channels/mcrm7xee5bnpzn7u9ktsd91dwy/users/knq189b88fdxbdkeeasdynia4o/o3to4ezua3bajj31mzpkn96n5e/harry-ron.jpg"}]}} {"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"harry","message":"what's up","create_at":1566376318568,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}} {"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ginny","message":"Who is going to Hogsmeade this weekend?","create_at":1566376226493,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}} -{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"I am going.","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}} +{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"\u0000\u0001\u0001Hello How Are you\u0001\u0000\u0000\u0000\u0000","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}} {"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ron","message":"I am going as well","create_at":1566376286363,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}} {"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"voldemort","message":"Hey Harry.","create_at":1566376318569,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}} {"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"harry","message":"Ahh. Here we go again.","create_at":1566376318579,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}