import: Add function to normalize messages to import.

This adds `normalize_body_for_import` to normalize messages from
third-party importers by removing NUL bytes and also updates import
test files data to test this.

Fixes #31930.

zerver/data_import/import_util.py

@@ -17,6 +17,7 @@ from django.utils.timezone import now as timezone_now
 
 from zerver.data_import.sequencer import NEXT_ID
 from zerver.lib.avatar_hash import user_avatar_base_path_from_ids
+from zerver.lib.message import normalize_body_for_import
 from zerver.lib.mime_types import guess_extension
 from zerver.lib.partial import partial
 from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS as STREAM_COLORS
@@ -499,6 +500,8 @@ def build_message(
     has_link: bool = False,
     has_attachment: bool = True,
 ) -> ZerverFieldsT:
+    # check and remove NULL Bytes if any.
+    content = normalize_body_for_import(content)
     zulip_message = Message(
         rendered_content_version=1,  # this is Zulip specific
         id=message_id,

zerver/lib/message.py

@@ -196,6 +196,12 @@ def normalize_body(body: str) -> str:
     return truncate_content(body, settings.MAX_MESSAGE_LENGTH, "\n[message truncated]")
 
 
+def normalize_body_for_import(body: str) -> str:
+    if "\x00" in body:
+        body = re.sub(r"\x00", "", body)
+    return truncate_content(body, settings.MAX_MESSAGE_LENGTH, "\n[message truncated]")
+
+
 def truncate_topic(topic_name: str) -> str:
     return truncate_content(topic_name, MAX_TOPIC_NAME_LENGTH, "...")
 

zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json

@@ -17,7 +17,7 @@
 {"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"ron","message":"hey harry","create_at":1566376137676,"flagged_by":null,"reactions":null,"replies":null,"attachments":[{"path":"20210622/teams/noteam/channels/mcrm7xee5bnpzn7u9ktsd91dwy/users/knq189b88fdxbdkeeasdynia4o/o3to4ezua3bajj31mzpkn96n5e/harry-ron.jpg"}]}}
 {"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"harry","message":"what's up","create_at":1566376318568,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
 {"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ginny","message":"Who is going to Hogsmeade this weekend?","create_at":1566376226493,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
-{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"I am going.","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
+{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"\u0000\u0001\u0001Hello How Are you\u0001\u0000\u0000\u0000\u0000","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
 {"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ron","message":"I am going as well","create_at":1566376286363,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
 {"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"voldemort","message":"Hey Harry.","create_at":1566376318569,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
 {"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"harry","message":"Ahh. Here we go again.","create_at":1566376318579,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}