Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

import: Add function to normalize messages to import. 

This adds `normalize_body_for_import` to normalize messages from
third-party importers by removing NUL bytes and also updates import
test files data to test this.

Fixes #31930.
This commit is contained in:
Harsh 2024-10-21 19:39:02 +05:30 committed by Tim Abbott
parent d556c0e0a5
commit 858fdeee39
3 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
zerver/data_import/import_util.py
View File

@ -17,6 +17,7 @@ from django.utils.timezone import now as timezone_now
from zerver.data_import.sequencer import NEXT_ID
from zerver.lib.avatar_hash import user_avatar_base_path_from_ids
from zerver.lib.message import normalize_body_for_import
from zerver.lib.mime_types import guess_extension
from zerver.lib.partial import partial
from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS as STREAM_COLORS
@ -499,6 +500,8 @@ def build_message(
has_link: bool = False,
has_attachment: bool = True,
) -> ZerverFieldsT:
# check and remove NULL Bytes if any.
content = normalize_body_for_import(content)
zulip_message = Message(
rendered_content_version=1, # this is Zulip specific
id=message_id,

6
zerver/lib/message.py
View File

@ -196,6 +196,12 @@ def normalize_body(body: str) -> str:
return truncate_content(body, settings.MAX_MESSAGE_LENGTH, "\n[message truncated]")
def normalize_body_for_import(body: str) -> str:
if "\x00" in body:
body = re.sub(r"\x00", "", body)
return truncate_content(body, settings.MAX_MESSAGE_LENGTH, "\n[message truncated]")
def truncate_topic(topic_name: str) -> str:
return truncate_content(topic_name, MAX_TOPIC_NAME_LENGTH, "...")

2
zerver/tests/fixtures/mattermost_fixtures/direct_channel/export.json vendored
View File

@ -17,7 +17,7 @@
{"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"ron","message":"hey harry","create_at":1566376137676,"flagged_by":null,"reactions":null,"replies":null,"attachments":[{"path":"20210622/teams/noteam/channels/mcrm7xee5bnpzn7u9ktsd91dwy/users/knq189b88fdxbdkeeasdynia4o/o3to4ezua3bajj31mzpkn96n5e/harry-ron.jpg"}]}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry"],"user":"harry","message":"what's up","create_at":1566376318568,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ginny","message":"Who is going to Hogsmeade this weekend?","create_at":1566376226493,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"I am going.","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"harry","message":"\u0000\u0001\u0001Hello How Are you\u0001\u0000\u0000\u0000\u0000","create_at":1566376311350,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["ron","harry","ginny"],"user":"ron","message":"I am going as well","create_at":1566376286363,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"voldemort","message":"Hey Harry.","create_at":1566376318569,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}
{"type":"direct_post","direct_post":{"channel_members":["harry","voldemort"],"user":"harry","message":"Ahh. Here we go again.","create_at":1566376318579,"flagged_by":null,"reactions":null,"replies":null,"attachments":null}}