From 61d0417e75a09075d9a61ac0ddd959efd6934478 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 6 Aug 2020 16:09:47 -0700 Subject: [PATCH] python: Replace ujson with orjson. Fixes #6507. Signed-off-by: Anders Kaseorg --- analytics/tests/test_counts.py | 4 +- analytics/tests/test_views.py | 4 +- corporate/lib/stripe.py | 13 +- corporate/tests/test_stripe.py | 61 +-- requirements/common.in | 4 +- requirements/dev.txt | 20 +- requirements/prod.txt | 20 +- stubs/orjson.pyi | 27 ++ tools/check-node-fixtures | 4 +- tools/create-test-api-docs | 9 +- tools/generate-integration-docs-screenshot | 15 +- tools/lib/sanity_check.py | 7 +- tools/setup/emoji/build_emoji | 20 +- tools/setup/emoji/export_emoji_names_to_csv | 6 +- tools/setup/emoji/generate_emoji_names_table | 14 +- tools/test-backend | 10 +- tools/test-js-with-node | 6 +- tools/test-locked-requirements | 14 +- version.py | 2 +- zerver/data_import/gitter.py | 10 +- zerver/data_import/hipchat.py | 18 +- zerver/data_import/import_util.py | 6 +- zerver/data_import/mattermost.py | 6 +- zerver/data_import/slack.py | 8 +- zerver/decorator.py | 4 +- zerver/lib/actions.py | 98 ++--- zerver/lib/create_user.py | 4 +- zerver/lib/emoji.py | 6 +- zerver/lib/export.py | 45 ++- zerver/lib/generate_test_data.py | 10 +- zerver/lib/i18n.py | 10 +- zerver/lib/import_realm.py | 30 +- zerver/lib/markdown/testing_mocks.py | 10 +- zerver/lib/message.py | 8 +- zerver/lib/push_notifications.py | 4 +- zerver/lib/queue.py | 8 +- zerver/lib/redis_utils.py | 6 +- zerver/lib/remote_server.py | 18 +- zerver/lib/request.py | 6 +- zerver/lib/response.py | 27 +- zerver/lib/send_email.py | 6 +- zerver/lib/test_classes.py | 20 +- zerver/lib/test_helpers.py | 27 +- zerver/lib/validator.py | 4 +- .../commands/add_users_to_mailing_list.py | 4 +- zerver/management/commands/compilemessages.py | 24 +- zerver/management/commands/enqueue_file.py | 4 +- .../commands/send_to_email_mirror.py | 6 +- .../commands/send_webhook_fixture_message.py | 10 +- .../migrations/0097_reactions_emoji_code.py | 5 +- zerver/migrations/0102_convert_muted_topic.py | 4 +- .../0209_user_profile_no_empty_password.py | 6 +- zerver/migrations/0277_migrate_alert_word.py | 6 +- ...84_convert_realm_admins_to_realm_owners.py | 6 +- zerver/tests/test_alert_words.py | 14 +- zerver/tests/test_audit_log.py | 44 +- zerver/tests/test_auth_backends.py | 48 +-- zerver/tests/test_bots.py | 54 +-- zerver/tests/test_custom_profile_data.py | 112 +++--- zerver/tests/test_decorators.py | 18 +- zerver/tests/test_docs.py | 4 +- zerver/tests/test_drafts.py | 36 +- zerver/tests/test_email_mirror.py | 14 +- zerver/tests/test_email_notifications.py | 22 +- zerver/tests/test_embedded_bot_system.py | 4 +- zerver/tests/test_event_queue.py | 12 +- zerver/tests/test_event_system.py | 76 ++-- zerver/tests/test_events.py | 22 +- zerver/tests/test_gitter_importer.py | 6 +- zerver/tests/test_home.py | 8 +- zerver/tests/test_hotspots.py | 6 +- zerver/tests/test_i18n.py | 4 +- zerver/tests/test_import_export.py | 12 +- zerver/tests/test_integrations_dev_panel.py | 28 +- zerver/tests/test_link_embed.py | 8 +- zerver/tests/test_management_commands.py | 14 +- zerver/tests/test_markdown.py | 6 +- zerver/tests/test_mattermost_importer.py | 10 +- zerver/tests/test_message_edit.py | 36 +- zerver/tests/test_message_fetch.py | 92 ++--- zerver/tests/test_message_flags.py | 34 +- zerver/tests/test_message_send.py | 52 +-- zerver/tests/test_outgoing_webhook_system.py | 16 +- zerver/tests/test_push_notifications.py | 30 +- zerver/tests/test_queue_worker.py | 12 +- zerver/tests/test_reactions.py | 6 +- zerver/tests/test_realm.py | 86 ++-- zerver/tests/test_realm_domains.py | 18 +- zerver/tests/test_realm_export.py | 10 +- zerver/tests/test_report.py | 4 +- zerver/tests/test_service_bot_system.py | 16 +- zerver/tests/test_settings.py | 24 +- zerver/tests/test_signup.py | 14 +- zerver/tests/test_slack_importer.py | 18 +- zerver/tests/test_slack_message_conversion.py | 6 +- zerver/tests/test_subs.py | 376 +++++++++--------- zerver/tests/test_thumbnail.py | 14 +- zerver/tests/test_tornado.py | 6 +- zerver/tests/test_tutorial.py | 4 +- zerver/tests/test_typing.py | 18 +- zerver/tests/test_upload.py | 28 +- zerver/tests/test_urls.py | 4 +- zerver/tests/test_user_groups.py | 44 +- zerver/tests/test_user_status.py | 6 +- zerver/tests/test_users.py | 68 ++-- zerver/tests/test_widgets.py | 14 +- zerver/tests/test_zephyr.py | 4 +- zerver/tornado/django_api.py | 18 +- zerver/tornado/event_queue.py | 13 +- zerver/tornado/views.py | 4 +- zerver/views/custom_profile_fields.py | 8 +- zerver/views/development/email_log.py | 4 +- zerver/views/development/integrations.py | 6 +- zerver/views/message_edit.py | 4 +- zerver/views/message_fetch.py | 4 +- zerver/views/portico.py | 6 +- zerver/views/realm_export.py | 4 +- zerver/views/streams.py | 4 +- zerver/views/submessage.py | 4 +- zerver/views/zephyr.py | 4 +- zerver/webhooks/front/tests.py | 6 +- zerver/webhooks/hellosign/view.py | 4 +- zerver/webhooks/librato/view.py | 6 +- zerver/webhooks/pivotal/view.py | 4 +- zerver/webhooks/semaphore/tests.py | 4 +- zerver/webhooks/slack_incoming/view.py | 6 +- zerver/webhooks/teamcity/tests.py | 4 +- zerver/webhooks/trello/view/__init__.py | 4 +- zerver/worker/queue_processors.py | 30 +- zilencer/management/commands/populate_db.py | 10 +- .../management/commands/render_messages.py | 14 +- zproject/backends.py | 6 +- 132 files changed, 1319 insertions(+), 1238 deletions(-) create mode 100644 stubs/orjson.pyi diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index f911891d06..17b13be61c 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple, Type from unittest import mock -import ujson +import orjson from django.apps import apps from django.db import models from django.db.models import Sum @@ -1393,7 +1393,7 @@ class TestRealmActiveHumans(AnalyticsTestCase): end_time = self.TIME_ZERO UserCount.objects.create( user=user, realm=user.realm, property='active_users_audit:is_bot:day', - subgroup=ujson.dumps(user.is_bot), end_time=end_time, value=1) + subgroup=orjson.dumps(user.is_bot).decode(), end_time=end_time, value=1) def mark_15day_active(self, user: UserProfile, end_time: Optional[datetime]=None) -> None: if end_time is None: diff --git a/analytics/tests/test_views.py b/analytics/tests/test_views.py index 170c454edc..6376e5d46b 100644 --- a/analytics/tests/test_views.py +++ b/analytics/tests/test_views.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta, timezone from typing import List, Optional from unittest import mock -import ujson +import orjson from django.http import HttpResponse from django.utils.timezone import now as timezone_now @@ -547,7 +547,7 @@ class TestSupportEndpoint(ZulipTestCase): stream_ids = [self.get_stream_id("Denmark")] invitee_emails = [self.nonreg_email("test1")] self.client_post("/json/invites", {"invitee_emails": invitee_emails, - "stream_ids": ujson.dumps(stream_ids), + "stream_ids": orjson.dumps(stream_ids).decode(), "invite_as": PreregistrationUser.INVITE_AS['MEMBER']}) result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")}) check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True) diff --git a/corporate/lib/stripe.py b/corporate/lib/stripe.py index dedea107e5..db23a57436 100644 --- a/corporate/lib/stripe.py +++ b/corporate/lib/stripe.py @@ -6,8 +6,8 @@ from decimal import Decimal from functools import wraps from typing import Callable, Dict, Optional, Tuple, TypeVar, cast +import orjson import stripe -import ujson from django.conf import settings from django.core.signing import Signer from django.db import transaction @@ -297,10 +297,10 @@ def make_end_of_cycle_updates_if_needed(plan: CustomerPlan, RealmAuditLog.objects.create( realm=new_plan.customer.realm, event_time=event_time, event_type=RealmAuditLog.CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ "monthly_plan_id": plan.id, "annual_plan_id": new_plan.id, - }) + }).decode() ) return new_plan, new_plan_ledger_entry @@ -348,6 +348,11 @@ def compute_plan_parameters( next_invoice_date = period_end return billing_cycle_anchor, next_invoice_date, period_end, price_per_license +def decimal_to_float(obj: object) -> object: + if isinstance(obj, Decimal): + return float(obj) + raise TypeError # nocoverage + # Only used for cloud signups @catch_stripe_errors def process_initial_upgrade(user: UserProfile, licenses: int, automanage_licenses: bool, @@ -424,7 +429,7 @@ def process_initial_upgrade(user: UserProfile, licenses: int, automanage_license RealmAuditLog.objects.create( realm=realm, acting_user=user, event_time=billing_cycle_anchor, event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED, - extra_data=ujson.dumps(plan_params)) + extra_data=orjson.dumps(plan_params, default=decimal_to_float).decode()) if not free_trial: stripe.InvoiceItem.create( diff --git a/corporate/tests/test_stripe.py b/corporate/tests/test_stripe.py index 136a0383d1..752fc09863 100644 --- a/corporate/tests/test_stripe.py +++ b/corporate/tests/test_stripe.py @@ -9,9 +9,9 @@ from functools import wraps from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, TypeVar, cast from unittest.mock import Mock, patch +import orjson import responses import stripe -import ujson from django.conf import settings from django.core import signing from django.http import HttpResponse @@ -127,7 +127,8 @@ def read_stripe_fixture(decorated_function_name: str, def _read_stripe_fixture(*args: Any, **kwargs: Any) -> Any: mock = operator.attrgetter(mocked_function_name)(sys.modules[__name__]) fixture_path = stripe_fixture_path(decorated_function_name, mocked_function_name, mock.call_count) - fixture = ujson.load(open(fixture_path)) + with open(fixture_path, "rb") as f: + fixture = orjson.loads(f.read()) # Check for StripeError fixtures if "json_body" in fixture: requestor = stripe.api_requestor.APIRequestor() @@ -331,7 +332,7 @@ class StripeTestCase(ZulipTestCase): if key in params: del params[key] for key, value in params.items(): - params[key] = ujson.dumps(value) + params[key] = orjson.dumps(value).decode() return self.client_post("/json/billing/upgrade", params, **host_args) # Upgrade without talking to Stripe @@ -469,7 +470,7 @@ class StripeTest(StripeTestCase): # TODO: Check for REALM_PLAN_TYPE_CHANGED # (RealmAuditLog.REALM_PLAN_TYPE_CHANGED, Kandra()), ]) - self.assertEqual(ujson.loads(RealmAuditLog.objects.filter( + self.assertEqual(orjson.loads(RealmAuditLog.objects.filter( event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED).values_list( 'extra_data', flat=True).first())['automanage_licenses'], True) # Check that we correctly updated Realm @@ -554,7 +555,7 @@ class StripeTest(StripeTestCase): # TODO: Check for REALM_PLAN_TYPE_CHANGED # (RealmAuditLog.REALM_PLAN_TYPE_CHANGED, Kandra()), ]) - self.assertEqual(ujson.loads(RealmAuditLog.objects.filter( + self.assertEqual(orjson.loads(RealmAuditLog.objects.filter( event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED).values_list( 'extra_data', flat=True).first())['automanage_licenses'], False) # Check that we correctly updated Realm @@ -631,7 +632,7 @@ class StripeTest(StripeTestCase): # TODO: Check for REALM_PLAN_TYPE_CHANGED # (RealmAuditLog.REALM_PLAN_TYPE_CHANGED, Kandra()), ]) - self.assertEqual(ujson.loads(RealmAuditLog.objects.filter( + self.assertEqual(orjson.loads(RealmAuditLog.objects.filter( event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED).values_list( 'extra_data', flat=True).first())['automanage_licenses'], True) @@ -785,7 +786,7 @@ class StripeTest(StripeTestCase): # TODO: Check for REALM_PLAN_TYPE_CHANGED # (RealmAuditLog.REALM_PLAN_TYPE_CHANGED, Kandra()), ]) - self.assertEqual(ujson.loads(RealmAuditLog.objects.filter( + self.assertEqual(orjson.loads(RealmAuditLog.objects.filter( event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED).values_list( 'extra_data', flat=True).first())['automanage_licenses'], False) @@ -977,7 +978,7 @@ class StripeTest(StripeTestCase): self.login_user(hamlet) response = self.upgrade(talk_to_stripe=False, salt='badsalt') self.assert_json_error_contains(response, "Something went wrong. Please contact") - self.assertEqual(ujson.loads(response.content)['error_description'], 'tampered seat count') + self.assertEqual(orjson.loads(response.content)['error_description'], 'tampered seat count') def test_upgrade_race_condition(self) -> None: hamlet = self.example_user('hamlet') @@ -995,7 +996,7 @@ class StripeTest(StripeTestCase): del_args: Sequence[str] = []) -> None: response = self.upgrade(talk_to_stripe=False, del_args=del_args, **upgrade_params) self.assert_json_error_contains(response, "Something went wrong. Please contact") - self.assertEqual(ujson.loads(response.content)['error_description'], error_description) + self.assertEqual(orjson.loads(response.content)['error_description'], error_description) hamlet = self.example_user('hamlet') self.login_user(hamlet) @@ -1015,13 +1016,13 @@ class StripeTest(StripeTestCase): response = self.upgrade(invoice=invoice, talk_to_stripe=False, del_args=del_args, **upgrade_params) self.assert_json_error_contains(response, f"at least {min_licenses_in_response} users") - self.assertEqual(ujson.loads(response.content)['error_description'], 'not enough licenses') + self.assertEqual(orjson.loads(response.content)['error_description'], 'not enough licenses') def check_max_licenses_error(licenses: int) -> None: response = self.upgrade(invoice=True, talk_to_stripe=False, licenses=licenses) self.assert_json_error_contains(response, f"with more than {MAX_INVOICED_LICENSES} licenses") - self.assertEqual(ujson.loads(response.content)['error_description'], 'too many licenses') + self.assertEqual(orjson.loads(response.content)['error_description'], 'too many licenses') def check_success(invoice: bool, licenses: Optional[int], upgrade_params: Dict[str, Any]={}) -> None: if licenses is None: @@ -1072,7 +1073,7 @@ class StripeTest(StripeTestCase): with patch("corporate.views.process_initial_upgrade", side_effect=Exception): response = self.upgrade(talk_to_stripe=False) self.assert_json_error_contains(response, "Something went wrong. Please contact desdemona+admin@zulip.com.") - self.assertEqual(ujson.loads(response.content)['error_description'], 'uncaught exception during upgrade') + self.assertEqual(orjson.loads(response.content)['error_description'], 'uncaught exception during upgrade') def test_request_sponsorship(self) -> None: user = self.example_user("hamlet") @@ -1081,9 +1082,9 @@ class StripeTest(StripeTestCase): self.login_user(user) data = { - "organization-type": ujson.dumps("Open-source"), - "website": ujson.dumps("https://infinispan.org/"), - "description": ujson.dumps("Infinispan is a distributed in-memory key/value data store with optional schema."), + "organization-type": orjson.dumps("Open-source").decode(), + "website": orjson.dumps("https://infinispan.org/").decode(), + "description": orjson.dumps("Infinispan is a distributed in-memory key/value data store with optional schema.").decode(), } response = self.client_post("/json/billing/sponsorship", data) self.assert_json_success(response) @@ -1281,10 +1282,10 @@ class StripeTest(StripeTestCase): with patch("corporate.lib.stripe.billing_logger.error") as mock_billing_logger: with patch("stripe.Invoice.list") as mock_invoice_list: response = self.client_post("/json/billing/sources/change", - {'stripe_token': ujson.dumps(stripe_token)}) + {'stripe_token': orjson.dumps(stripe_token).decode()}) mock_billing_logger.assert_called() mock_invoice_list.assert_not_called() - self.assertEqual(ujson.loads(response.content)['error_description'], 'card error') + self.assertEqual(orjson.loads(response.content)['error_description'], 'card error') self.assert_json_error_contains(response, 'Your card was declined') for stripe_source in stripe_get_customer(stripe_customer_id).sources: assert isinstance(stripe_source, stripe.Card) @@ -1295,9 +1296,9 @@ class StripeTest(StripeTestCase): stripe_token = stripe_create_token(card_number='4000000000000341').id with patch("corporate.lib.stripe.billing_logger.error") as mock_billing_logger: response = self.client_post("/json/billing/sources/change", - {'stripe_token': ujson.dumps(stripe_token)}) + {'stripe_token': orjson.dumps(stripe_token).decode()}) mock_billing_logger.assert_called() - self.assertEqual(ujson.loads(response.content)['error_description'], 'card error') + self.assertEqual(orjson.loads(response.content)['error_description'], 'card error') self.assert_json_error_contains(response, 'Your card was declined') for stripe_source in stripe_get_customer(stripe_customer_id).sources: assert isinstance(stripe_source, stripe.Card) @@ -1309,7 +1310,7 @@ class StripeTest(StripeTestCase): # Replace with a valid card stripe_token = stripe_create_token(card_number='5555555555554444').id response = self.client_post("/json/billing/sources/change", - {'stripe_token': ujson.dumps(stripe_token)}) + {'stripe_token': orjson.dumps(stripe_token).decode()}) self.assert_json_success(response) number_of_sources = 0 for stripe_source in stripe_get_customer(stripe_customer_id).sources: @@ -1435,8 +1436,8 @@ class StripeTest(StripeTestCase): self.assertEqual(annual_ledger_entries.values_list('licenses', 'licenses_at_next_renewal')[1], (25, 25)) audit_log = RealmAuditLog.objects.get(event_type=RealmAuditLog.CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN) self.assertEqual(audit_log.realm, user.realm) - self.assertEqual(ujson.loads(audit_log.extra_data)["monthly_plan_id"], monthly_plan.id) - self.assertEqual(ujson.loads(audit_log.extra_data)["annual_plan_id"], annual_plan.id) + self.assertEqual(orjson.loads(audit_log.extra_data)["monthly_plan_id"], monthly_plan.id) + self.assertEqual(orjson.loads(audit_log.extra_data)["annual_plan_id"], annual_plan.id) invoice_plans_as_needed(self.next_month) @@ -1839,7 +1840,7 @@ class RequiresBillingAccessTest(ZulipTestCase): self.login_user(self.example_user('hamlet')) with patch("corporate.views.do_replace_payment_source") as mocked1: response = self.client_post("/json/billing/sources/change", - {'stripe_token': ujson.dumps('token')}) + {'stripe_token': orjson.dumps('token').decode()}) self.assert_json_success(response) mocked1.assert_called() @@ -1847,7 +1848,7 @@ class RequiresBillingAccessTest(ZulipTestCase): self.login_user(self.example_user('desdemona')) with patch("corporate.views.do_replace_payment_source") as mocked2: response = self.client_post("/json/billing/sources/change", - {'stripe_token': ujson.dumps('token')}) + {'stripe_token': orjson.dumps('token').decode()}) self.assert_json_success(response) mocked2.assert_called() @@ -1858,21 +1859,21 @@ class RequiresBillingAccessTest(ZulipTestCase): self.assert_json_error_contains(response, error_message) verify_user_cant_access_endpoint("polonius", "/json/billing/upgrade", - {'billing_modality': ujson.dumps("charge_automatically"), 'schedule': ujson.dumps("annual"), - 'signed_seat_count': ujson.dumps('signed count'), 'salt': ujson.dumps('salt')}, + {'billing_modality': orjson.dumps("charge_automatically").decode(), 'schedule': orjson.dumps("annual").decode(), + 'signed_seat_count': orjson.dumps('signed count').decode(), 'salt': orjson.dumps('salt').decode()}, "Must be an organization member") verify_user_cant_access_endpoint("polonius", "/json/billing/sponsorship", - {'organization-type': ujson.dumps("event"), 'description': ujson.dumps("event description"), - 'website': ujson.dumps("example.com")}, + {'organization-type': orjson.dumps("event").decode(), 'description': orjson.dumps("event description").decode(), + 'website': orjson.dumps("example.com").decode()}, "Must be an organization member") for username in ["cordelia", "iago"]: self.login_user(self.example_user(username)) - verify_user_cant_access_endpoint(username, "/json/billing/sources/change", {'stripe_token': ujson.dumps('token')}, + verify_user_cant_access_endpoint(username, "/json/billing/sources/change", {'stripe_token': orjson.dumps('token').decode()}, "Must be a billing administrator or an organization owner") - verify_user_cant_access_endpoint(username, "/json/billing/plan/change", {'status': ujson.dumps(1)}, + verify_user_cant_access_endpoint(username, "/json/billing/plan/change", {'status': orjson.dumps(1).decode()}, "Must be a billing administrator or an organization owner") # Make sure that we are testing all the JSON endpoints diff --git a/requirements/common.in b/requirements/common.in index 179e07de7e..9a5053c619 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -101,9 +101,7 @@ sourcemap tornado==4.* # https://github.com/zulip/zulip/issues/8913 # Fast JSON parser -# Forked for an issue related to unpaired Unicode surrogates: -# https://github.com/zulip/zulip/issues/6332, https://github.com/esnme/ultrajson/pull/284 -https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git +orjson # Django extension for serving webpack modules django-webpack4-loader diff --git a/requirements/dev.txt b/requirements/dev.txt index 8fdb580c5a..b2fd31c1a7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -644,6 +644,23 @@ openapi-spec-validator==0.2.9 \ --hash=sha256:79381a69b33423ee400ae1624a461dae7725e450e2e306e32f2dd8d16a4d85cb \ --hash=sha256:ec1b01a00e20955a527358886991ae34b4b791b253027ee9f7df5f84b59d91c7 \ # via openapi-core +orjson==3.3.0 \ + --hash=sha256:00fba178d361db5fa12787dae018c8df25207feb7eb79ec52b70259dbeab96d0 \ + --hash=sha256:0d60993e1787d6d4e2bad7aac674a66bed0012251fa45c73dfe8ef9fe0b4433a \ + --hash=sha256:262aa8e8b47142d4d51dd7281afa51d2281480eb672183e79f2d90572f535e69 \ + --hash=sha256:28a401039e9a228ca72a0e0a1c43c80df85336ade0bf17e592e386c19ad49f47 \ + --hash=sha256:44f7344aa100c43a86716f2cf92163325792ebb8af3542f3ee084f328328207b \ + --hash=sha256:5e80e377979909283a221a3f62add9186b3aaa1028a469d6282a4d3694e04a05 \ + --hash=sha256:5ec5d1e127fc30addfb6c1d23ecc4b82ac89f0ca808572b553bcee913eb134be \ + --hash=sha256:64569ad07ae564ebdb712bfb199de521ce5dfa78d9459d70787cac3811f01797 \ + --hash=sha256:657415c522d04427d1ac46ee3bde525deb199bd494c2480db40f991755ee038e \ + --hash=sha256:67136be5a6ce5ad9995f0e178851d4956e8e0175ac1e8bb80987607c6986e970 \ + --hash=sha256:67b2ed9410b426cbaa4e49a484c67225f58bc57e84a9d593006140bcd70909ae \ + --hash=sha256:8649b36460f9ab5d2d3d4bea69cb608b0ebe9f70d7119478a59632cfe6554fd4 \ + --hash=sha256:d33524b1aab1a34b6fdc8ae7bad83e495eea5a02712dd860113223ed6fc82106 \ + --hash=sha256:d37758bc0351fee5a2b7387f61677b893b02c6b7351370afbf63ce82bd1f25cd \ + --hash=sha256:e81c14e31b4bb48b2ba6e9a47460e8cd0406f927741491859ab3bec53f447317 \ + # via -r requirements/common.in packaging==20.4 \ --hash=sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8 \ --hash=sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181 \ @@ -1161,9 +1178,6 @@ ua-parser==0.10.0 \ uhashring==1.2 \ --hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \ # via python-binary-memcached -https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \ - --hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \ - # via -r requirements/common.in urllib3==1.25.10 \ --hash=sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a \ --hash=sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461 \ diff --git a/requirements/prod.txt b/requirements/prod.txt index 2cfb40e5d1..10f9daf82b 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -453,6 +453,23 @@ openapi-spec-validator==0.2.9 \ --hash=sha256:79381a69b33423ee400ae1624a461dae7725e450e2e306e32f2dd8d16a4d85cb \ --hash=sha256:ec1b01a00e20955a527358886991ae34b4b791b253027ee9f7df5f84b59d91c7 \ # via openapi-core +orjson==3.3.0 \ + --hash=sha256:00fba178d361db5fa12787dae018c8df25207feb7eb79ec52b70259dbeab96d0 \ + --hash=sha256:0d60993e1787d6d4e2bad7aac674a66bed0012251fa45c73dfe8ef9fe0b4433a \ + --hash=sha256:262aa8e8b47142d4d51dd7281afa51d2281480eb672183e79f2d90572f535e69 \ + --hash=sha256:28a401039e9a228ca72a0e0a1c43c80df85336ade0bf17e592e386c19ad49f47 \ + --hash=sha256:44f7344aa100c43a86716f2cf92163325792ebb8af3542f3ee084f328328207b \ + --hash=sha256:5e80e377979909283a221a3f62add9186b3aaa1028a469d6282a4d3694e04a05 \ + --hash=sha256:5ec5d1e127fc30addfb6c1d23ecc4b82ac89f0ca808572b553bcee913eb134be \ + --hash=sha256:64569ad07ae564ebdb712bfb199de521ce5dfa78d9459d70787cac3811f01797 \ + --hash=sha256:657415c522d04427d1ac46ee3bde525deb199bd494c2480db40f991755ee038e \ + --hash=sha256:67136be5a6ce5ad9995f0e178851d4956e8e0175ac1e8bb80987607c6986e970 \ + --hash=sha256:67b2ed9410b426cbaa4e49a484c67225f58bc57e84a9d593006140bcd70909ae \ + --hash=sha256:8649b36460f9ab5d2d3d4bea69cb608b0ebe9f70d7119478a59632cfe6554fd4 \ + --hash=sha256:d33524b1aab1a34b6fdc8ae7bad83e495eea5a02712dd860113223ed6fc82106 \ + --hash=sha256:d37758bc0351fee5a2b7387f61677b893b02c6b7351370afbf63ce82bd1f25cd \ + --hash=sha256:e81c14e31b4bb48b2ba6e9a47460e8cd0406f927741491859ab3bec53f447317 \ + # via -r requirements/common.in parse==1.16.0 \ --hash=sha256:cd89e57aed38dcf3e0ff8253f53121a3b23e6181758993323658bffc048a5c19 \ # via openapi-core @@ -759,9 +776,6 @@ ua-parser==0.10.0 \ uhashring==1.2 \ --hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \ # via python-binary-memcached -https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \ - --hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \ - # via -r requirements/common.in urllib3==1.25.10 \ --hash=sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a \ --hash=sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461 \ diff --git a/stubs/orjson.pyi b/stubs/orjson.pyi new file mode 100644 index 0000000000..6abc1c78a2 --- /dev/null +++ b/stubs/orjson.pyi @@ -0,0 +1,27 @@ +# This can be deleted when orjson adds orjson.pyi and py.typed to the package, +# or mypy releases with https://github.com/python/typeshed/pull/4405 included. + +from typing import Any, Callable, Optional, Union + +__version__: str + +def dumps(__obj: Any, default: Optional[Callable[[Any], Any]] = ..., option: Optional[int] = ...,) -> bytes: ... +def loads(__obj: Union[bytes, bytearray, str]) -> Any: ... + +class JSONDecodeError(ValueError): ... +class JSONEncodeError(TypeError): ... + +OPT_APPEND_NEWLINE: int +OPT_INDENT_2: int +OPT_NAIVE_UTC: int +OPT_NON_STR_KEYS: int +OPT_OMIT_MICROSECONDS: int +OPT_PASSTHROUGH_DATACLASS: int +OPT_PASSTHROUGH_DATETIME: int +OPT_PASSTHROUGH_SUBCLASS: int +OPT_SERIALIZE_DATACLASS: int +OPT_SERIALIZE_NUMPY: int +OPT_SERIALIZE_UUID: int +OPT_SORT_KEYS: int +OPT_STRICT_INTEGER: int +OPT_UTC_Z: int diff --git a/tools/check-node-fixtures b/tools/check-node-fixtures index 9b999c835b..40f52645d8 100755 --- a/tools/check-node-fixtures +++ b/tools/check-node-fixtures @@ -5,7 +5,7 @@ import subprocess import sys from typing import Any, Callable, Dict, List, Optional -import ujson +import orjson TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) @@ -84,7 +84,7 @@ def read_fixtures() -> Dict[str, Any]: os.path.join(TOOLS_DIR, "node_lib/dump_fixtures.js"), ] schema = subprocess.check_output(cmd) - return ujson.loads(schema) + return orjson.loads(schema) def verify_fixtures_are_sorted(names: List[str]) -> None: diff --git a/tools/create-test-api-docs b/tools/create-test-api-docs index 7ba2beb421..5ee02cefd6 100755 --- a/tools/create-test-api-docs +++ b/tools/create-test-api-docs @@ -11,7 +11,7 @@ import pprint from collections import defaultdict from typing import Any, Dict, List, Set -import ujson +import orjson Call = Dict[str, Any] @@ -39,7 +39,7 @@ def encode_info(info: Any) -> str: try: result = '' try: - info = ujson.loads(info) + info = orjson.loads(info) result = '(stringified)\n' except Exception: pass @@ -106,9 +106,8 @@ def create_user_docs() -> None: ''') - calls = [] - for line in open(fn): - calls.append(ujson.loads(line)) + with open(fn, "rb") as coverage: + calls = [orjson.loads(line) for line in coverage] pattern_dict: Dict[str, List[Call]] = defaultdict(list) for call in calls: diff --git a/tools/generate-integration-docs-screenshot b/tools/generate-integration-docs-screenshot index 1c3e5d12e8..ee36884f7c 100755 --- a/tools/generate-integration-docs-screenshot +++ b/tools/generate-integration-docs-screenshot @@ -27,8 +27,8 @@ import subprocess from typing import Any, Dict, Optional from urllib.parse import parse_qsl, urlencode +import orjson import requests -import ujson from scripts.lib.zulip_tools import BOLDRED, ENDC from tools.lib.test_script import prepare_puppeteer_run @@ -106,7 +106,7 @@ def custom_headers(headers_json: str) -> Dict[str, str]: if not headers_json: return {} try: - return ujson.loads(headers_json) + return orjson.loads(headers_json) except ValueError as ve: raise argparse.ArgumentTypeError( 'Encountered an error while attempting to parse custom headers: {}\n' @@ -120,10 +120,11 @@ def send_bot_payload_message(bot: UserProfile, integration: WebhookIntegration, _, fixture_name = split_fixture_path(fixture_path) if fixture_name: - with open(fixture_path) as f: - if json_fixture: - data = ujson.load(f) - else: + if json_fixture: + with open(fixture_path, "rb") as fb: + data = orjson.loads(fb.read()) + else: + with open(fixture_path) as f: data = f.read().strip() else: data = '' @@ -150,7 +151,7 @@ def send_bot_payload_message(bot: UserProfile, integration: WebhookIntegration, params = parsed_params elif config.payload_as_query_param: - params[config.payload_param_name] = ujson.dumps(data) + params[config.payload_param_name] = orjson.dumps(data).decode() else: extra_args = {'json': data} diff --git a/tools/lib/sanity_check.py b/tools/lib/sanity_check.py index 6ebe777b90..ebb125e2e8 100644 --- a/tools/lib/sanity_check.py +++ b/tools/lib/sanity_check.py @@ -5,11 +5,14 @@ import sys def check_venv(filename: str) -> None: try: + # Here we import 3 modules that we expect to be in any valid + # Zulip virtualenv but are unlikely to all be present on a + # host system to help check whether we're in Vagrant. + import bitfield import django - import ujson import zulip + bitfield django - ujson zulip except ImportError: print(f"You need to run {filename} inside a Zulip dev environment.") diff --git a/tools/setup/emoji/build_emoji b/tools/setup/emoji/build_emoji index 2a5e50fa3d..83f32ca9e7 100755 --- a/tools/setup/emoji/build_emoji +++ b/tools/setup/emoji/build_emoji @@ -7,7 +7,7 @@ import shutil import sys from typing import Any, Dict, Iterator, List, Optional -import ujson +import orjson from emoji_names import EMOJI_NAME_MAPS from emoji_setup_utils import ( @@ -284,8 +284,8 @@ def setup_emoji_farms(cache_path: str, emoji_data: List[Dict[str, Any]]) -> None GOOGLE_BLOB_EMOJI_DATA_PATH = os.path.join(NODE_MODULES_PATH, 'emoji-datasource-google-blob', 'emoji.json') - with open(GOOGLE_BLOB_EMOJI_DATA_PATH) as fp: - blob_emoji_data = ujson.load(fp) + with open(GOOGLE_BLOB_EMOJI_DATA_PATH, "rb") as fp: + blob_emoji_data = orjson.loads(fp.read()) setup_emoji_farm('google-blob', blob_emoji_data, 'google') def setup_old_emoji_farm(cache_path: str, @@ -328,24 +328,24 @@ def generate_map_files(cache_path: str, emoji_catalog: Dict[str, List[str]]) -> name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS) EMOJI_CODES_FILE_PATH = os.path.join(cache_path, 'emoji_codes.json') - with open(EMOJI_CODES_FILE_PATH, 'w') as emoji_codes_file: - ujson.dump({ + with open(EMOJI_CODES_FILE_PATH, 'wb') as emoji_codes_file: + emoji_codes_file.write(orjson.dumps({ 'names': names, 'name_to_codepoint': name_to_codepoint, 'codepoint_to_name': codepoint_to_name, 'emoji_catalog': emoji_catalog, 'emoticon_conversions': EMOTICON_CONVERSIONS, - }, emoji_codes_file) + })) def dump_emojis(cache_path: str) -> None: - with open('emoji_map.json') as emoji_map_file: - emoji_map = ujson.load(emoji_map_file) + with open('emoji_map.json', "rb") as emoji_map_file: + emoji_map = orjson.loads(emoji_map_file.read()) # `emoji.json` or any other data file can be sourced from any of the supported # emojiset packages, they all contain the same data files. EMOJI_DATA_FILE_PATH = os.path.join(NODE_MODULES_PATH, 'emoji-datasource-google', 'emoji.json') - with open(EMOJI_DATA_FILE_PATH) as emoji_data_file: - emoji_data = ujson.load(emoji_data_file) + with open(EMOJI_DATA_FILE_PATH, "rb") as emoji_data_file: + emoji_data = orjson.loads(emoji_data_file.read()) emoji_catalog = generate_emoji_catalog(emoji_data, EMOJI_NAME_MAPS) # Setup emoji farms. diff --git a/tools/setup/emoji/export_emoji_names_to_csv b/tools/setup/emoji/export_emoji_names_to_csv index c1e44a7e96..11906d60b5 100755 --- a/tools/setup/emoji/export_emoji_names_to_csv +++ b/tools/setup/emoji/export_emoji_names_to_csv @@ -12,7 +12,7 @@ import os import re from typing import Any, Dict, List -import ujson +import orjson from emoji_setup_utils import get_emoji_code @@ -48,8 +48,8 @@ explanation_regex = re.compile(r" # (?P[^\r\n\t]+)") def prepare_sorting_info() -> None: emoji_data: List[Dict[str, Any]] = [] - with open(EMOJI_DATA_PATH) as fp: - emoji_data = ujson.load(fp) + with open(EMOJI_DATA_PATH, "rb") as fp: + emoji_data = orjson.loads(fp.read()) for emoji_dict in emoji_data: emoji_code = get_emoji_code(emoji_dict) diff --git a/tools/setup/emoji/generate_emoji_names_table b/tools/setup/emoji/generate_emoji_names_table index 63ed0445a5..fa9cdfb998 100755 --- a/tools/setup/emoji/generate_emoji_names_table +++ b/tools/setup/emoji/generate_emoji_names_table @@ -8,7 +8,7 @@ import os from typing import Any, Dict, List -import ujson +import orjson from emoji_names import EMOJI_NAME_MAPS from emoji_setup_utils import EMOJISETS, emoji_is_universal, get_emoji_code @@ -21,12 +21,12 @@ EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, 'node_modules', 'emoji-datasource-goo EMOJI_CACHE = os.path.join(ZULIP_PATH, 'static', 'generated', 'emoji') OUTPUT_FILE = os.path.join(EMOJI_CACHE, 'emoji_names_table.html') -with open(EMOJI_DATA_FILE) as fp: - EMOJI_DATA = ujson.load(fp) -with open(UNIFIED_REACTIONS_FILE) as fp: - UNIFIED_REACTIONS_MAP = ujson.load(fp) -with open(EMOJI_MAP_FILE) as fp: - EMOJI_MAP = ujson.load(fp) +with open(EMOJI_DATA_FILE, "rb") as fp: + EMOJI_DATA = orjson.loads(fp.read()) +with open(UNIFIED_REACTIONS_FILE, "rb") as fp: + UNIFIED_REACTIONS_MAP = orjson.loads(fp.read()) +with open(EMOJI_MAP_FILE, "rb") as fp: + EMOJI_MAP = orjson.loads(fp.read()) EMOJI_IMAGE_TEMPLATE = """ diff --git a/tools/test-backend b/tools/test-backend index 2577a25c26..dc7bc6fded 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -17,8 +17,8 @@ from typing import Iterator, List from unittest import mock import django +import orjson import responses -import ujson from django.conf import settings from django.test.utils import get_runner @@ -143,16 +143,16 @@ FAILED_TEST_PATH = 'var/last_test_failure.json' def get_failed_tests() -> List[str]: try: - with open(FAILED_TEST_PATH) as f: - return ujson.load(f) + with open(FAILED_TEST_PATH, "rb") as f: + return orjson.loads(f.read()) except OSError: print("var/last_test_failure.json doesn't exist; running all tests.") return [] def write_failed_tests(failed_tests: List[str]) -> None: if failed_tests: - with open(FAILED_TEST_PATH, 'w') as f: - ujson.dump(failed_tests, f) + with open(FAILED_TEST_PATH, 'wb') as f: + f.write(orjson.dumps(failed_tests)) @contextlib.contextmanager def block_internet() -> Iterator[None]: diff --git a/tools/test-js-with-node b/tools/test-js-with-node index 561f4d6ba4..02bf2221e1 100755 --- a/tools/test-js-with-node +++ b/tools/test-js-with-node @@ -17,7 +17,7 @@ from tools.lib import sanity_check sanity_check.check_venv(__file__) # Import this after we do the sanity_check so it doesn't crash. -import ujson +import orjson from zulint.printer import BOLDRED, CYAN, ENDC, GREEN INDEX_JS = 'frontend_tests/zjsunit/index.js' @@ -267,8 +267,8 @@ def check_line_coverage(fn: str, line_coverage: Dict[Any, Any], line_mapping: Di def read_coverage() -> Any: coverage_json = None try: - with open(NODE_COVERAGE_PATH) as f: - coverage_json = ujson.load(f) + with open(NODE_COVERAGE_PATH, "rb") as f: + coverage_json = orjson.loads(f.read()) except OSError: print(NODE_COVERAGE_PATH + " doesn't exist. Cannot enforce fully covered files.") raise diff --git a/tools/test-locked-requirements b/tools/test-locked-requirements index 152aab1bf3..1708c4dad6 100755 --- a/tools/test-locked-requirements +++ b/tools/test-locked-requirements @@ -10,7 +10,7 @@ import sys import tempfile from typing import List -import ujson +import orjson TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) ZULIP_PATH = os.path.dirname(TOOLS_DIR) @@ -64,12 +64,12 @@ def get_requirements_hash(tmp_dir: str, use_test_lock_files: bool = False) -> st def may_be_setup_cache() -> None: os.makedirs(CACHE_DIR, exist_ok=True) if not os.path.exists(CACHE_FILE): - with open(CACHE_FILE, 'w') as fp: - ujson.dump([], fp) + with open(CACHE_FILE, 'wb') as fp: + fp.write(orjson.dumps([])) def load_cache() -> List[str]: - with open(CACHE_FILE) as fp: - hash_list = ujson.load(fp) + with open(CACHE_FILE, "rb") as fp: + hash_list = orjson.loads(fp.read()) return hash_list def update_cache(hash_list: List[str]) -> None: @@ -77,8 +77,8 @@ def update_cache(hash_list: List[str]) -> None: # not a problem as it is cheap to do. if len(hash_list) > 100: hash_list = hash_list[-100:] - with open(CACHE_FILE, 'w') as fp: - ujson.dump(hash_list, fp) + with open(CACHE_FILE, 'wb') as fp: + fp.write(orjson.dumps(hash_list)) def main() -> None: may_be_setup_cache() diff --git a/version.py b/version.py index dd7912fae0..70fbc08032 100644 --- a/version.py +++ b/version.py @@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 27 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '96.0' +PROVISION_VERSION = '97.0' diff --git a/zerver/data_import/gitter.py b/zerver/data_import/gitter.py index 19387a4dd1..72cf1a19fe 100644 --- a/zerver/data_import/gitter.py +++ b/zerver/data_import/gitter.py @@ -4,7 +4,7 @@ import subprocess from typing import Any, Dict, List, Set, Tuple import dateutil.parser -import ujson +import orjson from django.conf import settings from django.forms.models import model_to_dict from django.utils.timezone import now as timezone_now @@ -278,8 +278,8 @@ def do_convert_data(gitter_data_file: str, output_dir: str, threads: int=6) -> N raise Exception("Output directory should be empty!") # Read data from the gitter file - with open(gitter_data_file) as fp: - gitter_data = ujson.load(fp) + with open(gitter_data_file, "rb") as fp: + gitter_data = orjson.loads(fp.read()) realm, avatar_list, user_map, stream_map = gitter_workspace_to_realm( domain_name, gitter_data, realm_subdomain) @@ -321,5 +321,5 @@ def do_convert_data(gitter_data_file: str, output_dir: str, threads: int=6) -> N logging.info("Zulip data dump created at %s", output_dir) def write_data_to_file(output_file: str, data: Any) -> None: - with open(output_file, "w") as f: - f.write(ujson.dumps(data, indent=4)) + with open(output_file, "wb") as f: + f.write(orjson.dumps(data, option=orjson.OPT_INDENT_2)) diff --git a/zerver/data_import/hipchat.py b/zerver/data_import/hipchat.py index 26a2c33eb9..9910270399 100755 --- a/zerver/data_import/hipchat.py +++ b/zerver/data_import/hipchat.py @@ -9,7 +9,7 @@ from typing import Any, Callable, Dict, List, Optional, Set import dateutil import hypchat -import ujson +import orjson from django.conf import settings from django.utils.timezone import now as timezone_now @@ -74,8 +74,8 @@ def untar_input_file(tar_file: str) -> str: def read_user_data(data_dir: str) -> List[ZerverFieldsT]: fn = 'users.json' data_file = os.path.join(data_dir, fn) - with open(data_file) as fp: - return ujson.load(fp) + with open(data_file, "rb") as fp: + return orjson.loads(fp.read()) def convert_user_data(user_handler: UserHandler, slim_mode: bool, @@ -192,8 +192,8 @@ def convert_avatar_data(avatar_folder: str, def read_room_data(data_dir: str) -> List[ZerverFieldsT]: fn = 'rooms.json' data_file = os.path.join(data_dir, fn) - with open(data_file) as f: - data = ujson.load(f) + with open(data_file, "rb") as f: + data = orjson.loads(f.read()) return data def convert_room_data(raw_data: List[ZerverFieldsT], @@ -349,8 +349,8 @@ def write_emoticon_data(realm_id: int, logging.warning("As a result, custom emoji cannot be imported.") return [] - with open(data_file) as f: - data = ujson.load(f) + with open(data_file, "rb") as f: + data = orjson.loads(f.read()) if isinstance(data, dict) and 'Emoticons' in data: # Handle the hc-migrate export format for emoticons.json. @@ -561,8 +561,8 @@ def process_message_file(realm_id: int, attachment_handler: AttachmentHandler) -> None: def get_raw_messages(fn: str) -> List[ZerverFieldsT]: - with open(fn) as f: - data = ujson.load(f) + with open(fn, "rb") as f: + data = orjson.loads(f.read()) flat_data = [ d[message_key] diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py index cc11319caa..f4e0b4e535 100644 --- a/zerver/data_import/import_util.py +++ b/zerver/data_import/import_util.py @@ -16,8 +16,8 @@ from typing import ( TypeVar, ) +import orjson import requests -import ujson from django.forms.models import model_to_dict from zerver.data_import.sequencer import NEXT_ID @@ -709,5 +709,5 @@ def process_emojis(zerver_realmemoji: List[ZerverFieldsT], emoji_dir: str, def create_converted_data_files(data: Any, output_dir: str, file_path: str) -> None: output_file = output_dir + file_path os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, 'w') as fp: - ujson.dump(data, fp, indent=4) + with open(output_file, 'wb') as fp: + fp.write(orjson.dumps(data, option=orjson.OPT_INDENT_2)) diff --git a/zerver/data_import/mattermost.py b/zerver/data_import/mattermost.py index 839edcee26..14ab4f88da 100644 --- a/zerver/data_import/mattermost.py +++ b/zerver/data_import/mattermost.py @@ -9,7 +9,7 @@ import shutil import subprocess from typing import Any, Callable, Dict, List, Set -import ujson +import orjson from django.conf import settings from django.forms.models import model_to_dict from django.utils.timezone import now as timezone_now @@ -690,9 +690,9 @@ def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, Any]: mattermost_data["emoji"] = [] mattermost_data["direct_channel"] = [] - with open(mattermost_data_file) as fp: + with open(mattermost_data_file, "rb") as fp: for line in fp: - row = ujson.loads(line.rstrip("\n")) + row = orjson.loads(line) data_type = row["type"] if data_type == "post": mattermost_data["post"]["channel_post"].append(row["post"]) diff --git a/zerver/data_import/slack.py b/zerver/data_import/slack.py index 56736e5507..d1dc63641d 100755 --- a/zerver/data_import/slack.py +++ b/zerver/data_import/slack.py @@ -7,8 +7,8 @@ from collections import defaultdict from typing import Any, Dict, Iterator, List, Optional, Set, Tuple from urllib.parse import urlencode +import orjson import requests -import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.timezone import now as timezone_now @@ -728,7 +728,7 @@ def channel_message_to_zerver_message(realm_id: int, message['text'], users, added_channels, slack_user_id_to_zulip_user_id) except Exception: print("Slack message unexpectedly missing text representation:") - print(ujson.dumps(message, indent=4)) + print(orjson.dumps(message, option=orjson.OPT_INDENT_2).decode()) continue rendered_content = None @@ -1120,8 +1120,8 @@ def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: i logging.info("Zulip data dump created at %s", output_dir) def get_data_file(path: str) -> Any: - with open(path) as fp: - data = ujson.load(fp) + with open(path, "rb") as fp: + data = orjson.loads(fp.read()) return data def log_token_warning(token: str) -> None: diff --git a/zerver/decorator.py b/zerver/decorator.py index 713517651d..68cfa40df4 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -7,7 +7,7 @@ from io import BytesIO from typing import Callable, Dict, Optional, Tuple, TypeVar, Union, cast import django_otp -import ujson +import orjson from django.conf import settings from django.contrib.auth import REDIRECT_FIELD_NAME from django.contrib.auth import login as django_login @@ -285,7 +285,7 @@ def log_exception_to_webhook_logger( if request.content_type == 'application/json': try: - payload = ujson.dumps(ujson.loads(payload), indent=4) + payload = orjson.dumps(orjson.loads(payload), option=orjson.OPT_INDENT_2).decode() except ValueError: request_body = str(payload) else: diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 0a3411ceeb..e5b971ea09 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -22,7 +22,7 @@ from typing import ( ) import django.db.utils -import ujson +import orjson from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError @@ -576,9 +576,9 @@ def do_create_user(email: str, password: Optional[str], realm: Realm, full_name: RealmAuditLog.objects.create( realm=user_profile.realm, acting_user=acting_user, modified_user=user_profile, event_type=RealmAuditLog.USER_CREATED, event_time=event_time, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm), - })) + }).decode()) do_increment_logging_stat(user_profile.realm, COUNT_STATS['active_users_log:is_bot:day'], user_profile.is_bot, event_time) if settings.BILLING_ENABLED: @@ -607,9 +607,9 @@ def do_activate_user(user_profile: UserProfile, acting_user: Optional[UserProfil RealmAuditLog.objects.create( realm=user_profile.realm, modified_user=user_profile, acting_user=acting_user, event_type=RealmAuditLog.USER_ACTIVATED, event_time=event_time, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm), - })) + }).decode()) do_increment_logging_stat(user_profile.realm, COUNT_STATS['active_users_log:is_bot:day'], user_profile.is_bot, event_time) if settings.BILLING_ENABLED: @@ -627,9 +627,9 @@ def do_reactivate_user(user_profile: UserProfile, acting_user: Optional[UserProf RealmAuditLog.objects.create( realm=user_profile.realm, modified_user=user_profile, acting_user=acting_user, event_type=RealmAuditLog.USER_REACTIVATED, event_time=event_time, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm), - })) + }).decode()) do_increment_logging_stat(user_profile.realm, COUNT_STATS['active_users_log:is_bot:day'], user_profile.is_bot, event_time) if settings.BILLING_ENABLED: @@ -668,11 +668,11 @@ def do_set_realm_property(realm: Realm, name: str, value: Any, event_time = timezone_now() RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time=event_time, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: value, 'property': name, - })) + }).decode()) if name == "email_address_visibility": if Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE not in [old_value, value]: @@ -703,11 +703,11 @@ def do_set_realm_authentication_methods(realm: Realm, updated_value = realm.authentication_methods_dict() RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time=timezone_now(), - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: updated_value, 'property': 'authentication_methods', - })) + }).decode()) event = dict( type="realm", op="update_dict", @@ -739,11 +739,11 @@ def do_set_realm_message_editing(realm: Realm, continue RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time=event_time, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_values[updated_property], RealmAuditLog.NEW_VALUE: updated_value, 'property': updated_property, - })) + }).decode()) realm.save(update_fields=list(updated_properties.keys())) event = dict( @@ -763,11 +763,11 @@ def do_set_realm_notifications_stream(realm: Realm, stream: Optional[Stream], st event_time = timezone_now() RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time=event_time, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: stream_id, 'property': 'notifications_stream', - })) + }).decode()) event = dict( type="realm", @@ -786,11 +786,11 @@ def do_set_realm_signup_notifications_stream(realm: Realm, stream: Optional[Stre event_time = timezone_now() RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time=event_time, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: stream_id, 'property': 'signup_notifications_stream', - })) + }).decode()) event = dict( type="realm", op="update", @@ -818,9 +818,9 @@ def do_deactivate_realm(realm: Realm, acting_user: Optional[UserProfile]=None) - event_time = timezone_now() RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_DEACTIVATED, event_time=event_time, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm), - })) + }).decode()) ScheduledEmail.objects.filter(realm=realm).delete() for user in active_humans_in_realm(realm): @@ -840,9 +840,9 @@ def do_reactivate_realm(realm: Realm) -> None: event_time = timezone_now() RealmAuditLog.objects.create( realm=realm, event_type=RealmAuditLog.REALM_REACTIVATED, event_time=event_time, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm), - })) + }).decode()) def do_change_realm_subdomain(realm: Realm, new_subdomain: str) -> None: realm.string_id = new_subdomain @@ -890,9 +890,9 @@ def do_deactivate_user(user_profile: UserProfile, RealmAuditLog.objects.create( realm=user_profile.realm, modified_user=user_profile, acting_user=acting_user, event_type=RealmAuditLog.USER_DEACTIVATED, event_time=event_time, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm), - })) + }).decode()) do_increment_logging_stat(user_profile.realm, COUNT_STATS['active_users_log:is_bot:day'], user_profile.is_bot, event_time, increment=-1) if settings.BILLING_ENABLED: @@ -2026,7 +2026,7 @@ def extract_stream_indicator(s: str) -> Union[str, int]: # it for legacy reasons. try: - data = ujson.loads(s) + data = orjson.loads(s) except (ValueError, TypeError): # If there was no JSON encoding, then we just # have a raw stream name. @@ -2054,7 +2054,7 @@ def extract_private_recipients(s: str) -> Union[List[str], List[int]]: # See test_extract_recipients() for examples of what we allow. try: - data = ujson.loads(s) + data = orjson.loads(s) except (ValueError, TypeError): data = s @@ -2374,7 +2374,7 @@ def check_message(sender: UserProfile, client: Client, addressee: Addressee, if widget_content is not None: try: - widget_content = ujson.loads(widget_content) + widget_content = orjson.loads(widget_content) except Exception: raise JsonableError(_('Widgets: API programmer sent invalid JSON content')) @@ -3143,11 +3143,11 @@ def do_change_subscription_property(user_profile: UserProfile, sub: Subscription RealmAuditLog.objects.create( realm=user_profile.realm, event_type=RealmAuditLog.SUBSCRIPTION_PROPERTY_CHANGED, event_time=event_time, modified_user=user_profile, acting_user=acting_user, - modified_stream=stream, extra_data=ujson.dumps({ + modified_stream=stream, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: database_value, 'property': database_property_name, - })) + }).decode()) event = dict(type="subscription", op="update", @@ -3437,10 +3437,10 @@ def do_change_default_sending_stream(user_profile: UserProfile, stream: Optional RealmAuditLog.objects.create( realm=user_profile.realm, event_type=RealmAuditLog.USER_DEFAULT_SENDING_STREAM_CHANGED, event_time=event_time, modified_user=user_profile, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: None if stream is None else stream.id, - })) + }).decode()) if user_profile.is_bot: if stream: @@ -3466,10 +3466,10 @@ def do_change_default_events_register_stream(user_profile: UserProfile, RealmAuditLog.objects.create( realm=user_profile.realm, event_type=RealmAuditLog.USER_DEFAULT_REGISTER_STREAM_CHANGED, event_time=event_time, modified_user=user_profile, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: None if stream is None else stream.id, - })) + }).decode()) if user_profile.is_bot: if stream: @@ -3494,10 +3494,10 @@ def do_change_default_all_public_streams(user_profile: UserProfile, value: bool, RealmAuditLog.objects.create( realm=user_profile.realm, event_type=RealmAuditLog.USER_DEFAULT_ALL_PUBLIC_STREAMS_CHANGED, event_time=event_time, modified_user=user_profile, - acting_user=acting_user, extra_data=ujson.dumps({ + acting_user=acting_user, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: value, - })) + }).decode()) if user_profile.is_bot: send_event(user_profile.realm, @@ -3515,11 +3515,11 @@ def do_change_user_role(user_profile: UserProfile, value: int, acting_user: Opti RealmAuditLog.objects.create( realm=user_profile.realm, modified_user=user_profile, acting_user=acting_user, event_type=RealmAuditLog.USER_ROLE_CHANGED, event_time=timezone_now(), - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: value, RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm), - })) + }).decode()) event = dict(type="realm_user", op="update", person=dict(user_id=user_profile.id, role=user_profile.role)) send_event(user_profile.realm, event, active_user_ids(user_profile.realm_id)) @@ -3590,10 +3590,10 @@ def do_rename_stream(stream: Stream, RealmAuditLog.objects.create( realm=stream.realm, acting_user=user_profile, modified_stream=stream, event_type=RealmAuditLog.STREAM_NAME_CHANGED, event_time=timezone_now(), - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_name, RealmAuditLog.NEW_VALUE: new_name, - })) + }).decode()) recipient_id = stream.recipient_id messages = Message.objects.filter(recipient_id=recipient_id).only("id") @@ -3766,11 +3766,11 @@ def do_change_notification_settings(user_profile: UserProfile, name: str, event_time = timezone_now() RealmAuditLog.objects.create( realm=user_profile.realm, event_type=RealmAuditLog.USER_NOTIFICATION_SETTINGS_CHANGED, event_time=event_time, - acting_user=acting_user, modified_user=user_profile, extra_data=ujson.dumps({ + acting_user=acting_user, modified_user=user_profile, extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: value, 'property': name, - })) + }).decode()) send_event(user_profile.realm, event, [user_profile.id]) @@ -4653,11 +4653,11 @@ def do_update_message(user_profile: UserProfile, message: Message, send_event(user_profile.realm, delete_event, delete_event_notify_user_ids) if message.edit_history is not None: - edit_history = ujson.loads(message.edit_history) + edit_history = orjson.loads(message.edit_history) edit_history.insert(0, edit_history_event) else: edit_history = [edit_history_event] - message.edit_history = ujson.dumps(edit_history) + message.edit_history = orjson.dumps(edit_history).decode() # This does message.save(update_fields=[...]) save_message_for_edit_use_case(message=message) @@ -5667,7 +5667,7 @@ def try_add_realm_default_custom_profile_field(realm: Realm, field = CustomProfileField(realm=realm, name=field_data['name'], field_type=CustomProfileField.EXTERNAL_ACCOUNT, hint=field_data['hint'], - field_data=ujson.dumps(dict(subtype=field_subtype))) + field_data=orjson.dumps(dict(subtype=field_subtype)).decode()) field.save() field.order = field.id field.save(update_fields=['order']) @@ -5681,7 +5681,7 @@ def try_add_realm_custom_profile_field(realm: Realm, name: str, field_type: int, field.hint = hint if (field.field_type == CustomProfileField.CHOICE or field.field_type == CustomProfileField.EXTERNAL_ACCOUNT): - field.field_data = ujson.dumps(field_data or {}) + field.field_data = orjson.dumps(field_data or {}).decode() field.save() field.order = field.id @@ -5707,7 +5707,7 @@ def try_update_realm_custom_profile_field(realm: Realm, field: CustomProfileFiel field.hint = hint if (field.field_type == CustomProfileField.CHOICE or field.field_type == CustomProfileField.EXTERNAL_ACCOUNT): - field.field_data = ujson.dumps(field_data or {}) + field.field_data = orjson.dumps(field_data or {}).decode() field.save() notify_realm_custom_profile_fields(realm, 'update') @@ -5726,7 +5726,7 @@ def notify_user_update_custom_profile_data(user_profile: UserProfile, field: Dict[str, Union[int, str, List[int], None]]) -> None: data = dict(id=field['id']) if field['type'] == CustomProfileField.USER: - data["value"] = ujson.dumps(field['value']) + data["value"] = orjson.dumps(field['value']).decode() else: data['value'] = field['value'] if field['rendered_value']: @@ -5988,11 +5988,11 @@ def notify_realm_export(user_profile: UserProfile) -> None: send_event(user_profile.realm, event, [user_profile.id]) def do_delete_realm_export(user_profile: UserProfile, export: RealmAuditLog) -> None: - # Give mypy a hint so it knows `ujson.loads` + # Give mypy a hint so it knows `orjson.loads` # isn't being passed an `Optional[str]`. export_extra_data = export.extra_data assert export_extra_data is not None - export_data = ujson.loads(export_extra_data) + export_data = orjson.loads(export_extra_data) export_path = export_data.get('export_path') if export_path: @@ -6000,7 +6000,7 @@ def do_delete_realm_export(user_profile: UserProfile, export: RealmAuditLog) -> delete_export_tarball(export_path) export_data.update({'deleted_timestamp': timezone_now().timestamp()}) - export.extra_data = ujson.dumps(export_data) + export.extra_data = orjson.dumps(export_data).decode() export.save(update_fields=['extra_data']) notify_realm_export(user_profile) diff --git a/zerver/lib/create_user.py b/zerver/lib/create_user.py index 93e2644ce3..85f2221f72 100644 --- a/zerver/lib/create_user.py +++ b/zerver/lib/create_user.py @@ -1,6 +1,6 @@ from typing import Optional -import ujson +import orjson from django.contrib.auth.models import UserManager from django.utils.timezone import now as timezone_now @@ -84,7 +84,7 @@ def create_user_profile(realm: Realm, email: str, password: Optional[str], tos_version=tos_version, timezone=timezone, tutorial_status=tutorial_status, enter_sends=enter_sends, - onboarding_steps=ujson.dumps([]), + onboarding_steps=orjson.dumps([]).decode(), default_language=realm.default_language, twenty_four_hour_time=realm.default_twenty_four_hour_time, delivery_email=email) diff --git a/zerver/lib/emoji.py b/zerver/lib/emoji.py index 30a87049dd..c302f83f9d 100644 --- a/zerver/lib/emoji.py +++ b/zerver/lib/emoji.py @@ -2,7 +2,7 @@ import os import re from typing import Optional, Tuple -import ujson +import orjson from django.utils.translation import ugettext as _ from zerver.lib.exceptions import OrganizationAdministratorRequired @@ -20,8 +20,8 @@ if not os.path.exists(emoji_codes_path): # nocoverage "../../static/generated/emoji/emoji_codes.json", ) -with open(emoji_codes_path) as fp: - emoji_codes = ujson.load(fp) +with open(emoji_codes_path, "rb") as fp: + emoji_codes = orjson.loads(fp.read()) name_to_codepoint = emoji_codes["name_to_codepoint"] codepoint_to_name = emoji_codes["codepoint_to_name"] diff --git a/zerver/lib/export.py b/zerver/lib/export.py index 512492f28f..fa2b08efd7 100644 --- a/zerver/lib/export.py +++ b/zerver/lib/export.py @@ -16,7 +16,7 @@ import tempfile from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import boto3 -import ujson +import orjson from boto3.resources.base import ServiceResource from django.apps import apps from django.conf import settings @@ -344,8 +344,13 @@ def sanity_check_output(data: TableData) -> None: logging.warning('??? NO DATA EXPORTED FOR TABLE %s!!!', table) def write_data_to_file(output_file: Path, data: Any) -> None: - with open(output_file, "w") as f: - f.write(ujson.dumps(data, indent=4)) + with open(output_file, "wb") as f: + # Because we don't pass a default handler, OPT_PASSTHROUGH_DATETIME + # actually causes orjson to raise a TypeError on datetime objects. This + # is what we want, because it helps us check that we correctly + # post-processed them to serialize to UNIX timestamps rather than ISO + # 8601 strings for historical reasons. + f.write(orjson.dumps(data, option=orjson.OPT_INDENT_2 | orjson.OPT_PASSTHROUGH_DATETIME)) def make_raw(query: Any, exclude: Optional[List[Field]]=None) -> List[Record]: ''' @@ -972,8 +977,8 @@ def export_usermessages_batch(input_path: Path, output_path: Path, batch of Message objects and adds the corresponding UserMessage objects. (This is called by the export_usermessage_batch management command).""" - with open(input_path) as input_file: - output = ujson.load(input_file) + with open(input_path, "rb") as input_file: + output = orjson.loads(input_file.read()) message_ids = [item['id'] for item in output['zerver_message']] user_profile_ids = set(output['zerver_userprofile_ids']) del output['zerver_userprofile_ids'] @@ -1317,8 +1322,8 @@ def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path, if (count % 100 == 0): logging.info("Finished %s", count) - with open(os.path.join(output_dir, "records.json"), "w") as records_file: - ujson.dump(records, records_file, indent=4) + with open(os.path.join(output_dir, "records.json"), "wb") as records_file: + records_file.write(orjson.dumps(records, option=orjson.OPT_INDENT_2)) def export_uploads_from_local(realm: Realm, local_dir: Path, output_dir: Path) -> None: @@ -1350,8 +1355,8 @@ def export_uploads_from_local(realm: Realm, local_dir: Path, output_dir: Path) - if (count % 100 == 0): logging.info("Finished %s", count) - with open(os.path.join(output_dir, "records.json"), "w") as records_file: - ujson.dump(records, records_file, indent=4) + with open(os.path.join(output_dir, "records.json"), "wb") as records_file: + records_file.write(orjson.dumps(records, option=orjson.OPT_INDENT_2)) def export_avatars_from_local(realm: Realm, local_dir: Path, output_dir: Path) -> None: @@ -1396,8 +1401,8 @@ def export_avatars_from_local(realm: Realm, local_dir: Path, output_dir: Path) - if (count % 100 == 0): logging.info("Finished %s", count) - with open(os.path.join(output_dir, "records.json"), "w") as records_file: - ujson.dump(records, records_file, indent=4) + with open(os.path.join(output_dir, "records.json"), "wb") as records_file: + records_file.write(orjson.dumps(records, option=orjson.OPT_INDENT_2)) def export_realm_icons(realm: Realm, local_dir: Path, output_dir: Path) -> None: records = [] @@ -1414,8 +1419,8 @@ def export_realm_icons(realm: Realm, local_dir: Path, output_dir: Path) -> None: s3_path=icon_relative_path) records.append(record) - with open(os.path.join(output_dir, "records.json"), "w") as records_file: - ujson.dump(records, records_file, indent=4) + with open(os.path.join(output_dir, "records.json"), "wb") as records_file: + records_file.write(orjson.dumps(records, option=orjson.OPT_INDENT_2)) def export_emoji_from_local(realm: Realm, local_dir: Path, output_dir: Path) -> None: @@ -1454,8 +1459,8 @@ def export_emoji_from_local(realm: Realm, local_dir: Path, output_dir: Path) -> count += 1 if (count % 100 == 0): logging.info("Finished %s", count) - with open(os.path.join(output_dir, "records.json"), "w") as records_file: - ujson.dump(records, records_file, indent=4) + with open(os.path.join(output_dir, "records.json"), "wb") as records_file: + records_file.write(orjson.dumps(records, option=orjson.OPT_INDENT_2)) def do_write_stats_file_for_realm_export(output_dir: Path) -> None: stats_file = os.path.join(output_dir, 'stats.txt') @@ -1469,8 +1474,8 @@ def do_write_stats_file_for_realm_export(output_dir: Path) -> None: with open(stats_file, 'w') as f: for fn in fns: f.write(os.path.basename(fn) + '\n') - with open(fn) as filename: - data = ujson.load(filename) + with open(fn, "rb") as filename: + data = orjson.loads(filename.read()) for k in sorted(data): f.write(f'{len(data[k]):5} {k}\n') f.write('\n') @@ -1480,8 +1485,8 @@ def do_write_stats_file_for_realm_export(output_dir: Path) -> None: for fn in [avatar_file, uploads_file]: f.write(fn+'\n') - with open(fn) as filename: - data = ujson.load(filename) + with open(fn, "rb") as filename: + data = orjson.loads(filename.read()) f.write(f'{len(data):5} records\n') f.write('\n') @@ -1810,7 +1815,7 @@ def get_realm_exports_serialized(user: UserProfile) -> List[Dict[str, Any]]: if export.extra_data is not None: pending = False - export_data = ujson.loads(export.extra_data) + export_data = orjson.loads(export.extra_data) deleted_timestamp = export_data.get('deleted_timestamp') failed_timestamp = export_data.get('failed_timestamp') export_path = export_data.get('export_path') diff --git a/zerver/lib/generate_test_data.py b/zerver/lib/generate_test_data.py index 450bc36c06..bfe197deb8 100644 --- a/zerver/lib/generate_test_data.py +++ b/zerver/lib/generate_test_data.py @@ -3,14 +3,14 @@ import os import random from typing import Any, Dict, List -import ujson +import orjson from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path def load_config() -> Dict[str, Any]: - with open("zerver/tests/fixtures/config.generate_data.json") as infile: - config = ujson.load(infile) + with open("zerver/tests/fixtures/config.generate_data.json", "rb") as infile: + config = orjson.loads(infile.read()) return config @@ -182,8 +182,8 @@ def remove_line_breaks(fh: Any) -> List[str]: def write_file(paragraphs: List[str], filename: str) -> None: - with open(filename, "w") as outfile: - outfile.write(ujson.dumps(paragraphs)) + with open(filename, "wb") as outfile: + outfile.write(orjson.dumps(paragraphs)) def create_test_data() -> None: diff --git a/zerver/lib/i18n.py b/zerver/lib/i18n.py index 8caf89ad9c..66c4bae272 100644 --- a/zerver/lib/i18n.py +++ b/zerver/lib/i18n.py @@ -4,7 +4,7 @@ import os from itertools import zip_longest from typing import Any, Dict, List -import ujson +import orjson from django.conf import settings from django.utils import translation from django.utils.lru_cache import lru_cache @@ -25,8 +25,8 @@ def with_language(string: str, language: str) -> str: @lru_cache() def get_language_list() -> List[Dict[str, Any]]: path = os.path.join(settings.DEPLOY_ROOT, 'locale', 'language_name_map.json') - with open(path) as reader: - languages = ujson.load(reader) + with open(path, "rb") as reader: + languages = orjson.loads(reader.read()) return languages['name_map'] def get_language_list_for_templates(default_language: str) -> List[Dict[str, Dict[str, str]]]: @@ -88,8 +88,8 @@ def get_language_translation_data(language: str) -> Dict[str, str]: language = 'id_ID' path = os.path.join(settings.DEPLOY_ROOT, 'locale', language, 'translations.json') try: - with open(path) as reader: - return ujson.load(reader) + with open(path, "rb") as reader: + return orjson.loads(reader.read()) except FileNotFoundError: print(f'Translation for {language} not found at {path}') return {} diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index 85395185bb..a0a364a47c 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -5,7 +5,7 @@ import shutil from typing import Any, Dict, Iterable, List, Optional, Tuple import boto3 -import ujson +import orjson from bs4 import BeautifulSoup from django.conf import settings from django.db import connection @@ -240,14 +240,14 @@ def fix_customprofilefield(data: TableData) -> None: for item in data['zerver_customprofilefieldvalue']: if item['field_id'] in field_type_USER_id_list: - old_user_id_list = ujson.loads(item['value']) + old_user_id_list = orjson.loads(item['value']) new_id_list = re_map_foreign_keys_many_to_many_internal( table='zerver_customprofilefieldvalue', field_name='value', related_table='user_profile', old_id_list=old_user_id_list) - item['value'] = ujson.dumps(new_id_list) + item['value'] = orjson.dumps(new_id_list).decode() def fix_message_rendered_content(realm: Realm, sender_map: Dict[int, Record], @@ -614,8 +614,8 @@ def import_uploads(realm: Realm, import_dir: Path, processes: int, processing_av logging.info("Importing uploaded files") records_filename = os.path.join(import_dir, "records.json") - with open(records_filename) as records_file: - records: List[Dict[str, Any]] = ujson.load(records_file) + with open(records_filename, "rb") as records_file: + records: List[Dict[str, Any]] = orjson.loads(records_file.read()) timestamp = datetime_to_timestamp(timezone_now()) re_map_foreign_keys_internal(records, 'records', 'realm_id', related_table="realm", @@ -795,8 +795,8 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int=1) -> Realm create_internal_realm() logging.info("Importing realm data from %s", realm_data_filename) - with open(realm_data_filename) as f: - data = ujson.load(f) + with open(realm_data_filename, "rb") as f: + data = orjson.loads(f.read()) remove_denormalized_recipient_column_from_data(data) sort_by_date = data.get('sort_by_date', False) @@ -1073,8 +1073,8 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int=1) -> Realm raise Exception("Missing attachment.json file!") logging.info("Importing attachment data from %s", fn) - with open(fn) as f: - data = ujson.load(f) + with open(fn, "rb") as f: + data = orjson.loads(f.read()) import_attachments(data) @@ -1149,8 +1149,8 @@ def get_incoming_message_ids(import_dir: Path, if not os.path.exists(message_filename): break - with open(message_filename) as f: - data = ujson.load(f) + with open(message_filename, "rb") as f: + data = orjson.loads(f.read()) # Aggressively free up memory. del data['zerver_usermessage'] @@ -1192,8 +1192,8 @@ def import_message_data(realm: Realm, if not os.path.exists(message_filename): break - with open(message_filename) as f: - data = ujson.load(f) + with open(message_filename, "rb") as f: + data = orjson.loads(f.read()) logging.info("Importing message dump %s", message_filename) re_map_foreign_keys(data, 'zerver_message', 'sender', related_table="user_profile") @@ -1311,8 +1311,8 @@ def import_analytics_data(realm: Realm, import_dir: Path) -> None: return logging.info("Importing analytics data from %s", analytics_filename) - with open(analytics_filename) as f: - data = ujson.load(f) + with open(analytics_filename, "rb") as f: + data = orjson.loads(f.read()) # Process the data through the fixer functions. fix_datetime_fields(data, 'analytics_realmcount') diff --git a/zerver/lib/markdown/testing_mocks.py b/zerver/lib/markdown/testing_mocks.py index 04677c36d6..aa87ce8fd7 100644 --- a/zerver/lib/markdown/testing_mocks.py +++ b/zerver/lib/markdown/testing_mocks.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Optional -import ujson +import orjson NORMAL_TWEET = """{ "created_at": "Sat Sep 10 22:23:38 +0000 2011", @@ -221,12 +221,12 @@ EMOJI_TWEET = """{ def twitter(tweet_id: str) -> Optional[Dict[str, Any]]: if tweet_id in ["112652479837110273", "287977969287315456", "287977969287315457"]: - return ujson.loads(NORMAL_TWEET) + return orjson.loads(NORMAL_TWEET) elif tweet_id == "287977969287315458": - return ujson.loads(MENTION_IN_LINK_TWEET) + return orjson.loads(MENTION_IN_LINK_TWEET) elif tweet_id == "287977969287315459": - return ujson.loads(MEDIA_TWEET) + return orjson.loads(MEDIA_TWEET) elif tweet_id == "287977969287315460": - return ujson.loads(EMOJI_TWEET) + return orjson.loads(EMOJI_TWEET) else: return None diff --git a/zerver/lib/message.py b/zerver/lib/message.py index 5358d6bd46..3b5eb1ad3f 100644 --- a/zerver/lib/message.py +++ b/zerver/lib/message.py @@ -4,7 +4,7 @@ import zlib from typing import Any, Dict, List, Optional, Sequence, Set, Tuple import ahocorasick -import ujson +import orjson from django.db import connection from django.db.models import Sum from django.utils.timezone import now as timezone_now @@ -154,10 +154,10 @@ def sew_messages_and_submessages(messages: List[Dict[str, Any]], message['submessages'].append(submessage) def extract_message_dict(message_bytes: bytes) -> Dict[str, Any]: - return ujson.loads(zlib.decompress(message_bytes).decode("utf-8")) + return orjson.loads(zlib.decompress(message_bytes)) def stringify_message_dict(message_dict: Dict[str, Any]) -> bytes: - return zlib.compress(ujson.dumps(message_dict).encode()) + return zlib.compress(orjson.dumps(message_dict)) @cache_with_key(to_dict_cache_key, timeout=3600*24) def message_to_dict_json(message: Message, realm_id: Optional[int]=None) -> bytes: @@ -395,7 +395,7 @@ class MessageDict: if last_edit_time is not None: obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time) assert edit_history is not None - obj['edit_history'] = ujson.loads(edit_history) + obj['edit_history'] = orjson.loads(edit_history) if Message.need_to_render_content(rendered_content, rendered_content_version, markdown_version): # We really shouldn't be rendering objects in this method, but there is diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index 43313457f9..d08b846c06 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Un import gcm import lxml.html -import ujson +import orjson from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import F @@ -249,7 +249,7 @@ def parse_gcm_options(options: Dict[str, Any], data: Dict[str, Any]) -> str: # one-way compatibility. raise JsonableError(_( "Invalid GCM options to bouncer: {}", - ).format(ujson.dumps(options))) + ).format(orjson.dumps(options).decode())) return priority # when this grows a second option, can make it a tuple diff --git a/zerver/lib/queue.py b/zerver/lib/queue.py index 69d2a6ce2c..1e5b47ae38 100644 --- a/zerver/lib/queue.py +++ b/zerver/lib/queue.py @@ -5,9 +5,9 @@ import time from collections import defaultdict from typing import Any, Callable, Dict, List, Mapping, Optional, Set +import orjson import pika import pika.adapters.tornado_connection -import ujson from django.conf import settings from pika.adapters.blocking_connection import BlockingChannel from pika.spec import Basic @@ -126,7 +126,7 @@ class SimpleQueueClient: self.ensure_queue(queue_name, do_publish) def json_publish(self, queue_name: str, body: Mapping[str, Any]) -> None: - data = ujson.dumps(body).encode() + data = orjson.dumps(body) try: self.publish(queue_name, data) return @@ -164,7 +164,7 @@ class SimpleQueueClient: method: Basic.Deliver, properties: pika.BasicProperties, body: bytes) -> None: - callback(ujson.loads(body)) + callback(orjson.loads(body)) self.register_consumer(queue_name, wrapped_callback) def drain_queue(self, queue_name: str) -> List[bytes]: @@ -185,7 +185,7 @@ class SimpleQueueClient: return messages def json_drain_queue(self, queue_name: str) -> List[Dict[str, Any]]: - return list(map(ujson.loads, self.drain_queue(queue_name))) + return list(map(orjson.loads, self.drain_queue(queue_name))) def queue_size(self) -> int: assert self.channel is not None diff --git a/zerver/lib/redis_utils.py b/zerver/lib/redis_utils.py index 7636b4e9c6..0c2825385a 100644 --- a/zerver/lib/redis_utils.py +++ b/zerver/lib/redis_utils.py @@ -1,8 +1,8 @@ import re from typing import Any, Dict, Mapping, Optional +import orjson import redis -import ujson from django.conf import settings from zerver.lib.utils import generate_random_token @@ -37,7 +37,7 @@ def put_dict_in_redis(redis_client: redis.StrictRedis, key_format: str, token = generate_random_token(token_length) key = key_format.format(token=token) with redis_client.pipeline() as pipeline: - pipeline.set(key, ujson.dumps(data_to_store)) + pipeline.set(key, orjson.dumps(data_to_store)) pipeline.expire(key, expiration_seconds) pipeline.execute() @@ -57,7 +57,7 @@ def get_dict_from_redis(redis_client: redis.StrictRedis, key_format: str, key: s data = redis_client.get(key) if data is None: return None - return ujson.loads(data) + return orjson.loads(data) def validate_key_fits_format(key: str, key_format: str) -> None: assert "{token}" in key_format diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index ccf93005e2..ced2bd4446 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -2,8 +2,8 @@ import logging import urllib from typing import Any, Dict, List, Mapping, Tuple, Union +import orjson import requests -import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ @@ -24,7 +24,7 @@ class PushNotificationBouncerRetryLaterError(JsonableError): def send_to_push_bouncer( method: str, endpoint: str, - post_data: Union[str, Mapping[str, Union[str, bytes]]], + post_data: Union[bytes, Mapping[str, Union[str, bytes]]], extra_headers: Mapping[str, str] = {}, ) -> Dict[str, object]: """While it does actually send the notice, this function has a lot of @@ -74,7 +74,7 @@ def send_to_push_bouncer( raise PushNotificationBouncerRetryLaterError(error_msg) elif res.status_code >= 400: # If JSON parsing errors, just let that exception happen - result_dict = ujson.loads(res.content) + result_dict = orjson.loads(res.content) msg = result_dict['msg'] if 'code' in result_dict and result_dict['code'] == 'INVALID_ZULIP_SERVER': # Invalid Zulip server credentials should email this server's admins @@ -93,13 +93,13 @@ def send_to_push_bouncer( f"Push notification bouncer returned unexpected status code {res.status_code}") # If we don't throw an exception, it's a successful bounce! - return ujson.loads(res.content) + return orjson.loads(res.content) def send_json_to_push_bouncer(method: str, endpoint: str, post_data: Mapping[str, object]) -> None: send_to_push_bouncer( method, endpoint, - ujson.dumps(post_data), + orjson.dumps(post_data), extra_headers={"Content-type": "application/json"}, ) @@ -157,10 +157,10 @@ def send_analytics_to_remote_server() -> None: return request = { - 'realm_counts': ujson.dumps(realm_count_data), - 'installation_counts': ujson.dumps(installation_count_data), - 'realmauditlog_rows': ujson.dumps(realmauditlog_data), - 'version': ujson.dumps(ZULIP_VERSION), + 'realm_counts': orjson.dumps(realm_count_data).decode(), + 'installation_counts': orjson.dumps(installation_count_data).decode(), + 'realmauditlog_rows': orjson.dumps(realmauditlog_data).decode(), + 'version': orjson.dumps(ZULIP_VERSION).decode(), } # Gather only entries with an ID greater than last_realm_count_id diff --git a/zerver/lib/request.py b/zerver/lib/request.py index ebb2cdfc8d..b08623442d 100644 --- a/zerver/lib/request.py +++ b/zerver/lib/request.py @@ -15,7 +15,7 @@ from typing import ( overload, ) -import ujson +import orjson from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -296,7 +296,7 @@ def has_request_variables(view_func: ViewFuncT) -> ViewFuncT: if param.argument_type == 'body': try: - val = ujson.loads(request.body) + val = orjson.loads(request.body) except ValueError: raise InvalidJSONError(_("Malformed JSON")) kwargs[func_var_name] = val @@ -345,7 +345,7 @@ def has_request_variables(view_func: ViewFuncT) -> ViewFuncT: # Validators are like converters, but they don't handle JSON parsing; we do. if param.validator is not None and not default_assigned: try: - val = ujson.loads(val) + val = orjson.loads(val) except Exception: raise JsonableError(_('Argument "{}" is not valid JSON.').format(post_var_name)) diff --git a/zerver/lib/response.py b/zerver/lib/response.py index 33e6631fe3..f9e99b8497 100644 --- a/zerver/lib/response.py +++ b/zerver/lib/response.py @@ -1,6 +1,6 @@ from typing import Any, List, Mapping, Optional -import ujson +import orjson from django.http import HttpResponse, HttpResponseNotAllowed from django.utils.translation import ugettext as _ @@ -24,15 +24,16 @@ def json_unauthorized(message: Optional[str]=None, if message is None: message = _("Not logged in: API authentication or user session required") resp = HttpResponseUnauthorized("zulip", www_authenticate=www_authenticate) - resp.content = (ujson.dumps({"result": "error", - "msg": message}) + "\n").encode() + resp.content = orjson.dumps( + {"result": "error", "msg": message}, option=orjson.OPT_APPEND_NEWLINE, + ) return resp def json_method_not_allowed(methods: List[str]) -> HttpResponseNotAllowed: resp = HttpResponseNotAllowed(methods) - resp.content = ujson.dumps({"result": "error", - "msg": "Method Not Allowed", - "allowed_methods": methods}).encode() + resp.content = orjson.dumps({"result": "error", + "msg": "Method Not Allowed", + "allowed_methods": methods}) return resp def json_response(res_type: str="success", @@ -41,8 +42,18 @@ def json_response(res_type: str="success", status: int=200) -> HttpResponse: content = {"result": res_type, "msg": msg} content.update(data) - return HttpResponse(content=ujson.dumps(content) + "\n", - content_type='application/json', status=status) + + # Because we don't pass a default handler, OPT_PASSTHROUGH_DATETIME + # actually causes orjson to raise a TypeError on datetime objects. This + # helps us avoid relying on the particular serialization used by orjson. + return HttpResponse( + content=orjson.dumps( + content, + option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_PASSTHROUGH_DATETIME, + ), + content_type='application/json', + status=status, + ) def json_success(data: Mapping[str, Any]={}) -> HttpResponse: return json_response(data=data) diff --git a/zerver/lib/send_email.py b/zerver/lib/send_email.py index f28ab5b917..3fb5687d74 100644 --- a/zerver/lib/send_email.py +++ b/zerver/lib/send_email.py @@ -8,7 +8,7 @@ from email.policy import default from email.utils import formataddr, parseaddr from typing import Any, Dict, List, Mapping, Optional, Tuple -import ujson +import orjson from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.core.management import CommandError @@ -187,7 +187,7 @@ def send_future_email(template_prefix: str, realm: Realm, to_user_ids: Optional[ type=EMAIL_TYPES[template_name], scheduled_timestamp=timezone_now() + delay, realm=realm, - data=ujson.dumps(email_fields)) + data=orjson.dumps(email_fields).decode()) # We store the recipients in the ScheduledEmail object itself, # rather than the JSON data object, so that we can find and clear @@ -241,7 +241,7 @@ def handle_send_email_format_changes(job: Dict[str, Any]) -> None: del job['to_user_id'] def deliver_email(email: ScheduledEmail) -> None: - data = ujson.loads(email.data) + data = orjson.loads(email.data) if email.users.exists(): data['to_user_ids'] = [user.id for user in email.users.all()] if email.address is not None: diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index cbb242c835..d26889d693 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, Union from unittest import mock -import ujson +import orjson from django.apps import apps from django.conf import settings from django.db import connection @@ -182,7 +182,7 @@ class ZulipTestCase(TestCase): if not (url.startswith("/json") or url.startswith("/api/v1")): return try: - content = ujson.loads(result.content) + content = orjson.loads(result.content) except ValueError: return json_url = False @@ -655,7 +655,7 @@ class ZulipTestCase(TestCase): use_first_unread_anchor: bool=False) -> Dict[str, List[Dict[str, Any]]]: post_params = {"anchor": anchor, "num_before": num_before, "num_after": num_after, - "use_first_unread_anchor": ujson.dumps(use_first_unread_anchor)} + "use_first_unread_anchor": orjson.dumps(use_first_unread_anchor).decode()} result = self.client_get("/json/messages", dict(post_params)) data = result.json() return data @@ -683,7 +683,7 @@ class ZulipTestCase(TestCase): "msg": ""}. """ try: - json = ujson.loads(result.content) + json = orjson.loads(result.content) except Exception: # nocoverage json = {'msg': "Error parsing JSON in response!"} self.assertEqual(result.status_code, 200, json['msg']) @@ -696,7 +696,7 @@ class ZulipTestCase(TestCase): def get_json_error(self, result: HttpResponse, status_code: int=400) -> Dict[str, Any]: try: - json = ujson.loads(result.content) + json = orjson.loads(result.content) except Exception: # nocoverage json = {'msg': "Error parsing JSON in response!"} self.assertEqual(result.status_code, status_code, msg=json.get('msg')) @@ -826,9 +826,9 @@ class ZulipTestCase(TestCase): is_web_public: bool=False, allow_fail: bool=False, **kwargs: Any) -> HttpResponse: - post_data = {'subscriptions': ujson.dumps([{"name": stream} for stream in streams]), - 'is_web_public': ujson.dumps(is_web_public), - 'invite_only': ujson.dumps(invite_only)} + post_data = {'subscriptions': orjson.dumps([{"name": stream} for stream in streams]).decode(), + 'is_web_public': orjson.dumps(is_web_public).decode(), + 'invite_only': orjson.dumps(invite_only).decode()} post_data.update(extra_post_data) result = self.api_post(user, "/api/v1/users/me/subscriptions", post_data, **kwargs) if not allow_fail: @@ -929,7 +929,7 @@ class ZulipTestCase(TestCase): directory. If new user entries are needed to test for some additional unusual scenario, it's most likely best to add that to directory.json. """ - directory = ujson.loads(self.fixture_data("directory.json", type="ldap")) + directory = orjson.loads(self.fixture_data("directory.json", type="ldap")) for dn, attrs in directory.items(): if 'uid' in attrs: @@ -1072,7 +1072,7 @@ class WebhookTestCase(ZulipTestCase): """Can be implemented either as returning a dictionary containing the post parameters or as string containing the body of the request.""" assert self.FIXTURE_DIR_NAME is not None - return ujson.dumps(ujson.loads(self.webhook_fixture_data(self.FIXTURE_DIR_NAME, fixture_name))) + return orjson.dumps(orjson.loads(self.webhook_fixture_data(self.FIXTURE_DIR_NAME, fixture_name))).decode() def do_test_topic(self, msg: Message, expected_topic: Optional[str]) -> None: if expected_topic is not None: diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index fcb52212b5..f5ce637273 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -26,7 +26,7 @@ from unittest import mock import boto3 import fakeldap import ldap -import ujson +import orjson from boto3.resources.base import ServiceResource from django.conf import settings from django.db.migrations.state import StateApps @@ -336,6 +336,16 @@ def instrument_url(f: UrlFuncT) -> UrlFuncT: else: extra_info = '' + if isinstance(info, POSTRequestMock): + info = "" + elif isinstance(info, bytes): + info = "" + elif isinstance(info, dict): + info = { + k: "" if hasattr(v, "read") and callable(getattr(v, "read")) else v + for k, v in info.items() + } + append_instrumentation_data(dict( url=url, status_code=result.status_code, @@ -425,20 +435,9 @@ def write_instrumentation_reports(full_suite: bool, include_webhooks: bool) -> N var_dir = 'var' # TODO make sure path is robust here fn = os.path.join(var_dir, 'url_coverage.txt') - with open(fn, 'w') as f: + with open(fn, 'wb') as f: for call in calls: - try: - line = ujson.dumps(call) - f.write(line + '\n') - except OverflowError: # nocoverage -- test suite error handling - print(''' - A JSON overflow error was encountered while - producing the URL coverage report. Sometimes - this indicates that a test is passing objects - into methods like client_post(), which is - unnecessary and leads to false positives. - ''') - print(call) + f.write(orjson.dumps(call, option=orjson.OPT_APPEND_NEWLINE)) if full_suite: print(f'INFO: URL coverage report is in {fn}') diff --git a/zerver/lib/validator.py b/zerver/lib/validator.py index d62fb8cb82..d989b2b4d9 100644 --- a/zerver/lib/validator.py +++ b/zerver/lib/validator.py @@ -31,7 +31,7 @@ import re from datetime import datetime from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union, cast, overload -import ujson +import orjson from django.core.exceptions import ValidationError from django.core.validators import URLValidator, validate_email from django.utils.translation import ugettext as _ @@ -315,7 +315,7 @@ def validate_choice_field(var_name: str, field_data: str, value: object) -> str: choice field. This is not used to validate admin data. """ s = check_string(var_name, value) - field_data_dict = ujson.loads(field_data) + field_data_dict = orjson.loads(field_data) if s not in field_data_dict: msg = _("'{value}' is not a valid choice for '{field_name}'.") raise ValidationError(msg.format(value=value, field_name=var_name)) diff --git a/zerver/management/commands/add_users_to_mailing_list.py b/zerver/management/commands/add_users_to_mailing_list.py index d053176e7b..a0ee04d53d 100644 --- a/zerver/management/commands/add_users_to_mailing_list.py +++ b/zerver/management/commands/add_users_to_mailing_list.py @@ -2,8 +2,8 @@ import argparse from datetime import datetime from typing import Any, Optional +import orjson import requests -import ujson from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django.utils.timezone import now as timezone_now @@ -66,7 +66,7 @@ class Command(BaseCommand): }, } r = requests.post(endpoint, auth=('apikey', api_key), json=data, timeout=10) - if r.status_code == 400 and ujson.loads(r.text)['title'] == 'Member Exists': + if r.status_code == 400 and orjson.loads(r.content)['title'] == 'Member Exists': print("{} is already a part of the list.".format(data['email_address'])) elif r.status_code >= 400: print(r.text) diff --git a/zerver/management/commands/compilemessages.py b/zerver/management/commands/compilemessages.py index fff408548f..2dc19a919a 100644 --- a/zerver/management/commands/compilemessages.py +++ b/zerver/management/commands/compilemessages.py @@ -4,8 +4,8 @@ import re from subprocess import CalledProcessError, check_output from typing import Any, Dict, List +import orjson import polib -import ujson from django.conf import settings from django.conf.locale import LANG_INFO from django.core.management.base import CommandParser @@ -38,8 +38,8 @@ class Command(compilemessages.Command): path = join(deploy_root, 'locale', 'language_options.json') output_path = join(deploy_root, 'locale', 'language_name_map.json') - with open(path) as reader: - languages = ujson.load(reader) + with open(path, "rb") as reader: + languages = orjson.loads(reader.read()) lang_list = [] for lang_info in languages['languages']: lang_info['name'] = lang_info['name_local'] @@ -48,9 +48,13 @@ class Command(compilemessages.Command): lang_list.sort(key=lambda lang: lang['name']) - with open(output_path, 'w') as output_file: - ujson.dump({'name_map': lang_list}, output_file, indent=4, sort_keys=True) - output_file.write('\n') + with open(output_path, 'wb') as output_file: + output_file.write( + orjson.dumps( + {'name_map': lang_list}, + option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS, + ) + ) def get_po_filename(self, locale_path: str, locale: str) -> str: po_template = '{}/{}/LC_MESSAGES/django.po' @@ -145,15 +149,15 @@ class Command(compilemessages.Command): total = len(po.translated_entries()) + not_translated # frontend stats - with open(self.get_json_filename(locale_path, locale)) as reader: - for key, value in ujson.load(reader).items(): + with open(self.get_json_filename(locale_path, locale), "rb") as reader: + for key, value in orjson.loads(reader.read()).items(): total += 1 if value == '': not_translated += 1 # mobile stats - with open(os.path.join(locale_path, 'mobile_info.json')) as mob: - mobile_info = ujson.load(mob) + with open(os.path.join(locale_path, 'mobile_info.json'), "rb") as mob: + mobile_info = orjson.loads(mob.read()) try: info = mobile_info[locale] except KeyError: diff --git a/zerver/management/commands/enqueue_file.py b/zerver/management/commands/enqueue_file.py index 68ab4fda85..105678f72c 100644 --- a/zerver/management/commands/enqueue_file.py +++ b/zerver/management/commands/enqueue_file.py @@ -2,7 +2,7 @@ import sys from argparse import ArgumentParser from typing import IO, Any -import ujson +import orjson from django.core.management.base import BaseCommand from zerver.lib.queue import queue_json_publish @@ -51,7 +51,7 @@ You can use "-" to represent stdin. print(f'Queueing to queue {queue_name}: {payload}') # Verify that payload is valid json. - data = ujson.loads(payload) + data = orjson.loads(payload) # This is designed to use the `error` method rather than # the call_consume_in_tests flow. diff --git a/zerver/management/commands/send_to_email_mirror.py b/zerver/management/commands/send_to_email_mirror.py index 1a1db0ab33..c9e39f8554 100644 --- a/zerver/management/commands/send_to_email_mirror.py +++ b/zerver/management/commands/send_to_email_mirror.py @@ -5,7 +5,7 @@ import os from email.message import EmailMessage from typing import Optional -import ujson +import orjson from django.conf import settings from django.core.management.base import CommandParser @@ -80,8 +80,8 @@ Example: return os.path.exists(fixture_path) def _parse_email_json_fixture(self, fixture_path: str) -> EmailMessage: - with open(fixture_path) as fp: - json_content = ujson.load(fp)[0] + with open(fixture_path, "rb") as fp: + json_content = orjson.loads(fp.read())[0] message = EmailMessage() message['From'] = json_content['from'] diff --git a/zerver/management/commands/send_webhook_fixture_message.py b/zerver/management/commands/send_webhook_fixture_message.py index 9da940be72..ac085c31fa 100644 --- a/zerver/management/commands/send_webhook_fixture_message.py +++ b/zerver/management/commands/send_webhook_fixture_message.py @@ -1,7 +1,7 @@ import os from typing import Dict, Optional, Union -import ujson +import orjson from django.conf import settings from django.core.management.base import CommandParser from django.test import Client @@ -55,7 +55,7 @@ approach shown above. if not custom_headers: return {} try: - custom_headers_dict = ujson.loads(custom_headers) + custom_headers_dict = orjson.loads(custom_headers) except ValueError as ve: raise CommandError('Encountered an error while attempting to parse custom headers: {}\n' 'Note: all strings must be enclosed within "" instead of \'\''.format(ve)) @@ -90,6 +90,6 @@ approach shown above. def _does_fixture_path_exist(self, fixture_path: str) -> bool: return os.path.exists(fixture_path) - def _get_fixture_as_json(self, fixture_path: str) -> str: - with open(fixture_path) as f: - return ujson.dumps(ujson.load(f)) + def _get_fixture_as_json(self, fixture_path: str) -> bytes: + with open(fixture_path, "rb") as f: + return orjson.dumps(orjson.loads(f.read())) diff --git a/zerver/migrations/0097_reactions_emoji_code.py b/zerver/migrations/0097_reactions_emoji_code.py index a553452ebb..aae9e78164 100644 --- a/zerver/migrations/0097_reactions_emoji_code.py +++ b/zerver/migrations/0097_reactions_emoji_code.py @@ -1,7 +1,7 @@ # Generated by Django 1.11.2 on 2017-06-18 21:26 import os -import ujson +import orjson from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps @@ -11,7 +11,8 @@ def populate_new_fields(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> # Open the JSON file which contains the data to be used for migration. MIGRATION_DATA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "management", "data") path_to_unified_reactions = os.path.join(MIGRATION_DATA_PATH, "unified_reactions.json") - unified_reactions = ujson.load(open(path_to_unified_reactions)) + with open(path_to_unified_reactions, "rb") as f: + unified_reactions = orjson.loads(f.read()) Reaction = apps.get_model('zerver', 'Reaction') for reaction in Reaction.objects.all(): diff --git a/zerver/migrations/0102_convert_muted_topic.py b/zerver/migrations/0102_convert_muted_topic.py index 6cc3307659..9da9ba4e60 100644 --- a/zerver/migrations/0102_convert_muted_topic.py +++ b/zerver/migrations/0102_convert_muted_topic.py @@ -1,5 +1,5 @@ # Generated by Django 1.11.4 on 2017-08-30 00:26 -import ujson +import orjson from django.db import connection, migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps @@ -46,7 +46,7 @@ def convert_muted_topics(apps: StateApps, schema_editor: DatabaseSchemaEditor) - realm_id = row['realm_id'] muted_topics = row['muted_topics'] - tups = ujson.loads(muted_topics) + tups = orjson.loads(muted_topics) for (stream_name, topic_name) in tups: stream_name = stream_name.lower() val = stream_dict.get((stream_name, realm_id)) diff --git a/zerver/migrations/0209_user_profile_no_empty_password.py b/zerver/migrations/0209_user_profile_no_empty_password.py index 212f671984..9ffb209043 100644 --- a/zerver/migrations/0209_user_profile_no_empty_password.py +++ b/zerver/migrations/0209_user_profile_no_empty_password.py @@ -2,7 +2,7 @@ from typing import Any, Set, Union -import ujson +import orjson from django.conf import settings from django.contrib.auth.hashers import check_password, make_password from django.db import migrations @@ -146,10 +146,10 @@ def ensure_no_empty_passwords(apps: StateApps, schema_editor: DatabaseSchemaEdit modified_user=user_profile, event_type=event_type, event_time=event_time, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ 'migration_id': MIGRATION_ID, 'affected_user_type': affected_user_type, - }), + }).decode(), ) # If Zulip's built-in password authentication is not enabled on diff --git a/zerver/migrations/0277_migrate_alert_word.py b/zerver/migrations/0277_migrate_alert_word.py index cca8edc8ef..a5bba106e7 100644 --- a/zerver/migrations/0277_migrate_alert_word.py +++ b/zerver/migrations/0277_migrate_alert_word.py @@ -1,6 +1,6 @@ from typing import Dict, List -import ujson +import orjson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps @@ -11,7 +11,7 @@ def move_to_seperate_table(apps: StateApps, schema_editor: DatabaseSchemaEditor) AlertWord = apps.get_model('zerver', 'AlertWord') for user_profile in UserProfile.objects.all(): - list_of_words = ujson.loads(user_profile.alert_words) + list_of_words = orjson.loads(user_profile.alert_words) # Remove duplicates with our case-insensitive model. word_dict: Dict[str, str] = {} @@ -36,7 +36,7 @@ def move_back_to_user_profile(apps: StateApps, schema_editor: DatabaseSchemaEdit for (user_id, words) in user_ids_with_words.items(): user_profile = UserProfile.objects.get(id=user_id) - user_profile.alert_words = ujson.dumps(words) + user_profile.alert_words = orjson.dumps(words).decode() user_profile.save(update_fields=['alert_words']) class Migration(migrations.Migration): diff --git a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py index bbd56d5e0e..17bdcaaa90 100644 --- a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py +++ b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py @@ -1,7 +1,7 @@ # Generated by Django 2.2.12 on 2020-05-16 18:34 from typing import Any, Dict -import ujson +import orjson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps @@ -47,11 +47,11 @@ def set_realm_admins_as_realm_owners(apps: StateApps, schema_editor: DatabaseSch audit_log_entry = RealmAuditLog(realm=user.realm, modified_user=user, event_type=RealmAuditLog.USER_ROLE_CHANGED, event_time=timezone_now(), - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: UserProfile.ROLE_REALM_ADMINISTRATOR, RealmAuditLog.NEW_VALUE: UserProfile.ROLE_REALM_OWNER, RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user.realm), - })) + }).decode()) objects_to_create.append(audit_log_entry) RealmAuditLog.objects.bulk_create(objects_to_create) diff --git a/zerver/tests/test_alert_words.py b/zerver/tests/test_alert_words.py index 0a66b5e8e5..a72eda0956 100644 --- a/zerver/tests/test_alert_words.py +++ b/zerver/tests/test_alert_words.py @@ -1,4 +1,4 @@ -import ujson +import orjson from zerver.lib.actions import do_add_alert_words, do_remove_alert_words from zerver.lib.alert_words import alert_words_in_realm, user_alert_words @@ -24,7 +24,7 @@ class AlertWordTests(ZulipTestCase): self.login_user(user) params = { - 'alert_words': ujson.dumps(['milk', 'cookies']), + 'alert_words': orjson.dumps(['milk', 'cookies']).decode(), } result = self.client_post('/json/users/me/alert_words', params) self.assert_json_success(result) @@ -135,7 +135,7 @@ class AlertWordTests(ZulipTestCase): self.login_user(user) result = self.client_post('/json/users/me/alert_words', - {'alert_words': ujson.dumps(['one ', '\n two', 'three'])}) + {'alert_words': orjson.dumps(['one ', '\n two', 'three']).decode()}) self.assert_json_success(result) self.assertEqual(set(result.json()['alert_words']), {'one', 'two', 'three'}) @@ -144,12 +144,12 @@ class AlertWordTests(ZulipTestCase): self.login_user(user) result = self.client_post('/json/users/me/alert_words', - {'alert_words': ujson.dumps(['one', 'two', 'three'])}) + {'alert_words': orjson.dumps(['one', 'two', 'three']).decode()}) self.assert_json_success(result) self.assertEqual(set(result.json()['alert_words']), {'one', 'two', 'three'}) result = self.client_delete('/json/users/me/alert_words', - {'alert_words': ujson.dumps(['one'])}) + {'alert_words': orjson.dumps(['one']).decode()}) self.assert_json_success(result) self.assertEqual(set(result.json()['alert_words']), {'two', 'three'}) @@ -164,7 +164,7 @@ class AlertWordTests(ZulipTestCase): self.login_user(user) result = self.client_post('/json/users/me/alert_words', - {'alert_words': ujson.dumps(['one', 'two', 'three'])}) + {'alert_words': orjson.dumps(['one', 'two', 'three']).decode()}) self.assert_json_success(result) self.assertEqual(set(result.json()['alert_words']), {'one', 'two', 'three'}) @@ -192,7 +192,7 @@ class AlertWordTests(ZulipTestCase): self.login_user(user) result = self.client_post('/json/users/me/alert_words', - {'alert_words': ujson.dumps(['ALERT'])}) + {'alert_words': orjson.dumps(['ALERT']).decode()}) content = 'this is an ALERT for you' self.send_stream_message(user, "Denmark", content) diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index b3ab5b72bb..fd9ddd5b33 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -1,7 +1,7 @@ from datetime import timedelta from typing import Any, Dict, Union -import ujson +import orjson from django.contrib.auth.password_validation import validate_password from django.utils.timezone import now as timezone_now @@ -81,7 +81,7 @@ class TestRealmAuditLog(ZulipTestCase): for event in RealmAuditLog.objects.filter( realm=realm, acting_user=user, modified_user=user, modified_stream=None, event_time__gte=now, event_time__lte=now+timedelta(minutes=60)): - extra_data = ujson.loads(event.extra_data) + extra_data = orjson.loads(event.extra_data) self.check_role_count_schema(extra_data[RealmAuditLog.ROLE_COUNT]) self.assertNotIn(RealmAuditLog.OLD_VALUE, extra_data) @@ -102,7 +102,7 @@ class TestRealmAuditLog(ZulipTestCase): event_type=RealmAuditLog.USER_ROLE_CHANGED, realm=realm, modified_user=user_profile, acting_user=acting_user, event_time__gte=now, event_time__lte=now+timedelta(minutes=60)): - extra_data = ujson.loads(event.extra_data) + extra_data = orjson.loads(event.extra_data) self.check_role_count_schema(extra_data[RealmAuditLog.ROLE_COUNT]) self.assertIn(RealmAuditLog.OLD_VALUE, extra_data) self.assertIn(RealmAuditLog.NEW_VALUE, extra_data) @@ -149,7 +149,7 @@ class TestRealmAuditLog(ZulipTestCase): start = timezone_now() new_name = 'George Hamletovich' self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user("hamlet").id), req) self.assertTrue(result.status_code == 200) query = RealmAuditLog.objects.filter(event_type=RealmAuditLog.USER_FULL_NAME_CHANGED, @@ -228,12 +228,12 @@ class TestRealmAuditLog(ZulipTestCase): realm = get_realm('zulip') do_deactivate_realm(realm) log_entry = RealmAuditLog.objects.get(realm=realm, event_type=RealmAuditLog.REALM_DEACTIVATED) - extra_data = ujson.loads(log_entry.extra_data) + extra_data = orjson.loads(log_entry.extra_data) self.check_role_count_schema(extra_data[RealmAuditLog.ROLE_COUNT]) do_reactivate_realm(realm) log_entry = RealmAuditLog.objects.get(realm=realm, event_type=RealmAuditLog.REALM_REACTIVATED) - extra_data = ujson.loads(log_entry.extra_data) + extra_data = orjson.loads(log_entry.extra_data) self.check_role_count_schema(extra_data[RealmAuditLog.ROLE_COUNT]) def test_create_stream_if_needed(self) -> None: @@ -268,7 +268,7 @@ class TestRealmAuditLog(ZulipTestCase): realm_audit_logs = RealmAuditLog.objects.filter(realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time__gte=now, acting_user=user) self.assertEqual(realm_audit_logs.count(), 1) - extra_data = ujson.loads(realm_audit_logs[0].extra_data) + extra_data = orjson.loads(realm_audit_logs[0].extra_data) expected_new_value = auth_method_dict self.assertEqual(extra_data[RealmAuditLog.OLD_VALUE], expected_old_value) self.assertEqual(extra_data[RealmAuditLog.NEW_VALUE], expected_new_value) @@ -305,7 +305,7 @@ class TestRealmAuditLog(ZulipTestCase): realm_audit_logs = RealmAuditLog.objects.filter(realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time__gte=now, acting_user=user).order_by("id") self.assertEqual(realm_audit_logs.count(), 2) - self.assertEqual([ujson.loads(entry.extra_data) for entry in realm_audit_logs], + self.assertEqual([orjson.loads(entry.extra_data) for entry in realm_audit_logs], values_expected) def test_set_realm_notifications_stream(self) -> None: @@ -320,11 +320,11 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time__gte=now, acting_user=user, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: stream.id, 'property': 'notifications_stream', - })).count(), 1) + }).decode()).count(), 1) def test_set_realm_signup_notifications_stream(self) -> None: now = timezone_now() @@ -338,11 +338,11 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time__gte=now, acting_user=user, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: stream.id, 'property': 'signup_notifications_stream', - })).count(), 1) + }).decode()).count(), 1) def test_change_icon_source(self) -> None: test_start = timezone_now() @@ -389,7 +389,7 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=user.realm, event_type=RealmAuditLog.SUBSCRIPTION_PROPERTY_CHANGED, event_time__gte=now, acting_user=user, modified_user=user, - extra_data=ujson.dumps(expected_extra_data)).count(), 1) + extra_data=orjson.dumps(expected_extra_data).decode()).count(), 1) self.assertEqual(getattr(sub, property), value) def test_change_default_streams(self) -> None: @@ -402,10 +402,10 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=user.realm, event_type=RealmAuditLog.USER_DEFAULT_SENDING_STREAM_CHANGED, event_time__gte=now, acting_user=user, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: stream.id, - })).count(), 1) + }).decode()).count(), 1) self.assertEqual(user.default_sending_stream, stream) old_value = user.default_events_register_stream_id @@ -413,10 +413,10 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=user.realm, event_type=RealmAuditLog.USER_DEFAULT_REGISTER_STREAM_CHANGED, event_time__gte=now, acting_user=user, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: stream.id, - })).count(), 1) + }).decode()).count(), 1) self.assertEqual(user.default_events_register_stream, stream) old_value = user.default_all_public_streams @@ -424,10 +424,10 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=user.realm, event_type=RealmAuditLog.USER_DEFAULT_ALL_PUBLIC_STREAMS_CHANGED, event_time__gte=now, acting_user=user, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: False - })).count(), 1) + }).decode()).count(), 1) self.assertEqual(user.default_all_public_streams, False) def test_rename_stream(self) -> None: @@ -439,10 +439,10 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=user.realm, event_type=RealmAuditLog.STREAM_NAME_CHANGED, event_time__gte=now, acting_user=user, modified_stream=stream, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_name, RealmAuditLog.NEW_VALUE: 'updated name' - })).count(), 1) + }).decode()).count(), 1) self.assertEqual(stream.name, 'updated name') def test_change_notification_settings(self) -> None: @@ -467,5 +467,5 @@ class TestRealmAuditLog(ZulipTestCase): self.assertEqual(RealmAuditLog.objects.filter( realm=user.realm, event_type=RealmAuditLog.USER_NOTIFICATION_SETTINGS_CHANGED, event_time__gte=now, acting_user=user, modified_user=user, - extra_data=ujson.dumps(expected_extra_data)).count(), 1) + extra_data=orjson.dumps(expected_extra_data).decode()).count(), 1) self.assertEqual(getattr(user, setting), value) diff --git a/zerver/tests/test_auth_backends.py b/zerver/tests/test_auth_backends.py index 24d82b7a01..3f01d7a729 100644 --- a/zerver/tests/test_auth_backends.py +++ b/zerver/tests/test_auth_backends.py @@ -11,9 +11,9 @@ from unittest import mock import jwt import ldap +import orjson import requests import responses -import ujson from bs4 import BeautifulSoup from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings @@ -324,8 +324,8 @@ class AuthBackendTest(ZulipTestCase): self.assert_in_response(realm.name, result) self.assert_in_response("Log in to Zulip", result) - data = dict(description=ujson.dumps("New realm description"), - name=ujson.dumps("New Zulip")) + data = dict(description=orjson.dumps("New realm description").decode(), + name=orjson.dumps("New Zulip").decode()) result = self.client_patch('/json/realm', data) self.assert_json_success(result) @@ -1525,7 +1525,7 @@ class SAMLAuthBackendTest(SocialAuthBase): parsed_url = urllib.parse.urlparse(result.url) relay_state = urllib.parse.parse_qs(parsed_url.query)['RelayState'][0] # Make sure params are getting encoded into RelayState: - data = SAMLAuthBackend.get_data_from_redis(ujson.loads(relay_state)['state_token']) + data = SAMLAuthBackend.get_data_from_redis(orjson.loads(relay_state)['state_token']) assert data is not None if next: self.assertEqual(data['next'], next) @@ -1632,9 +1632,9 @@ class SAMLAuthBackendTest(SocialAuthBase): self.assertLogs(self.logger_string, level='INFO') as m: # This mock causes AuthFailed to be raised. saml_response = self.generate_saml_response(self.email, self.name) - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"SAMLResponse": saml_response, "RelayState": relay_state} result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -1648,9 +1648,9 @@ class SAMLAuthBackendTest(SocialAuthBase): side_effect=AuthStateForbidden('State forbidden')), \ self.assertLogs(self.logger_string, level='WARNING') as m: saml_response = self.generate_saml_response(self.email, self.name) - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"SAMLResponse": saml_response, "RelayState": relay_state} result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -1670,9 +1670,9 @@ class SAMLAuthBackendTest(SocialAuthBase): # Check that POSTing the RelayState, but with missing SAMLResponse, # doesn't cause errors either: with self.assertLogs(self.logger_string, level='INFO') as m: - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"RelayState": relay_state} result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -1681,9 +1681,9 @@ class SAMLAuthBackendTest(SocialAuthBase): # Now test bad SAMLResponses. with self.assertLogs(self.logger_string, level='INFO') as m: - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"RelayState": relay_state, 'SAMLResponse': ''} result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -1691,9 +1691,9 @@ class SAMLAuthBackendTest(SocialAuthBase): self.assertTrue(m.output != '') with self.assertLogs(self.logger_string, level='INFO') as m: - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"RelayState": relay_state, 'SAMLResponse': 'b'} result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -1701,9 +1701,9 @@ class SAMLAuthBackendTest(SocialAuthBase): self.assertTrue(m.output != '') with self.assertLogs(self.logger_string, level='INFO') as m: - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"RelayState": relay_state, 'SAMLResponse': 'dGVzdA=='} # base64 encoded 'test' result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -1725,9 +1725,9 @@ class SAMLAuthBackendTest(SocialAuthBase): )]) def test_social_auth_complete_wrong_issuing_idp(self) -> None: - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() saml_response = self.generate_saml_response(email=self.example_email("hamlet"), name="King Hamlet") @@ -1753,9 +1753,9 @@ class SAMLAuthBackendTest(SocialAuthBase): with self.assertLogs(self.logger_string, level='INFO') as m, \ mock.patch.object(SAMLAuthBackend, 'get_issuing_idp', return_value='test_idp'): - relay_state = ujson.dumps(dict( + relay_state = orjson.dumps(dict( state_token=SAMLAuthBackend.put_data_in_redis({"subdomain": "zulip"}), - )) + )).decode() post_params = {"RelayState": relay_state, 'SAMLResponse': 'dGVzdA=='} result = self.client_post('/complete/saml/', post_params) self.assertEqual(result.status_code, 302) @@ -4944,12 +4944,12 @@ class TestAdminSetBackends(ZulipTestCase): # Log in as admin self.login('iago') result = self.client_patch("/json/realm", { - 'authentication_methods': ujson.dumps({'Email': False, 'Dev': True})}) + 'authentication_methods': orjson.dumps({'Email': False, 'Dev': True}).decode()}) self.assert_json_error(result, 'Must be an organization owner') self.login('desdemona') result = self.client_patch("/json/realm", { - 'authentication_methods': ujson.dumps({'Email': False, 'Dev': True})}) + 'authentication_methods': orjson.dumps({'Email': False, 'Dev': True}).decode()}) self.assert_json_success(result) realm = get_realm('zulip') self.assertFalse(password_auth_enabled(realm)) @@ -4959,7 +4959,7 @@ class TestAdminSetBackends(ZulipTestCase): # Log in as admin self.login('desdemona') result = self.client_patch("/json/realm", { - 'authentication_methods': ujson.dumps({'Email': False, 'Dev': False})}) + 'authentication_methods': orjson.dumps({'Email': False, 'Dev': False}).decode()}) self.assert_json_error(result, 'At least one authentication method must be enabled.') realm = get_realm('zulip') self.assertTrue(password_auth_enabled(realm)) @@ -4970,7 +4970,7 @@ class TestAdminSetBackends(ZulipTestCase): self.login('desdemona') # Set some supported and unsupported backends result = self.client_patch("/json/realm", { - 'authentication_methods': ujson.dumps({'Email': False, 'Dev': True, 'GitHub': False})}) + 'authentication_methods': orjson.dumps({'Email': False, 'Dev': True, 'GitHub': False}).decode()}) self.assert_json_success(result) realm = get_realm('zulip') # Check that unsupported backend is not enabled diff --git a/zerver/tests/test_bots.py b/zerver/tests/test_bots.py index db1ad3fe96..3ba5b801b2 100644 --- a/zerver/tests/test_bots.py +++ b/zerver/tests/test_bots.py @@ -3,7 +3,7 @@ import os from typing import Any, Dict, List, Mapping, Optional from unittest.mock import MagicMock, patch -import ujson +import orjson from django.core import mail from django.test import override_settings from zulip_bots.custom_exceptions import ConfigValidationError @@ -198,7 +198,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): ) users_result = self.client_get('/json/users') - members = ujson.loads(users_result.content)['members'] + members = orjson.loads(users_result.content)['members'] bots = [m for m in members if m['email'] == 'hambot-bot@zulip.testserver'] self.assertEqual(len(bots), 1) bot = bots[0] @@ -330,7 +330,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): ) users_result = self.client_get('/json/users') - members = ujson.loads(users_result.content)['members'] + members = orjson.loads(users_result.content)['members'] bots = [m for m in members if m['email'] == 'hambot-bot@zulip.testserver'] self.assertEqual(len(bots), 1) bot = bots[0] @@ -525,7 +525,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): def test_add_bot_with_default_all_public_streams(self) -> None: self.login('hamlet') self.assert_num_bots_equal(0) - result = self.create_bot(default_all_public_streams=ujson.dumps(True)) + result = self.create_bot(default_all_public_streams=orjson.dumps(True).decode()) self.assert_num_bots_equal(1) self.assertTrue(result['default_all_public_streams']) @@ -1302,7 +1302,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): result = self.client_post("/json/bots", bot_info) self.assert_json_success(result) bot_info = { - 'default_all_public_streams': ujson.dumps(True), + 'default_all_public_streams': orjson.dumps(True).decode(), } email = 'hambot-bot@zulip.testserver' result = self.client_patch(f"/json/bots/{self.get_bot_user(email).id}", bot_info) @@ -1322,7 +1322,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): result = self.client_post("/json/bots", bot_info) self.assert_json_success(result) bot_info = { - 'default_all_public_streams': ujson.dumps(False), + 'default_all_public_streams': orjson.dumps(False).decode(), } email = 'hambot-bot@zulip.testserver' result = self.client_patch(f"/json/bots/{self.get_bot_user(email).id}", bot_info) @@ -1374,23 +1374,23 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): 'full_name': 'The Bot of Hamlet', 'short_name': 'hambot', 'bot_type': UserProfile.OUTGOING_WEBHOOK_BOT, - 'payload_url': ujson.dumps("http://foo.bar.com"), + 'payload_url': orjson.dumps("http://foo.bar.com").decode(), 'service_interface': Service.GENERIC, } result = self.client_post("/json/bots", bot_info) self.assert_json_success(result) bot_info = { - 'service_payload_url': ujson.dumps("http://foo.bar2.com"), + 'service_payload_url': orjson.dumps("http://foo.bar2.com").decode(), 'service_interface': Service.SLACK, } email = 'hambot-bot@zulip.testserver' result = self.client_patch(f"/json/bots/{self.get_bot_user(email).id}", bot_info) self.assert_json_success(result) - service_interface = ujson.loads(result.content)['service_interface'] + service_interface = orjson.loads(result.content)['service_interface'] self.assertEqual(service_interface, Service.SLACK) - service_payload_url = ujson.loads(result.content)['service_payload_url'] + service_payload_url = orjson.loads(result.content)['service_payload_url'] self.assertEqual(service_payload_url, "http://foo.bar2.com") @patch('zulip_bots.bots.giphy.giphy.GiphyHandler.validate_config') @@ -1399,13 +1399,13 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): full_name='Bot with config data', bot_type=UserProfile.EMBEDDED_BOT, service_name='giphy', - config_data=ujson.dumps({'key': '12345678'})) - bot_info = {'config_data': ujson.dumps({'key': '87654321'})} + config_data=orjson.dumps({'key': '12345678'}).decode()) + bot_info = {'config_data': orjson.dumps({'key': '87654321'}).decode()} email = 'test-bot@zulip.testserver' result = self.client_patch(f"/json/bots/{self.get_bot_user(email).id}", bot_info) self.assert_json_success(result) - config_data = ujson.loads(result.content)['config_data'] - self.assertEqual(config_data, ujson.loads(bot_info['config_data'])) + config_data = orjson.loads(result.content)['config_data'] + self.assertEqual(config_data, orjson.loads(bot_info['config_data'])) def test_outgoing_webhook_invalid_interface(self) -> None: self.login('hamlet') @@ -1413,7 +1413,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): 'full_name': 'Outgoing Webhook test bot', 'short_name': 'outgoingservicebot', 'bot_type': UserProfile.OUTGOING_WEBHOOK_BOT, - 'payload_url': ujson.dumps('http://127.0.0.1:5002'), + 'payload_url': orjson.dumps('http://127.0.0.1:5002').decode(), 'interface_type': -1, } result = self.client_post("/json/bots", bot_info) @@ -1429,7 +1429,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): 'full_name': 'Outgoing Webhook test bot', 'short_name': 'outgoingservicebot', 'bot_type': UserProfile.OUTGOING_WEBHOOK_BOT, - 'payload_url': ujson.dumps('http://127.0.0.1:5002'), + 'payload_url': orjson.dumps('http://127.0.0.1:5002').decode(), } bot_info.update(extras) result = self.client_post("/json/bots", bot_info) @@ -1447,7 +1447,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): self.assertEqual(service.user_profile, bot) # invalid URL test case. - bot_info['payload_url'] = ujson.dumps('http://127.0.0.:5002') + bot_info['payload_url'] = orjson.dumps('http://127.0.0.:5002').decode() result = self.client_post("/json/bots", bot_info) self.assert_json_error(result, "payload_url is not a URL") @@ -1472,7 +1472,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): 'full_name': 'Outgoing Webhook test bot', 'short_name': 'outgoingservicebot', 'bot_type': UserProfile.OUTGOING_WEBHOOK_BOT, - 'payload_url': ujson.dumps('http://127.0.0.1:5002'), + 'payload_url': orjson.dumps('http://127.0.0.1:5002').decode(), 'interface_type': -1, } result = self.client_post("/json/bots", bot_info) @@ -1489,7 +1489,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): user_profile=self.example_user("hamlet"), bot_type=UserProfile.EMBEDDED_BOT, service_name='followup', - config_data=ujson.dumps({'key': 'value'}), + config_data=orjson.dumps({'key': 'value'}).decode(), assert_json_error_msg='Embedded bots are not enabled.', **extras, ) @@ -1500,7 +1500,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): user_profile=self.example_user("hamlet"), bot_type=UserProfile.EMBEDDED_BOT, service_name='followup', - config_data=ujson.dumps(bot_config_info), + config_data=orjson.dumps(bot_config_info).decode(), **extras) bot_email = "embeddedservicebot-bot@zulip.testserver" bot_realm = get_realm('zulip') @@ -1528,7 +1528,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): short_name='embeddedservicebot', user_profile=self.example_user("hamlet"), service_name='followup', - config_data=ujson.dumps({'invalid': ['config', 'value']}), + config_data=orjson.dumps({'invalid': ['config', 'value']}).decode(), assert_json_error_msg='config_data contains a value that is not a string', **extras, ) @@ -1540,7 +1540,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): 'short_name': 'embeddedservicebot3', 'bot_type': UserProfile.EMBEDDED_BOT, 'service_name': 'giphy', - 'config_data': ujson.dumps(incorrect_bot_config_info), + 'config_data': orjson.dumps(incorrect_bot_config_info).decode(), } bot_info.update(extras) with patch('zulip_bots.bots.giphy.giphy.GiphyHandler.validate_config', side_effect=ConfigValidationError): @@ -1564,7 +1564,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): "short_name": "my-stripe", "bot_type": UserProfile.INCOMING_WEBHOOK_BOT, "service_name": "stripe", - "config_data": ujson.dumps({"stripe_api_key": "sample-api-key"}), + "config_data": orjson.dumps({"stripe_api_key": "sample-api-key"}).decode(), } self.create_bot(**bot_metadata) new_bot = UserProfile.objects.get(full_name="My Stripe Bot") @@ -1580,12 +1580,12 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): "short_name": "my-stripe", "bot_type": UserProfile.INCOMING_WEBHOOK_BOT, "service_name": "stripe", - "config_data": ujson.dumps({"stripe_api_key": "_invalid_key"}), + "config_data": orjson.dumps({"stripe_api_key": "_invalid_key"}).decode(), } response = self.client_post("/json/bots", bot_metadata) self.assertEqual(response.status_code, 400) expected_error_message = 'Invalid stripe_api_key value _invalid_key (stripe_api_key starts with a "_" and is hence invalid.)' - self.assertEqual(ujson.loads(response.content.decode('utf-8'))["msg"], expected_error_message) + self.assertEqual(orjson.loads(response.content.decode('utf-8'))["msg"], expected_error_message) with self.assertRaises(UserProfile.DoesNotExist): UserProfile.objects.get(full_name="My Stripe Bot") @@ -1601,7 +1601,7 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): response = self.client_post("/json/bots", bot_metadata) self.assertEqual(response.status_code, 400) expected_error_message = "Missing configuration parameters: {'stripe_api_key'}" - self.assertEqual(ujson.loads(response.content.decode('utf-8'))["msg"], expected_error_message) + self.assertEqual(orjson.loads(response.content.decode('utf-8'))["msg"], expected_error_message) with self.assertRaises(UserProfile.DoesNotExist): UserProfile.objects.get(full_name="My Stripe Bot") @@ -1630,6 +1630,6 @@ class BotTest(ZulipTestCase, UploadSerializeMixin): response = self.client_post("/json/bots", bot_metadata) self.assertEqual(response.status_code, 400) expected_error_message = "Invalid integration 'stripes'." - self.assertEqual(ujson.loads(response.content.decode('utf-8'))["msg"], expected_error_message) + self.assertEqual(orjson.loads(response.content.decode('utf-8'))["msg"], expected_error_message) with self.assertRaises(UserProfile.DoesNotExist): UserProfile.objects.get(full_name="My Stripe Bot") diff --git a/zerver/tests/test_custom_profile_data.py b/zerver/tests/test_custom_profile_data.py index 8de09fa5b1..a8bb50007d 100644 --- a/zerver/tests/test_custom_profile_data.py +++ b/zerver/tests/test_custom_profile_data.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Union from unittest import mock -import ujson +import orjson from zerver.lib.actions import ( do_remove_realm_custom_profile_field, @@ -85,49 +85,49 @@ class CreateCustomProfileFieldTest(CustomProfileFieldTestCase): error_msg = "Bad value for 'field_data': invalid" self.assert_json_error(result, error_msg) - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'python': ['1'], 'java': ['2'], - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data is not a dict') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'python': {'text': 'Python'}, 'java': {'text': 'Java'}, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "order key is missing from field_data") - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'python': {'text': 'Python', 'order': ''}, 'java': {'text': 'Java', 'order': '2'}, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["order"] cannot be blank.') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ '': {'text': 'Python', 'order': '1'}, 'java': {'text': 'Java', 'order': '2'}, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "'value' cannot be blank.") - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'python': {'text': 'Python', 'order': 1}, 'java': {'text': 'Java', 'order': '2'}, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["order"] is not a string') - data["field_data"] = ujson.dumps({}) + data["field_data"] = orjson.dumps({}).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Field must have at least one choice.') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'python': {'text': 'Python', 'order': '1'}, 'java': {'text': 'Java', 'order': '2'}, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_success(result) @@ -135,9 +135,9 @@ class CreateCustomProfileFieldTest(CustomProfileFieldTestCase): self.login('iago') realm = get_realm("zulip") field_type: int = CustomProfileField.EXTERNAL_ACCOUNT - field_data: str = ujson.dumps({ + field_data: str = orjson.dumps({ 'subtype': 'twitter', - }) + }).decode() invalid_field_name: str = "Not required field name" invalid_field_hint: str = "Not required field hint" @@ -189,86 +189,86 @@ class CreateCustomProfileFieldTest(CustomProfileFieldTestCase): result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "Bad value for 'field_data': invalid") - data['field_data'] = ujson.dumps({}) + data['field_data'] = orjson.dumps({}).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "subtype key is missing from field_data") - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': '', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["subtype"] cannot be blank.') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': '123', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Invalid external account type') non_default_external_account = 'linkedin' - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': non_default_external_account, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Invalid external account type') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'twitter', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_success(result) twitter_field = CustomProfileField.objects.get(name="Twitter", realm=realm) self.assertEqual(twitter_field.field_type, CustomProfileField.EXTERNAL_ACCOUNT) self.assertEqual(twitter_field.name, "Twitter") - self.assertEqual(ujson.loads(twitter_field.field_data)['subtype'], 'twitter') + self.assertEqual(orjson.loads(twitter_field.field_data)['subtype'], 'twitter') data['name'] = 'Reddit' - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'custom', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Custom external account must define url pattern') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 123, - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["url_pattern"] is not a string') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'invalid', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Malformed URL pattern.') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'https://www.reddit.com/%(username)s/user/%(username)s', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Malformed URL pattern.') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'reddit.com/%(username)s', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["url_pattern"] is not a URL') - data["field_data"] = ujson.dumps({ + data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'https://www.reddit.com/user/%(username)s', - }) + }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_success(result) custom_field = CustomProfileField.objects.get(name="Reddit", realm=realm) self.assertEqual(custom_field.field_type, CustomProfileField.EXTERNAL_ACCOUNT) self.assertEqual(custom_field.name, "Reddit") - field_data = ujson.loads(custom_field.field_data) + field_data = orjson.loads(custom_field.field_data) self.assertEqual(field_data['subtype'], 'custom') self.assertEqual(field_data['url_pattern'], 'https://www.reddit.com/user/%(username)s') @@ -311,7 +311,7 @@ class DeleteCustomProfileFieldTest(CustomProfileFieldTestCase): invalid_field_id = 1234 result = self.client_delete("/json/users/me/profile_data", { - 'data': ujson.dumps([invalid_field_id]), + 'data': orjson.dumps([invalid_field_id]).decode(), }) self.assert_json_error(result, f'Field id {invalid_field_id} not found.') @@ -327,13 +327,13 @@ class DeleteCustomProfileFieldTest(CustomProfileFieldTestCase): self.assertEqual([self.example_user("aaron").id], converter(iago_value.value)) result = self.client_delete("/json/users/me/profile_data", { - 'data': ujson.dumps([field.id]), + 'data': orjson.dumps([field.id]).decode(), }) self.assert_json_success(result) # Don't throw an exception here result = self.client_delete("/json/users/me/profile_data", { - 'data': ujson.dumps([field.id]), + 'data': orjson.dumps([field.id]).decode(), }) self.assert_json_success(result) @@ -420,21 +420,21 @@ class UpdateCustomProfileFieldTest(CustomProfileFieldTestCase): 'field_data': 'invalid'}) self.assert_json_error(result, "Bad value for 'field_data': invalid") - field_data = ujson.dumps({ + field_data = orjson.dumps({ 'vim': 'Vim', 'emacs': {'order': '2', 'text': 'Emacs'}, - }) + }).decode() result = self.client_patch( f"/json/realm/profile_fields/{field.id}", info={'name': 'Favorite editor', 'field_data': field_data}) self.assert_json_error(result, "field_data is not a dict") - field_data = ujson.dumps({ + field_data = orjson.dumps({ 'vim': {'order': '1', 'text': 'Vim'}, 'emacs': {'order': '2', 'text': 'Emacs'}, 'notepad': {'order': '3', 'text': 'Notepad'}, - }) + }).decode() result = self.client_patch( f"/json/realm/profile_fields/{field.id}", info={'name': 'Favorite editor', @@ -465,14 +465,14 @@ class UpdateCustomProfileFieldTest(CustomProfileFieldTestCase): # Update value of field result = self.client_patch("/json/users/me/profile_data", - {'data': ujson.dumps([{"id": field.id, "value": new_value}])}) + {'data': orjson.dumps([{"id": field.id, "value": new_value}]).decode()}) self.assert_json_error(result, error_msg) def test_update_invalid_field(self) -> None: self.login('iago') data = [{'id': 1234, 'value': '12'}] result = self.client_patch("/json/users/me/profile_data", { - 'data': ujson.dumps(data), + 'data': orjson.dumps(data).decode(), }) self.assert_json_error(result, "Field id 1234 not found.") @@ -527,7 +527,7 @@ class UpdateCustomProfileFieldTest(CustomProfileFieldTestCase): # Update value of field result = self.client_patch( "/json/users/me/profile_data", - {"data": ujson.dumps([{"id": f["id"], "value": f["value"]} for f in data])}, + {"data": orjson.dumps([{"id": f["id"], "value": f["value"]} for f in data]).decode()}, ) self.assert_json_success(result) @@ -554,7 +554,7 @@ class UpdateCustomProfileFieldTest(CustomProfileFieldTestCase): }] result = self.client_patch("/json/users/me/profile_data", - {'data': ujson.dumps(data)}) + {'data': orjson.dumps(data).decode()}) self.assert_json_success(result) for field_dict in iago.profile_data: if field_dict['id'] == field.id: @@ -575,7 +575,7 @@ class UpdateCustomProfileFieldTest(CustomProfileFieldTestCase): }] result = self.client_patch("/json/users/me/profile_data", - {'data': ujson.dumps(data)}) + {'data': orjson.dumps(data).decode()}) self.assert_json_success(result) def test_null_value_and_rendered_value(self) -> None: @@ -727,7 +727,7 @@ class ReorderCustomProfileFieldTest(CustomProfileFieldTestCase): .values_list('order', flat=True) ) result = self.client_patch("/json/realm/profile_fields", - info={'order': ujson.dumps(order)}) + info={'order': orjson.dumps(order).decode()}) self.assert_json_success(result) fields = CustomProfileField.objects.filter(realm=realm).order_by('order') for field in fields: @@ -743,7 +743,7 @@ class ReorderCustomProfileFieldTest(CustomProfileFieldTestCase): ) order.append(4) result = self.client_patch("/json/realm/profile_fields", - info={'order': ujson.dumps(order)}) + info={'order': orjson.dumps(order).decode()}) self.assert_json_success(result) fields = CustomProfileField.objects.filter(realm=realm).order_by('order') for field in fields: @@ -758,18 +758,18 @@ class ReorderCustomProfileFieldTest(CustomProfileFieldTestCase): .values_list('order', flat=True) ) result = self.client_patch("/json/realm/profile_fields", - info={'order': ujson.dumps(order)}) + info={'order': orjson.dumps(order).decode()}) self.assert_json_error(result, "Must be an organization administrator") def test_reorder_invalid(self) -> None: self.login('iago') order = [100, 200, 300] result = self.client_patch("/json/realm/profile_fields", - info={'order': ujson.dumps(order)}) + info={'order': orjson.dumps(order).decode()}) self.assert_json_error( result, 'Invalid order mapping.') order = [1, 2] result = self.client_patch("/json/realm/profile_fields", - info={'order': ujson.dumps(order)}) + info={'order': orjson.dumps(order).decode()}) self.assert_json_error( result, 'Invalid order mapping.') diff --git a/zerver/tests/test_decorators.py b/zerver/tests/test_decorators.py index 9b3f8530d5..d24200f78e 100644 --- a/zerver/tests/test_decorators.py +++ b/zerver/tests/test_decorators.py @@ -5,7 +5,7 @@ from collections import defaultdict from typing import Any, Dict, Iterable, List, Tuple from unittest import mock -import ujson +import orjson from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse @@ -141,7 +141,7 @@ class DecoratorTestCase(ZulipTestCase): def test_REQ_converter(self) -> None: def my_converter(data: str) -> List[int]: - lst = ujson.loads(data) + lst = orjson.loads(data) if not isinstance(lst, list): raise ValueError('not a list') if 13 in lst: @@ -166,17 +166,17 @@ class DecoratorTestCase(ZulipTestCase): get_total(request) self.assertEqual(str(cm.exception), "Bad value for 'numbers': bad_value") - request.POST['numbers'] = ujson.dumps('{fun: unfun}') + request.POST['numbers'] = orjson.dumps('{fun: unfun}').decode() with self.assertRaises(JsonableError) as cm: get_total(request) self.assertEqual(str(cm.exception), 'Bad value for \'numbers\': "{fun: unfun}"') - request.POST['numbers'] = ujson.dumps([2, 3, 5, 8, 13, 21]) + request.POST['numbers'] = orjson.dumps([2, 3, 5, 8, 13, 21]).decode() with self.assertRaises(JsonableError) as cm: get_total(request) self.assertEqual(str(cm.exception), "13 is an unlucky number!") - request.POST['numbers'] = ujson.dumps([1, 2, 3, 4, 5, 6]) + request.POST['numbers'] = orjson.dumps([1, 2, 3, 4, 5, 6]).decode() result = get_total(request) self.assertEqual(result, 21) @@ -201,12 +201,12 @@ class DecoratorTestCase(ZulipTestCase): get_total(request) self.assertEqual(str(cm.exception), 'Argument "numbers" is not valid JSON.') - request.POST['numbers'] = ujson.dumps([1, 2, "what?", 4, 5, 6]) + request.POST['numbers'] = orjson.dumps([1, 2, "what?", 4, 5, 6]).decode() with self.assertRaises(JsonableError) as cm: get_total(request) self.assertEqual(str(cm.exception), 'numbers[2] is not an integer') - request.POST['numbers'] = ujson.dumps([1, 2, 3, 4, 5, 6]) + request.POST['numbers'] = orjson.dumps([1, 2, 3, 4, 5, 6]).decode() result = get_total(request) self.assertEqual(result, 21) @@ -1711,7 +1711,7 @@ class ReturnSuccessOnHeadRequestDecorator(ZulipTestCase): response = test_function(request) self.assert_json_success(response) - self.assertNotEqual(ujson.loads(response.content).get('msg'), 'from_test_function') + self.assertNotEqual(orjson.loads(response.content).get('msg'), 'from_test_function') def test_returns_normal_response_if_request_method_is_not_head(self) -> None: class HeadRequest: @@ -1724,7 +1724,7 @@ class ReturnSuccessOnHeadRequestDecorator(ZulipTestCase): return json_response(msg='from_test_function') response = test_function(request) - self.assertEqual(ujson.loads(response.content).get('msg'), 'from_test_function') + self.assertEqual(orjson.loads(response.content).get('msg'), 'from_test_function') class RestAPITest(ZulipTestCase): def test_method_not_allowed(self) -> None: diff --git a/zerver/tests/test_docs.py b/zerver/tests/test_docs.py index f4897362ef..f308ad90f3 100644 --- a/zerver/tests/test_docs.py +++ b/zerver/tests/test_docs.py @@ -3,7 +3,7 @@ from typing import Any, Dict, Sequence from unittest import mock from urllib.parse import urlsplit -import ujson +import orjson from django.conf import settings from django.http import HttpResponse from django.test import override_settings @@ -28,7 +28,7 @@ class DocPageTest(ZulipTestCase): return print("Error processing URL:", url) if response.get('Content-Type') == 'application/json': - content = ujson.loads(response.content) + content = orjson.loads(response.content) print() print("======================================================================") print("ERROR: {}".format(content.get('msg'))) diff --git a/zerver/tests/test_drafts.py b/zerver/tests/test_drafts.py index cfd4127ab8..19e65ea1d3 100644 --- a/zerver/tests/test_drafts.py +++ b/zerver/tests/test_drafts.py @@ -2,7 +2,7 @@ import time from copy import deepcopy from typing import Any, Dict, List -import ujson +import orjson from zerver.lib.test_classes import ZulipTestCase from zerver.models import Draft @@ -18,7 +18,7 @@ class DraftCreationTests(ZulipTestCase): self.assertEqual(Draft.objects.count(), 0) # Now send a POST request to the API endpoint. - payload = {"drafts": ujson.dumps(draft_dicts)} + payload = {"drafts": orjson.dumps(draft_dicts).decode()} resp = self.api_post(hamlet, "/api/v1/drafts", payload) self.assert_json_success(resp) @@ -35,7 +35,7 @@ class DraftCreationTests(ZulipTestCase): self.assertEqual(Draft.objects.count(), 0) # Now send a POST request to the API endpoint. - payload = {"drafts": ujson.dumps(draft_dicts)} + payload = {"drafts": orjson.dumps(draft_dicts).decode()} resp = self.api_post(hamlet, "/api/v1/drafts", payload) self.assert_json_error(resp, expected_message) @@ -172,7 +172,7 @@ class DraftCreationTests(ZulipTestCase): self.assertEqual(Draft.objects.count(), 0) current_time = round(time.time(), 6) - payload = {"drafts": ujson.dumps(draft_dicts)} + payload = {"drafts": orjson.dumps(draft_dicts).decode()} resp = self.api_post(hamlet, "/api/v1/drafts", payload) self.assert_json_success(resp) @@ -321,9 +321,9 @@ class DraftEditTests(ZulipTestCase): "content": "The API should be good", "timestamp": 1595505700.85247 } - resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": ujson.dumps([draft_dict])}) + resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": orjson.dumps([draft_dict]).decode()}) self.assert_json_success(resp) - new_draft_id = ujson.loads(resp.content)["ids"][0] + new_draft_id = orjson.loads(resp.content)["ids"][0] # Change the draft data. draft_dict["content"] = "The API needs to be structured yet simple to use." @@ -333,7 +333,7 @@ class DraftEditTests(ZulipTestCase): # Update this change in the backend. resp = self.api_patch(hamlet, f"/api/v1/drafts/{new_draft_id}", - {"draft": ujson.dumps(draft_dict)}) + {"draft": orjson.dumps(draft_dict).decode()}) self.assert_json_success(resp) # Now make sure that the change was made successfully. @@ -356,7 +356,7 @@ class DraftEditTests(ZulipTestCase): "timestamp": 1595505700.85247 } resp = self.api_patch(hamlet, "/api/v1/drafts/999999999", - {"draft": ujson.dumps(draft_dict)}) + {"draft": orjson.dumps(draft_dict).decode()}) self.assert_json_error(resp, "Draft does not exist", status_code=404) # Now make sure that no changes were made. @@ -378,9 +378,9 @@ class DraftEditTests(ZulipTestCase): "content": "The API should be good", "timestamp": 1595505700.85247 } - resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": ujson.dumps([draft_dict])}) + resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": orjson.dumps([draft_dict]).decode()}) self.assert_json_success(resp) - new_draft_id = ujson.loads(resp.content)["ids"][0] + new_draft_id = orjson.loads(resp.content)["ids"][0] # Change the draft data. modified_draft_dict = deepcopy(draft_dict) @@ -389,7 +389,7 @@ class DraftEditTests(ZulipTestCase): # Update this change in the backend as a different user. zoe = self.example_user("ZOE") resp = self.api_patch(zoe, f"/api/v1/drafts/{new_draft_id}", - {"draft": ujson.dumps(draft_dict)}) + {"draft": orjson.dumps(draft_dict).decode()}) self.assert_json_error(resp, "Draft does not exist", status_code=404) # Now make sure that no changes were made. @@ -414,9 +414,9 @@ class DraftDeleteTests(ZulipTestCase): "content": "The API should be good", "timestamp": 1595505700.85247 } - resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": ujson.dumps([draft_dict])}) + resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": orjson.dumps([draft_dict]).decode()}) self.assert_json_success(resp) - new_draft_id = ujson.loads(resp.content)["ids"][0] + new_draft_id = orjson.loads(resp.content)["ids"][0] # Make sure that exactly 1 draft exists now. self.assertEqual(Draft.objects.count(), 1) @@ -457,9 +457,9 @@ class DraftDeleteTests(ZulipTestCase): "content": "The API should be good", "timestamp": 1595505700.85247 } - resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": ujson.dumps([draft_dict])}) + resp = self.api_post(hamlet, "/api/v1/drafts", {"drafts": orjson.dumps([draft_dict]).decode()}) self.assert_json_success(resp) - new_draft_id = ujson.loads(resp.content)["ids"][0] + new_draft_id = orjson.loads(resp.content)["ids"][0] # Delete this draft in the backend as a different user. zoe = self.example_user("ZOE") @@ -505,7 +505,7 @@ class DraftFetchTest(ZulipTestCase): "timestamp": 1595479021.439161, }, ] - payload = {"drafts": ujson.dumps(draft_dicts)} + payload = {"drafts": orjson.dumps(draft_dicts).decode()} resp = self.api_post(hamlet, "/api/v1/drafts", payload) self.assert_json_success(resp) @@ -520,7 +520,7 @@ class DraftFetchTest(ZulipTestCase): "timestamp": 1595479019.439159, }, ] - payload = {"drafts": ujson.dumps(zoe_draft_dicts)} + payload = {"drafts": orjson.dumps(zoe_draft_dicts).decode()} resp = self.api_post(zoe, "/api/v1/drafts", payload) self.assert_json_success(resp) @@ -530,7 +530,7 @@ class DraftFetchTest(ZulipTestCase): # his drafts and exactly as he made them. resp = self.api_get(hamlet, "/api/v1/drafts") self.assert_json_success(resp) - data = ujson.loads(resp.content) + data = orjson.loads(resp.content) self.assertEqual(data["count"], 3) first_draft_id = Draft.objects.order_by("id")[0].id diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py index ecb0aae4df..5cf9b4b50a 100644 --- a/zerver/tests/test_email_mirror.py +++ b/zerver/tests/test_email_mirror.py @@ -7,7 +7,7 @@ from email.message import EmailMessage, MIMEPart from typing import Any, Callable, Dict, Mapping, Optional from unittest import mock -import ujson +import orjson from django.conf import settings from django.http import HttpResponse @@ -730,7 +730,7 @@ class TestMissedMessageEmailMessages(ZulipTestCase): result = self.client_post("/json/messages", {"type": "private", "content": "test_receive_missed_message_email_messages", "client": "test suite", - "to": ujson.dumps([othello.id])}) + "to": orjson.dumps([othello.id]).decode()}) self.assert_json_success(result) user_profile = self.example_user('othello') @@ -770,7 +770,7 @@ class TestMissedMessageEmailMessages(ZulipTestCase): result = self.client_post("/json/messages", {"type": "private", "content": "test_receive_missed_message_email_messages", "client": "test suite", - "to": ujson.dumps([cordelia.id, iago.id])}) + "to": orjson.dumps([cordelia.id, iago.id]).decode()}) self.assert_json_success(result) user_profile = self.example_user('cordelia') @@ -990,7 +990,7 @@ class TestEmptyGatewaySetting(ZulipTestCase): type="private", content="test_receive_missed_message_email_messages", client="test suite", - to=ujson.dumps([cordelia.id, iago.id]), + to=orjson.dumps([cordelia.id, iago.id]).decode(), ) result = self.client_post("/json/messages", payload) self.assert_json_success(result) @@ -1166,7 +1166,7 @@ class TestEmailMirrorTornadoView(ZulipTestCase): "type": "private", "content": "test_receive_missed_message_email_messages", "client": "test suite", - "to": ujson.dumps([cordelia.id, iago.id]), + "to": orjson.dumps([cordelia.id, iago.id]).decode(), }) self.assert_json_success(result) @@ -1277,7 +1277,7 @@ class TestStreamEmailMessagesSubjectStripping(ZulipTestCase): self.assertEqual("(no topic)", message.topic_name()) def test_strip_from_subject(self) -> None: - subject_list = ujson.loads(self.fixture_data('subjects.json', type='email')) + subject_list = orjson.loads(self.fixture_data('subjects.json', type='email')) for subject in subject_list: stripped = strip_from_subject(subject['original_subject']) self.assertEqual(stripped, subject['stripped_subject']) @@ -1414,7 +1414,7 @@ class TestEmailMirrorLogAndReport(ZulipTestCase): "type": "private", "content": "test_redact_email_message", "client": "test suite", - "to": ujson.dumps([cordelia.email, iago.email]), + "to": orjson.dumps([cordelia.email, iago.email]).decode(), }) self.assert_json_success(result) diff --git a/zerver/tests/test_email_notifications.py b/zerver/tests/test_email_notifications.py index 5be5015272..f6b8f64027 100644 --- a/zerver/tests/test_email_notifications.py +++ b/zerver/tests/test_email_notifications.py @@ -5,7 +5,7 @@ from typing import List, Sequence from unittest.mock import patch import ldap -import ujson +import orjson from django.conf import settings from django.core import mail from django.test import override_settings @@ -110,7 +110,7 @@ class TestFollowupEmails(ZulipTestCase): hamlet = self.example_user("hamlet") enqueue_welcome_emails(hamlet) scheduled_emails = ScheduledEmail.objects.filter(users=hamlet) - email_data = ujson.loads(scheduled_emails[0].data) + email_data = orjson.loads(scheduled_emails[0].data) self.assertEqual(email_data["context"]["email"], self.example_email("hamlet")) self.assertEqual(email_data["context"]["is_realm_admin"], False) self.assertEqual(email_data["context"]["getting_started_link"], "https://zulip.com") @@ -121,7 +121,7 @@ class TestFollowupEmails(ZulipTestCase): iago = self.example_user("iago") enqueue_welcome_emails(iago) scheduled_emails = ScheduledEmail.objects.filter(users=iago) - email_data = ujson.loads(scheduled_emails[0].data) + email_data = orjson.loads(scheduled_emails[0].data) self.assertEqual(email_data["context"]["email"], self.example_email("iago")) self.assertEqual(email_data["context"]["is_realm_admin"], True) self.assertEqual(email_data["context"]["getting_started_link"], @@ -147,7 +147,7 @@ class TestFollowupEmails(ZulipTestCase): scheduled_emails = ScheduledEmail.objects.filter(users=user) self.assertEqual(len(scheduled_emails), 2) - email_data = ujson.loads(scheduled_emails[0].data) + email_data = orjson.loads(scheduled_emails[0].data) self.assertEqual(email_data["context"]["ldap"], True) self.assertEqual(email_data["context"]["ldap_username"], "newuser_email_as_uid@zulip.com") @@ -167,7 +167,7 @@ class TestFollowupEmails(ZulipTestCase): scheduled_emails = ScheduledEmail.objects.filter(users=user) self.assertEqual(len(scheduled_emails), 2) - email_data = ujson.loads(scheduled_emails[0].data) + email_data = orjson.loads(scheduled_emails[0].data) self.assertEqual(email_data["context"]["ldap"], True) self.assertEqual(email_data["context"]["ldap_username"], "newuser") @@ -186,7 +186,7 @@ class TestFollowupEmails(ZulipTestCase): scheduled_emails = ScheduledEmail.objects.filter(users=user) self.assertEqual(len(scheduled_emails), 2) - email_data = ujson.loads(scheduled_emails[0].data) + email_data = orjson.loads(scheduled_emails[0].data) self.assertEqual(email_data["context"]["ldap"], True) self.assertEqual(email_data["context"]["ldap_username"], "newuser_with_email") @@ -199,8 +199,8 @@ class TestFollowupEmails(ZulipTestCase): scheduled_emails = ScheduledEmail.objects.filter(users=hamlet).order_by( "scheduled_timestamp") self.assertEqual(2, len(scheduled_emails)) - self.assertEqual(ujson.loads(scheduled_emails[1].data)["template_prefix"], 'zerver/emails/followup_day2') - self.assertEqual(ujson.loads(scheduled_emails[0].data)["template_prefix"], 'zerver/emails/followup_day1') + self.assertEqual(orjson.loads(scheduled_emails[1].data)["template_prefix"], 'zerver/emails/followup_day2') + self.assertEqual(orjson.loads(scheduled_emails[0].data)["template_prefix"], 'zerver/emails/followup_day1') ScheduledEmail.objects.all().delete() @@ -208,7 +208,7 @@ class TestFollowupEmails(ZulipTestCase): scheduled_emails = ScheduledEmail.objects.filter(users=cordelia) # Cordelia has account in more than 1 realm so day2 email should not be sent self.assertEqual(len(scheduled_emails), 1) - email_data = ujson.loads(scheduled_emails[0].data) + email_data = orjson.loads(scheduled_emails[0].data) self.assertEqual(email_data["template_prefix"], 'zerver/emails/followup_day1') class TestMissedMessages(ZulipTestCase): @@ -875,7 +875,7 @@ class TestMissedMessages(ZulipTestCase): # Run `relative_to_full_url()` function over test fixtures present in # 'markdown_test_cases.json' and check that it converts all the relative # URLs to absolute URLs. - fixtures = ujson.loads(self.fixture_data("markdown_test_cases.json")) + fixtures = orjson.loads(self.fixture_data("markdown_test_cases.json")) test_fixtures = {} for test in fixtures['regular_tests']: test_fixtures[test['name']] = test @@ -949,7 +949,7 @@ class TestMissedMessages(ZulipTestCase): self.assertEqual(actual_output, expected_output) # test against our markdown_test_cases so these features do not get out of sync. - fixtures = ujson.loads(self.fixture_data("markdown_test_cases.json")) + fixtures = orjson.loads(self.fixture_data("markdown_test_cases.json")) test_fixtures = {} for test in fixtures['regular_tests']: if 'spoiler' in test['name']: diff --git a/zerver/tests/test_embedded_bot_system.py b/zerver/tests/test_embedded_bot_system.py index 0ad5c9beb4..41d6d02148 100644 --- a/zerver/tests/test_embedded_bot_system.py +++ b/zerver/tests/test_embedded_bot_system.py @@ -1,6 +1,6 @@ from unittest.mock import patch -import ujson +import orjson from zerver.lib.bot_lib import EmbeddedBotQuitException from zerver.lib.test_classes import ZulipTestCase @@ -21,7 +21,7 @@ class TestEmbeddedBotMessaging(ZulipTestCase): full_name='Embedded bot', bot_type=UserProfile.EMBEDDED_BOT, service_name='helloworld', - config_data=ujson.dumps({'foo': 'bar'})) + config_data=orjson.dumps({'foo': 'bar'}).decode()) def test_pm_to_embedded_bot(self) -> None: assert self.bot_profile is not None diff --git a/zerver/tests/test_event_queue.py b/zerver/tests/test_event_queue.py index 1aeebb18ad..1149e64b21 100644 --- a/zerver/tests/test_event_queue.py +++ b/zerver/tests/test_event_queue.py @@ -2,7 +2,7 @@ import time from typing import Any, Callable, Dict, List, Tuple from unittest import mock -import ujson +import orjson from django.http import HttpRequest, HttpResponse from zerver.lib.actions import do_change_subscription_property, do_mute_topic @@ -242,13 +242,13 @@ class MissedMessageNotificationsTest(ZulipTestCase): def allocate_event_queue() -> ClientDescriptor: result = self.tornado_call(get_events, user_profile, - {"apply_markdown": ujson.dumps(True), - "client_gravatar": ujson.dumps(True), - "event_types": ujson.dumps(["message"]), + {"apply_markdown": orjson.dumps(True).decode(), + "client_gravatar": orjson.dumps(True).decode(), + "event_types": orjson.dumps(["message"]).decode(), "user_client": "website", - "dont_block": ujson.dumps(True)}) + "dont_block": orjson.dumps(True).decode()}) self.assert_json_success(result) - queue_id = ujson.loads(result.content)["queue_id"] + queue_id = orjson.loads(result.content)["queue_id"] return get_client_descriptor(queue_id) def destroy_event_queue(queue_id: str) -> None: diff --git a/zerver/tests/test_event_system.py b/zerver/tests/test_event_system.py index 9c31e63182..93f45055d3 100644 --- a/zerver/tests/test_event_system.py +++ b/zerver/tests/test_event_system.py @@ -2,7 +2,7 @@ import time from typing import Any, Callable, Dict, List from unittest import mock -import ujson +import orjson from django.conf import settings from django.http import HttpRequest, HttpResponse @@ -60,11 +60,11 @@ class EventsEndpointTest(ZulipTestCase): # Test that call is made to deal with a returning soft deactivated user. with mock.patch('zerver.lib.events.reactivate_user_if_soft_deactivated') as fa: with stub_event_queue_user_events(return_event_queue, return_user_events): - result = self.api_post(user, '/json/register', dict(event_types=ujson.dumps([event_type]))) + result = self.api_post(user, '/json/register', dict(event_types=orjson.dumps([event_type]).decode())) self.assertEqual(fa.call_count, 1) with stub_event_queue_user_events(return_event_queue, return_user_events): - result = self.api_post(user, '/json/register', dict(event_types=ujson.dumps([event_type]))) + result = self.api_post(user, '/json/register', dict(event_types=orjson.dumps([event_type]).decode())) self.assert_json_success(result) result_dict = result.json() @@ -76,7 +76,7 @@ class EventsEndpointTest(ZulipTestCase): return_user_events = [test_event] with stub_event_queue_user_events(return_event_queue, return_user_events): - result = self.api_post(user, '/json/register', dict(event_types=ujson.dumps([event_type]))) + result = self.api_post(user, '/json/register', dict(event_types=orjson.dumps([event_type]).decode())) self.assert_json_success(result) result_dict = result.json() @@ -90,8 +90,8 @@ class EventsEndpointTest(ZulipTestCase): return_event_queue = '15:13' with stub_event_queue_user_events(return_event_queue, return_user_events): result = self.api_post(user, '/json/register', - dict(event_types=ujson.dumps([event_type]), - fetch_event_types=ujson.dumps(['message']))) + dict(event_types=orjson.dumps([event_type]).decode(), + fetch_event_types=orjson.dumps(['message']).decode())) self.assert_json_success(result) result_dict = result.json() self.assertEqual(result_dict['last_event_id'], 6) @@ -105,8 +105,8 @@ class EventsEndpointTest(ZulipTestCase): # Now test with `fetch_event_types` matching the event with stub_event_queue_user_events(return_event_queue, return_user_events): result = self.api_post(user, '/json/register', - dict(fetch_event_types=ujson.dumps([event_type]), - event_types=ujson.dumps(['message']))) + dict(fetch_event_types=orjson.dumps([event_type]).decode(), + event_types=orjson.dumps(['message']).decode())) self.assert_json_success(result) result_dict = result.json() self.assertEqual(result_dict['last_event_id'], 6) @@ -124,14 +124,14 @@ class EventsEndpointTest(ZulipTestCase): # the /notify_tornado endpoint, so we can have 100% URL coverage, # but it does exercise a little bit of the codepath. post_data = dict( - data=ujson.dumps( + data=orjson.dumps( dict( event=dict( type='other', ), users=[self.example_user('hamlet').id], ), - ), + ).decode(), ) req = POSTRequestMock(post_data, user_profile=None) req.META['REMOTE_ADDR'] = '127.0.0.1' @@ -159,32 +159,32 @@ class GetEventsTest(ZulipTestCase): self.login_user(user_profile) result = self.tornado_call(get_events, user_profile, - {"apply_markdown": ujson.dumps(True), - "client_gravatar": ujson.dumps(True), - "event_types": ujson.dumps(["message"]), + {"apply_markdown": orjson.dumps(True).decode(), + "client_gravatar": orjson.dumps(True).decode(), + "event_types": orjson.dumps(["message"]).decode(), "user_client": "website", - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) self.assert_json_success(result) - queue_id = ujson.loads(result.content)["queue_id"] + queue_id = orjson.loads(result.content)["queue_id"] recipient_result = self.tornado_call(get_events, recipient_user_profile, - {"apply_markdown": ujson.dumps(True), - "client_gravatar": ujson.dumps(True), - "event_types": ujson.dumps(["message"]), + {"apply_markdown": orjson.dumps(True).decode(), + "client_gravatar": orjson.dumps(True).decode(), + "event_types": orjson.dumps(["message"]).decode(), "user_client": "website", - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) self.assert_json_success(recipient_result) - recipient_queue_id = ujson.loads(recipient_result.content)["queue_id"] + recipient_queue_id = orjson.loads(recipient_result.content)["queue_id"] result = self.tornado_call(get_events, user_profile, {"queue_id": queue_id, "user_client": "website", "last_event_id": -1, - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) - events = ujson.loads(result.content)["events"] + events = orjson.loads(result.content)["events"] self.assert_json_success(result) self.assert_length(events, 0) @@ -204,9 +204,9 @@ class GetEventsTest(ZulipTestCase): {"queue_id": queue_id, "user_client": "website", "last_event_id": -1, - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) - events = ujson.loads(result.content)["events"] + events = orjson.loads(result.content)["events"] self.assert_json_success(result) self.assert_length(events, 1) self.assertEqual(events[0]["type"], "message") @@ -233,9 +233,9 @@ class GetEventsTest(ZulipTestCase): {"queue_id": queue_id, "user_client": "website", "last_event_id": last_event_id, - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) - events = ujson.loads(result.content)["events"] + events = orjson.loads(result.content)["events"] self.assert_json_success(result) self.assert_length(events, 1) self.assertEqual(events[0]["type"], "message") @@ -248,9 +248,9 @@ class GetEventsTest(ZulipTestCase): {"queue_id": recipient_queue_id, "user_client": "website", "last_event_id": -1, - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) - recipient_events = ujson.loads(recipient_result.content)["events"] + recipient_events = orjson.loads(recipient_result.content)["events"] self.assert_json_success(recipient_result) self.assertEqual(len(recipient_events), 2) self.assertEqual(recipient_events[0]["type"], "message") @@ -269,25 +269,25 @@ class GetEventsTest(ZulipTestCase): get_events, user_profile, dict( - apply_markdown=ujson.dumps(apply_markdown), - client_gravatar=ujson.dumps(client_gravatar), - event_types=ujson.dumps(["message"]), - narrow=ujson.dumps([["stream", "denmark"]]), + apply_markdown=orjson.dumps(apply_markdown).decode(), + client_gravatar=orjson.dumps(client_gravatar).decode(), + event_types=orjson.dumps(["message"]).decode(), + narrow=orjson.dumps([["stream", "denmark"]]).decode(), user_client="website", - dont_block=ujson.dumps(True), + dont_block=orjson.dumps(True).decode(), ), ) self.assert_json_success(result) - queue_id = ujson.loads(result.content)["queue_id"] + queue_id = orjson.loads(result.content)["queue_id"] result = self.tornado_call(get_events, user_profile, {"queue_id": queue_id, "user_client": "website", "last_event_id": -1, - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) - events = ujson.loads(result.content)["events"] + events = orjson.loads(result.content)["events"] self.assert_json_success(result) self.assert_length(events, 0) @@ -298,9 +298,9 @@ class GetEventsTest(ZulipTestCase): {"queue_id": queue_id, "user_client": "website", "last_event_id": -1, - "dont_block": ujson.dumps(True), + "dont_block": orjson.dumps(True).decode(), }) - events = ujson.loads(result.content)["events"] + events = orjson.loads(result.content)["events"] self.assert_json_success(result) self.assert_length(events, 1) self.assertEqual(events[0]["type"], "message") diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 402c1f53d8..a2e831683f 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -7,7 +7,7 @@ from io import StringIO from typing import Any, Callable, Dict, List, Optional, Set from unittest import mock -import ujson +import orjson from django.utils.timezone import now as timezone_now from zerver.lib.actions import ( @@ -224,9 +224,9 @@ class BaseAction(ZulipTestCase): events = client.event_queue.contents() content = { 'queue_id': '123.12', - # The ujson wrapper helps in converting tuples to lists + # The JSON wrapper helps in converting tuples to lists # as tuples aren't valid JSON structure. - 'events': ujson.loads(ujson.dumps(copy.deepcopy(events))), + 'events': orjson.loads(orjson.dumps(events)), 'msg': '', 'result': 'success' } @@ -234,17 +234,17 @@ class BaseAction(ZulipTestCase): self.assertEqual(len(events), num_events) initial_state = copy.deepcopy(hybrid_state) post_process_state(self.user_profile, initial_state, notification_settings_null) - before = ujson.dumps(initial_state) + before = orjson.dumps(initial_state) apply_events(hybrid_state, events, self.user_profile, client_gravatar=client_gravatar, slim_presence=slim_presence, include_subscribers=include_subscribers) post_process_state(self.user_profile, hybrid_state, notification_settings_null) - after = ujson.dumps(hybrid_state) + after = orjson.dumps(hybrid_state) if state_change_expected: if before == after: # nocoverage - print(ujson.dumps(initial_state, indent=2)) + print(orjson.dumps(initial_state, option=orjson.OPT_INDENT_2).decode()) print(events) raise AssertionError('Test does not exercise enough code -- events do not change state.') else: @@ -1437,7 +1437,7 @@ class NormalActionsTest(BaseAction): action = lambda: self.create_bot('test_outgoing_webhook', full_name='Outgoing Webhook Bot', - payload_url=ujson.dumps('https://foo.bar.com'), + payload_url=orjson.dumps('https://foo.bar.com').decode(), interface_type=Service.GENERIC, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT) events = self.verify_action(action, num_events=2) @@ -1448,7 +1448,7 @@ class NormalActionsTest(BaseAction): action = lambda: self.create_bot('test_embedded', full_name='Embedded Bot', service_name='helloworld', - config_data=ujson.dumps({'foo': 'bar'}), + config_data=orjson.dumps({'foo': 'bar'}).decode(), bot_type=UserProfile.EMBEDDED_BOT) events = self.verify_action(action, num_events=2) check_realm_bot_add('events[1]', events[1]) @@ -1574,7 +1574,7 @@ class NormalActionsTest(BaseAction): bot = self.create_test_bot('test', self.user_profile, full_name='Test Bot', bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, - payload_url=ujson.dumps('http://hostname.domain2.com'), + payload_url=orjson.dumps('http://hostname.domain2.com').decode(), interface_type=Service.GENERIC, ) action = lambda: do_update_outgoing_webhook_service(bot, 2, 'http://hostname.domain2.com') @@ -2018,11 +2018,11 @@ class RealmPropertyActionTest(BaseAction): self.assertEqual(RealmAuditLog.objects.filter( realm=self.user_profile.realm, event_type=RealmAuditLog.REALM_PROPERTY_CHANGED, event_time__gte=now, acting_user=self.user_profile, - extra_data=ujson.dumps({ + extra_data=orjson.dumps({ RealmAuditLog.OLD_VALUE: old_value, RealmAuditLog.NEW_VALUE: val, 'property': name, - })).count(), 1) + }).decode()).count(), 1) check_realm_update('events[0]', events[0], name) def test_change_realm_property(self) -> None: diff --git a/zerver/tests/test_gitter_importer.py b/zerver/tests/test_gitter_importer.py index 6404cf37bd..adbd789b1d 100644 --- a/zerver/tests/test_gitter_importer.py +++ b/zerver/tests/test_gitter_importer.py @@ -2,7 +2,7 @@ import os from typing import Any from unittest import mock -import ujson +import orjson from zerver.data_import.gitter import do_convert_data, get_usermentions from zerver.lib.import_realm import do_import_realm @@ -21,8 +21,8 @@ class GitterImporter(ZulipTestCase): def read_file(output_file: str) -> Any: full_path = os.path.join(output_dir, output_file) - with open(full_path) as f: - return ujson.load(f) + with open(full_path, "rb") as f: + return orjson.loads(f.read()) self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')), True) self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')), True) diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index f1972165e0..d56d930c34 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -5,7 +5,7 @@ from typing import Any, Dict from unittest.mock import patch import lxml.html -import ujson +import orjson from django.conf import settings from django.http import HttpResponse from django.utils.timezone import now as timezone_now @@ -268,7 +268,7 @@ class HomeTest(ZulipTestCase): self.assertEqual(actual_keys, expected_keys) # TODO: Inspect the page_params data further. - # print(ujson.dumps(page_params, indent=2)) + # print(orjson.dumps(page_params, option=orjson.OPT_INDENT_2).decode()) realm_bots_expected_keys = [ 'api_key', 'avatar_url', @@ -366,7 +366,7 @@ class HomeTest(ZulipTestCase): doc = lxml.html.document_fromstring(result.content) [div] = doc.xpath("//div[@id='page-params']") page_params_json = div.get("data-params") - page_params = ujson.loads(page_params_json) + page_params = orjson.loads(page_params_json) return page_params def _sanity_check(self, result: HttpResponse) -> None: @@ -836,7 +836,7 @@ class HomeTest(ZulipTestCase): hamlet = self.example_user('hamlet') self.login_user(hamlet) self.client_post("/json/messages/flags", - {"messages": ujson.dumps([msg_id]), + {"messages": orjson.dumps([msg_id]).decode(), "op": "add", "flag": "read"}) diff --git a/zerver/tests/test_hotspots.py b/zerver/tests/test_hotspots.py index 13dec9866b..724cfcb1a2 100644 --- a/zerver/tests/test_hotspots.py +++ b/zerver/tests/test_hotspots.py @@ -1,4 +1,4 @@ -import ujson +import orjson from zerver.lib.actions import do_create_user, do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS, get_next_hotspots @@ -52,13 +52,13 @@ class TestHotspots(ZulipTestCase): user = self.example_user('hamlet') self.login_user(user) result = self.client_post('/json/users/me/hotspots', - {'hotspot': ujson.dumps('intro_reply')}) + {'hotspot': orjson.dumps('intro_reply').decode()}) self.assert_json_success(result) self.assertEqual(list(UserHotspot.objects.filter(user=user) .values_list('hotspot', flat=True)), ['intro_reply']) result = self.client_post('/json/users/me/hotspots', - {'hotspot': ujson.dumps('invalid')}) + {'hotspot': orjson.dumps('invalid').decode()}) self.assert_json_error(result, "Unknown hotspot: invalid") self.assertEqual(list(UserHotspot.objects.filter(user=user) .values_list('hotspot', flat=True)), ['intro_reply']) diff --git a/zerver/tests/test_i18n.py b/zerver/tests/test_i18n.py index ea61a49930..781a95e95d 100644 --- a/zerver/tests/test_i18n.py +++ b/zerver/tests/test_i18n.py @@ -2,7 +2,7 @@ from http.cookies import SimpleCookie from typing import Any from unittest import mock -import ujson +import orjson from django.conf import settings from django.core import mail from django.http import HttpResponse @@ -44,7 +44,7 @@ class EmailTranslationTestCase(ZulipTestCase): check_translation("Incrível!", "post", "/accounts/home/", {"email": "new-email@zulip.com"}, HTTP_ACCEPT_LANGUAGE="pt") check_translation("Danke, dass du", "post", '/accounts/find/', {'emails': hamlet.delivery_email}) check_translation("Hallo", "post", "/json/invites", {"invitee_emails": "new-email@zulip.com", - "stream_ids": ujson.dumps([stream.id])}) + "stream_ids": orjson.dumps([stream.id]).decode()}) with self.settings(DEVELOPMENT_LOG_EMAILS=True): enqueue_welcome_emails(hamlet) diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 3c45e4a244..3d521c51c1 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -2,7 +2,7 @@ import os from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple from unittest.mock import patch -import ujson +import orjson from django.conf import settings from django.db.models import Q from django.utils.timezone import now as timezone_now @@ -229,8 +229,8 @@ class ImportExportTest(ZulipTestCase): def read_file(fn: str) -> Any: full_fn = os.path.join(output_dir, fn) - with open(full_fn) as f: - return ujson.load(f) + with open(full_fn, "rb") as f: + return orjson.loads(f.read()) result = {} result['realm'] = read_file('realm.json') @@ -644,8 +644,8 @@ class ImportExportTest(ZulipTestCase): def read_file(fn: str) -> Any: full_fn = os.path.join(output_dir, fn) - with open(full_fn) as f: - return ujson.load(f) + with open(full_fn, "rb") as f: + return orjson.loads(f.read()) messages = read_file('messages-000001.json') user = read_file('user.json') @@ -809,7 +809,7 @@ class ImportExportTest(ZulipTestCase): return UserProfile.objects.get(id=user_id).email def get_email_from_value(field_value: CustomProfileFieldValue) -> Set[str]: - user_id_list = ujson.loads(field_value.value) + user_id_list = orjson.loads(field_value.value) return {get_email(user_id) for user_id in user_id_list} def custom_profile_field_values_for(fields: List[CustomProfileField]) -> Set[FrozenSet[str]]: diff --git a/zerver/tests/test_integrations_dev_panel.py b/zerver/tests/test_integrations_dev_panel.py index 1cb7984c9a..8aab019845 100644 --- a/zerver/tests/test_integrations_dev_panel.py +++ b/zerver/tests/test_integrations_dev_panel.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock, patch -import ujson +import orjson from zerver.lib.test_classes import ZulipTestCase from zerver.models import Message, Stream, get_realm, get_user @@ -27,7 +27,7 @@ class TestIntegrationsDevPanel(ZulipTestCase): self.assertEqual(response.status_code, 500) # Since the response would be forwarded. expected_response = {"result": "error", "msg": "Internal server error"} - self.assertEqual(ujson.loads(response.content), expected_response) + self.assertEqual(orjson.loads(response.content), expected_response) # Intention of this test looks like to trigger keyError # so just testing KeyError is printed along with Traceback in logs @@ -51,8 +51,8 @@ class TestIntegrationsDevPanel(ZulipTestCase): response = self.client_post(target_url, data) expected_response = {'responses': [{'status_code': 200, 'message': {"result": "success", "msg": ""}}], 'result': 'success', 'msg': ''} - response_content = ujson.loads(response.content) - response_content["responses"][0]["message"] = ujson.loads(response_content["responses"][0]["message"]) + response_content = orjson.loads(response.content) + response_content["responses"][0]["message"] = orjson.loads(response_content["responses"][0]["message"]) self.assertEqual(response.status_code, 200) self.assertEqual(response_content, expected_response) @@ -72,7 +72,7 @@ class TestIntegrationsDevPanel(ZulipTestCase): data = { "url": url, "body": body, - "custom_headers": ujson.dumps({"X_GITHUB_EVENT": "ping"}), + "custom_headers": orjson.dumps({"X_GITHUB_EVENT": "ping"}).decode(), "is_json": "true", } @@ -95,7 +95,7 @@ class TestIntegrationsDevPanel(ZulipTestCase): data = { "url": url, "body": body, - "custom_headers": ujson.dumps({"Content-Type": "application/x-www-form-urlencoded"}), + "custom_headers": orjson.dumps({"Content-Type": "application/x-www-form-urlencoded"}).decode(), "is_json": "false", } @@ -113,7 +113,7 @@ class TestIntegrationsDevPanel(ZulipTestCase): response = self.client_get(target_url) expected_response = {'msg': '"somerandomnonexistantintegration" is not a valid webhook integration.', 'result': 'error'} self.assertEqual(response.status_code, 404) - self.assertEqual(ujson.loads(response.content), expected_response) + self.assertEqual(orjson.loads(response.content), expected_response) @patch("zerver.views.development.integrations.os.path.exists") def test_get_fixtures_for_integration_without_fixtures(self, os_path_exists_mock: MagicMock) -> None: @@ -122,13 +122,13 @@ class TestIntegrationsDevPanel(ZulipTestCase): response = self.client_get(target_url) expected_response = {'msg': 'The integration "airbrake" does not have fixtures.', 'result': 'error'} self.assertEqual(response.status_code, 404) - self.assertEqual(ujson.loads(response.content), expected_response) + self.assertEqual(orjson.loads(response.content), expected_response) def test_get_fixtures_for_success(self) -> None: target_url = "/devtools/integrations/airbrake/fixtures" response = self.client_get(target_url) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(ujson.loads(response.content)["fixtures"]) + self.assertIsNotNone(orjson.loads(response.content)["fixtures"]) def test_get_dev_panel_page(self) -> None: # Just to satisfy the test suite. @@ -160,9 +160,9 @@ class TestIntegrationsDevPanel(ZulipTestCase): "message": {"msg": "", "result": "success"}, }, ] - responses = ujson.loads(response.content)["responses"] + responses = orjson.loads(response.content)["responses"] for r in responses: - r["message"] = ujson.loads(r["message"]) + r["message"] = orjson.loads(r["message"]) self.assertEqual(response.status_code, 200) for r in responses: # We have to use this roundabout manner since the order may vary each time. @@ -234,9 +234,9 @@ class TestIntegrationsDevPanel(ZulipTestCase): "status_code": 400, }, ] - responses = ujson.loads(response.content)["responses"] + responses = orjson.loads(response.content)["responses"] for r in responses: - r["message"] = ujson.loads(r["message"]) + r["message"] = orjson.loads(r["message"]) self.assertEqual(response.status_code, 200) for r in responses: # We have to use this roundabout manner since the order may vary each time. This is not @@ -261,4 +261,4 @@ class TestIntegrationsDevPanel(ZulipTestCase): response = self.client_post("/devtools/integrations/send_all_webhook_fixture_messages", data) expected_response = {'msg': 'The integration "appfollow" does not have fixtures.', 'result': 'error'} self.assertEqual(response.status_code, 404) - self.assertEqual(ujson.loads(response.content), expected_response) + self.assertEqual(orjson.loads(response.content), expected_response) diff --git a/zerver/tests/test_link_embed.py b/zerver/tests/test_link_embed.py index 424972b494..93321b2968 100644 --- a/zerver/tests/test_link_embed.py +++ b/zerver/tests/test_link_embed.py @@ -1,7 +1,7 @@ from typing import Any, Callable, Dict, Optional from unittest import mock -import ujson +import orjson from django.test import override_settings from django.utils.html import escape from requests.exceptions import ConnectionError @@ -48,7 +48,7 @@ class OembedTestCase(ZulipTestCase): 'version': '1.0', 'width': 658, 'height': 400} - response.text = ujson.dumps(response_data) + response.text = orjson.dumps(response_data).decode() url = 'http://instagram.com/p/BLtI2WdAymy' data = get_oembed_data(url) self.assertIsInstance(data, dict) @@ -72,7 +72,7 @@ class OembedTestCase(ZulipTestCase): 'version': '1.0', 'width': 658, 'height': 400} - response.text = ujson.dumps(response_data) + response.text = orjson.dumps(response_data).decode() url = 'http://imgur.com/photo/158727223' data = get_oembed_data(url) self.assertIsInstance(data, dict) @@ -96,7 +96,7 @@ class OembedTestCase(ZulipTestCase): 'version': '1.0', 'width': 658, 'height': 400} - response.text = ujson.dumps(response_data) + response.text = orjson.dumps(response_data).decode() url = 'http://blip.tv/video/158727223' data = get_oembed_data(url) self.assertIsInstance(data, dict) diff --git a/zerver/tests/test_management_commands.py b/zerver/tests/test_management_commands.py index e3ea84b3b7..ac7c71c42d 100644 --- a/zerver/tests/test_management_commands.py +++ b/zerver/tests/test_management_commands.py @@ -231,25 +231,25 @@ class TestSendWebhookFixtureMessage(ZulipTestCase): @patch('zerver.management.commands.send_webhook_fixture_message.os.path.exists') @patch('zerver.management.commands.send_webhook_fixture_message.Client') - @patch('zerver.management.commands.send_webhook_fixture_message.ujson') + @patch('zerver.management.commands.send_webhook_fixture_message.orjson') @patch("zerver.management.commands.send_webhook_fixture_message.open", create=True) def test_check_if_command_post_request_to_url_with_fixture(self, open_mock: MagicMock, - ujson_mock: MagicMock, + orjson_mock: MagicMock, client_mock: MagicMock, os_path_exists_mock: MagicMock) -> None: - ujson_mock.load.return_value = {} - ujson_mock.dumps.return_value = "{}" + orjson_mock.loads.return_value = {} + orjson_mock.dumps.return_value = b"{}" os_path_exists_mock.return_value = True client = client_mock() with self.assertRaises(CommandError): call_command(self.COMMAND_NAME, fixture=self.fixture_path, url=self.url) - self.assertTrue(ujson_mock.dumps.called) - self.assertTrue(ujson_mock.load.called) + self.assertTrue(orjson_mock.dumps.called) + self.assertTrue(orjson_mock.loads.called) self.assertTrue(open_mock.called) - client.post.assert_called_once_with(self.url, "{}", content_type="application/json", + client.post.assert_called_once_with(self.url, b"{}", content_type="application/json", HTTP_HOST="zulip.testserver") class TestGenerateRealmCreationLink(ZulipTestCase): diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index 85965ecc4c..0f1f39e263 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -5,7 +5,7 @@ from textwrap import dedent from typing import Any, Dict, List, Optional, Set, Tuple from unittest import mock -import ujson +import orjson from django.conf import settings from django.test import override_settings @@ -371,8 +371,8 @@ class MarkdownTest(ZulipTestCase): def load_markdown_tests(self) -> Tuple[Dict[str, Any], List[List[str]]]: test_fixtures = {} - with open(os.path.join(os.path.dirname(__file__), 'fixtures/markdown_test_cases.json')) as f: - data = ujson.load(f) + with open(os.path.join(os.path.dirname(__file__), 'fixtures/markdown_test_cases.json'), "rb") as f: + data = orjson.loads(f.read()) for test in data['regular_tests']: test_fixtures[test['name']] = test diff --git a/zerver/tests/test_mattermost_importer.py b/zerver/tests/test_mattermost_importer.py index 8d34ae49e0..0c2dbdf9fd 100644 --- a/zerver/tests/test_mattermost_importer.py +++ b/zerver/tests/test_mattermost_importer.py @@ -3,7 +3,7 @@ import os from typing import Any, Dict, List from unittest.mock import call, patch -import ujson +import orjson from zerver.data_import.import_util import SubscriberHandler from zerver.data_import.mattermost import ( @@ -315,8 +315,8 @@ class MatterMostImporter(ZulipTestCase): self.assertEqual(zerver_realm_emoji[1]["deactivated"], False) records_file = os.path.join(output_dir, "emoji", "records.json") - with open(records_file) as f: - records_json = ujson.load(f) + with open(records_file, "rb") as f: + records_json = orjson.loads(f.read()) self.assertEqual(records_json[0]["file_name"], "peerdium") self.assertEqual(records_json[0]["realm_id"], 3) @@ -495,8 +495,8 @@ class MatterMostImporter(ZulipTestCase): def read_file(self, team_output_dir: str, output_file: str) -> Any: full_path = os.path.join(team_output_dir, output_file) - with open(full_path) as f: - return ujson.load(f) + with open(full_path, "rb") as f: + return orjson.loads(f.read()) def test_do_convert_data(self) -> None: mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures") diff --git a/zerver/tests/test_message_edit.py b/zerver/tests/test_message_edit.py index 6025e78d4f..35b6eed21c 100644 --- a/zerver/tests/test_message_edit.py +++ b/zerver/tests/test_message_edit.py @@ -3,7 +3,7 @@ from operator import itemgetter from typing import Any, Dict, List, Tuple from unittest import mock -import ujson +import orjson from django.db import IntegrityError from django.http import HttpResponse @@ -73,7 +73,7 @@ class EditMessageTest(ZulipTestCase): if msg.edit_history: self.assertEqual( fetch_message_dict['edit_history'], - ujson.loads(msg.edit_history), + orjson.loads(msg.edit_history), ) def test_query_count_on_to_dict_uncached(self) -> None: @@ -249,7 +249,7 @@ class EditMessageTest(ZulipTestCase): messages_result = self.client_get("/json/messages", {"anchor": msg_id_1, "num_before": 0, "num_after": 10}) self.assert_json_success(messages_result) - json_messages = ujson.loads( + json_messages = orjson.loads( messages_result.content.decode('utf-8')) for msg in json_messages['messages']: self.assertNotIn("edit_history", msg) @@ -271,7 +271,7 @@ class EditMessageTest(ZulipTestCase): message_edit_history_1 = self.client_get( "/json/messages/" + str(msg_id_1) + "/history") - json_response_1 = ujson.loads( + json_response_1 = orjson.loads( message_edit_history_1.content.decode('utf-8')) message_history_1 = json_response_1['message_history'] @@ -307,7 +307,7 @@ class EditMessageTest(ZulipTestCase): message_edit_history_2 = self.client_get( "/json/messages/" + str(msg_id_2) + "/history") - json_response_2 = ujson.loads( + json_response_2 = orjson.loads( message_edit_history_2.content.decode('utf-8')) message_history_2 = json_response_2['message_history'] @@ -342,7 +342,7 @@ class EditMessageTest(ZulipTestCase): message_edit_history_1 = self.client_get( "/json/messages/" + str(msg_id_1) + "/history") - json_response_1 = ujson.loads( + json_response_1 = orjson.loads( message_edit_history_1.content.decode('utf-8')) message_history_1 = json_response_1['message_history'] @@ -407,7 +407,7 @@ class EditMessageTest(ZulipTestCase): 'content': 'content 2', }) self.assert_json_success(result) - history = ujson.loads(Message.objects.get(id=msg_id).edit_history) + history = orjson.loads(Message.objects.get(id=msg_id).edit_history) self.assertEqual(history[0]['prev_content'], 'content 1') self.assertEqual(history[0]['user_id'], hamlet.id) self.assertEqual(set(history[0].keys()), @@ -419,7 +419,7 @@ class EditMessageTest(ZulipTestCase): 'topic': 'topic 2', }) self.assert_json_success(result) - history = ujson.loads(Message.objects.get(id=msg_id).edit_history) + history = orjson.loads(Message.objects.get(id=msg_id).edit_history) self.assertEqual(history[0][LEGACY_PREV_TOPIC], 'topic 1') self.assertEqual(history[0]['user_id'], hamlet.id) self.assertEqual(set(history[0].keys()), {'timestamp', LEGACY_PREV_TOPIC, 'user_id'}) @@ -430,7 +430,7 @@ class EditMessageTest(ZulipTestCase): 'topic': 'topic 3', }) self.assert_json_success(result) - history = ujson.loads(Message.objects.get(id=msg_id).edit_history) + history = orjson.loads(Message.objects.get(id=msg_id).edit_history) self.assertEqual(history[0]['prev_content'], 'content 2') self.assertEqual(history[0][LEGACY_PREV_TOPIC], 'topic 2') self.assertEqual(history[0]['user_id'], hamlet.id) @@ -443,7 +443,7 @@ class EditMessageTest(ZulipTestCase): 'content': 'content 4', }) self.assert_json_success(result) - history = ujson.loads(Message.objects.get(id=msg_id).edit_history) + history = orjson.loads(Message.objects.get(id=msg_id).edit_history) self.assertEqual(history[0]['prev_content'], 'content 3') self.assertEqual(history[0]['user_id'], hamlet.id) @@ -453,11 +453,11 @@ class EditMessageTest(ZulipTestCase): 'topic': 'topic 4', }) self.assert_json_success(result) - history = ujson.loads(Message.objects.get(id=msg_id).edit_history) + history = orjson.loads(Message.objects.get(id=msg_id).edit_history) self.assertEqual(history[0][LEGACY_PREV_TOPIC], 'topic 3') self.assertEqual(history[0]['user_id'], self.example_user('iago').id) - history = ujson.loads(Message.objects.get(id=msg_id).edit_history) + history = orjson.loads(Message.objects.get(id=msg_id).edit_history) self.assertEqual(history[0][LEGACY_PREV_TOPIC], 'topic 3') self.assertEqual(history[2][LEGACY_PREV_TOPIC], 'topic 2') self.assertEqual(history[3][LEGACY_PREV_TOPIC], 'topic 1') @@ -469,7 +469,7 @@ class EditMessageTest(ZulipTestCase): # correct filled-out fields message_edit_history = self.client_get("/json/messages/" + str(msg_id) + "/history") - json_response = ujson.loads(message_edit_history.content.decode('utf-8')) + json_response = orjson.loads(message_edit_history.content.decode('utf-8')) # We reverse the message history view output so that the IDs line up with the above. message_history = list(reversed(json_response['message_history'])) @@ -511,9 +511,9 @@ class EditMessageTest(ZulipTestCase): message_content_edit_limit_seconds: int, allow_community_topic_editing: bool) -> None: result = self.client_patch("/json/realm", { - 'allow_message_editing': ujson.dumps(allow_message_editing), + 'allow_message_editing': orjson.dumps(allow_message_editing).decode(), 'message_content_edit_limit_seconds': message_content_edit_limit_seconds, - 'allow_community_topic_editing': ujson.dumps(allow_community_topic_editing), + 'allow_community_topic_editing': orjson.dumps(allow_community_topic_editing).decode(), }) self.assert_json_success(result) @@ -583,9 +583,9 @@ class EditMessageTest(ZulipTestCase): message_content_edit_limit_seconds: int, allow_community_topic_editing: bool) -> None: result = self.client_patch("/json/realm", { - 'allow_message_editing': ujson.dumps(allow_message_editing), + 'allow_message_editing': orjson.dumps(allow_message_editing).decode(), 'message_content_edit_limit_seconds': message_content_edit_limit_seconds, - 'allow_community_topic_editing': ujson.dumps(allow_community_topic_editing), + 'allow_community_topic_editing': orjson.dumps(allow_community_topic_editing).decode(), }) self.assert_json_success(result) @@ -1112,7 +1112,7 @@ class DeleteMessageTest(ZulipTestCase): message_content_delete_limit_seconds: int) -> None: self.login('iago') result = self.client_patch("/json/realm", { - 'allow_message_deleting': ujson.dumps(allow_message_deleting), + 'allow_message_deleting': orjson.dumps(allow_message_deleting).decode(), 'message_content_delete_limit_seconds': message_content_delete_limit_seconds, }) self.assert_json_success(result) diff --git a/zerver/tests/test_message_fetch.py b/zerver/tests/test_message_fetch.py index f0e2a33219..ab065dc47c 100644 --- a/zerver/tests/test_message_fetch.py +++ b/zerver/tests/test_message_fetch.py @@ -3,7 +3,7 @@ import os from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union from unittest import mock -import ujson +import orjson from django.db import connection from django.test import override_settings from django.utils.timezone import now as timezone_now @@ -464,8 +464,8 @@ class NarrowLibraryTest(ZulipTestCase): def test_build_narrow_filter(self) -> None: fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures/narrow.json') - with open(fixtures_path) as f: - scenarios = ujson.load(f) + with open(fixtures_path, "rb") as f: + scenarios = orjson.loads(f.read()) self.assertTrue(len(scenarios) == 9) for scenario in scenarios: narrow = scenario['narrow'] @@ -1059,7 +1059,7 @@ class GetOldMessagesTest(ZulipTestCase): self.assertEqual(set(payload["Cache-Control"].split(", ")), {"must-revalidate", "no-store", "no-cache", "max-age=0"}) - result = ujson.loads(payload.content) + result = orjson.loads(payload.content) self.assertIn("messages", result) self.assertIsInstance(result["messages"], list) @@ -1074,11 +1074,11 @@ class GetOldMessagesTest(ZulipTestCase): message_ids: List[int], pivot_index: int) -> None: num_before = len(message_ids) - post_params = dict(narrow=ujson.dumps(narrow), num_before=num_before, + post_params = dict(narrow=orjson.dumps(narrow).decode(), num_before=num_before, num_after=0, anchor=LARGER_THAN_MAX_MESSAGE_ID) payload = self.client_get("/json/messages", dict(post_params)) self.assert_json_success(payload) - result = ujson.loads(payload.content) + result = orjson.loads(payload.content) self.assertEqual(len(result["messages"]), len(message_ids)) for message in result["messages"]: @@ -1090,7 +1090,7 @@ class GetOldMessagesTest(ZulipTestCase): payload = self.client_get("/json/messages", dict(post_params)) self.assert_json_success(payload) - result = ujson.loads(payload.content) + result = orjson.loads(payload.content) self.assertEqual(len(result["messages"]), len(message_ids[pivot_index:])) for message in result["messages"]: @@ -1123,7 +1123,7 @@ class GetOldMessagesTest(ZulipTestCase): def get_content_type(apply_markdown: bool) -> str: req: Dict[str, Any] = dict( - apply_markdown=ujson.dumps(apply_markdown), + apply_markdown=orjson.dumps(apply_markdown).decode(), ) result = self.get_and_check_messages(req) message = result['messages'][0] @@ -1183,17 +1183,17 @@ class GetOldMessagesTest(ZulipTestCase): # clients around, which might include third party home-grown bots. self.get_and_check_messages( dict( - narrow=ujson.dumps( + narrow=orjson.dumps( [['pm-with', othello_email]], - ), + ).decode(), ), ) self.get_and_check_messages( dict( - narrow=ujson.dumps( + narrow=orjson.dumps( [dict(operator='pm-with', operand=othello_email)], - ), + ).decode(), ), ) @@ -1213,14 +1213,14 @@ class GetOldMessagesTest(ZulipTestCase): message = result['messages'][0] self.assertIn('gravatar.com', message['avatar_url']) - result = self.get_and_check_messages(dict(client_gravatar=ujson.dumps(True))) + result = self.get_and_check_messages(dict(client_gravatar=orjson.dumps(True).decode())) message = result['messages'][0] self.assertEqual(message['avatar_url'], None) # Now verify client_gravatar doesn't run with EMAIL_ADDRESS_VISIBILITY_ADMINS do_set_realm_property(hamlet.realm, "email_address_visibility", Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS) - result = self.get_and_check_messages(dict(client_gravatar=ujson.dumps(True))) + result = self.get_and_check_messages(dict(client_gravatar=orjson.dumps(True).decode())) message = result['messages'][0] self.assertIn('gravatar.com', message['avatar_url']) @@ -1264,7 +1264,7 @@ class GetOldMessagesTest(ZulipTestCase): emails = dr_emails(get_display_recipient(personal.recipient)) self.login_user(me) narrow: List[Dict[str, Any]] = [dict(operator='pm-with', operand=emails)] - result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow))) + result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(dr_emails(message['display_recipient']), emails) @@ -1272,7 +1272,7 @@ class GetOldMessagesTest(ZulipTestCase): # check passing id is conistent with passing emails as operand ids = dr_ids(get_display_recipient(personal.recipient)) narrow = [dict(operator='pm-with', operand=ids)] - result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow))) + result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(dr_emails(message['display_recipient']), emails) @@ -1340,7 +1340,7 @@ class GetOldMessagesTest(ZulipTestCase): test_operands = [cordelia.email, cordelia.id] for operand in test_operands: narrow = [dict(operator='group-pm-with', operand=operand)] - result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow))) + result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertIn(message["id"], matching_message_ids) self.assertNotIn(message["id"], non_matching_message_ids) @@ -1396,7 +1396,7 @@ class GetOldMessagesTest(ZulipTestCase): ] req = dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=LARGER_THAN_MAX_MESSAGE_ID, num_before=100, num_after=100, @@ -1404,7 +1404,7 @@ class GetOldMessagesTest(ZulipTestCase): payload = self.client_get('/json/messages', req) self.assert_json_success(payload) - result = ujson.loads(payload.content) + result = orjson.loads(payload.content) messages = result['messages'] self.assertEqual(len(messages), 2) @@ -1437,7 +1437,7 @@ class GetOldMessagesTest(ZulipTestCase): for operand in [stream_name, stream_id]: narrow = [dict(operator='stream', operand=operand)] - result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow))) + result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(message["type"], "stream") @@ -1476,7 +1476,7 @@ class GetOldMessagesTest(ZulipTestCase): narrow = [dict(operator='stream', operand='\u03bb-stream')] result = self.get_and_check_messages(dict(num_after=2, - narrow=ujson.dumps(narrow)), + narrow=orjson.dumps(narrow).decode()), subdomain="zephyr") messages = get_user_messages(self.mit_user("starnine")) @@ -1507,7 +1507,7 @@ class GetOldMessagesTest(ZulipTestCase): narrow = [dict(operator='topic', operand='\u03bb-topic')] result = self.get_and_check_messages( - dict(num_after=100, narrow=ujson.dumps(narrow)), + dict(num_after=100, narrow=orjson.dumps(narrow).decode()), subdomain="zephyr") messages = get_user_messages(mit_user_profile) @@ -1542,7 +1542,7 @@ class GetOldMessagesTest(ZulipTestCase): result = self.get_and_check_messages( dict(num_before=50, num_after=50, - narrow=ujson.dumps(narrow)), + narrow=orjson.dumps(narrow).decode()), subdomain="zephyr") messages = get_user_messages(mit_user_profile) @@ -1574,7 +1574,7 @@ class GetOldMessagesTest(ZulipTestCase): test_operands = [othello.email, othello.id] for operand in test_operands: narrow = [dict(operator='sender', operand=operand)] - result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow))) + result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(message["sender_id"], othello.id) @@ -1616,7 +1616,7 @@ class GetOldMessagesTest(ZulipTestCase): ] raw_params = dict(msg_ids=msg_ids, narrow=narrow) - params = {k: ujson.dumps(v) for k, v in raw_params.items()} + params = {k: orjson.dumps(v).decode() for k, v in raw_params.items()} result = self.client_get('/json/messages/matches_narrow', params) self.assert_json_success(result) messages = result.json()['messages'] @@ -1661,7 +1661,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='lunch'), ] result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_before=0, num_after=10, @@ -1671,7 +1671,7 @@ class GetOldMessagesTest(ZulipTestCase): narrow = [dict(operator='search', operand='https://google.com')] link_search_result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_before=0, num_after=10, @@ -1709,7 +1709,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='after'), ] multi_search_result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(multi_search_narrow), + narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1722,7 +1722,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='日本'), ] result = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1758,7 +1758,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='今日は'), ] multi_search_result = self.get_and_check_messages(dict( - narrow=ujson.dumps(multi_search_narrow), + narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1806,7 +1806,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='stream', operand='newstream'), ] stream_search_result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(stream_search_narrow), + narrow=orjson.dumps(stream_search_narrow).decode(), anchor=0, num_after=10, num_before=10, @@ -1853,7 +1853,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='日本'), ] result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1889,7 +1889,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='wiki'), ] multi_search_result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(multi_search_narrow), + narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1904,7 +1904,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='べました'), ] multi_search_result = self.get_and_check_messages(dict( - narrow=ujson.dumps(multi_search_narrow), + narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1915,7 +1915,7 @@ class GetOldMessagesTest(ZulipTestCase): narrow = [dict(operator='search', operand='https://google.com')] link_search_result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1929,7 +1929,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='butter'), ] special_search_result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(special_search_narrow), + narrow=orjson.dumps(special_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1942,7 +1942,7 @@ class GetOldMessagesTest(ZulipTestCase): dict(operator='search', operand='&'), ] special_search_result = self.get_and_check_messages(dict( - narrow=ujson.dumps(special_search_narrow), + narrow=orjson.dumps(special_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, @@ -1976,7 +1976,7 @@ class GetOldMessagesTest(ZulipTestCase): ] raw_params = dict(msg_ids=msg_ids, narrow=narrow) - params = {k: ujson.dumps(v) for k, v in raw_params.items()} + params = {k: orjson.dumps(v).decode() for k, v in raw_params.items()} result = self.client_get('/json/messages/matches_narrow', params) self.assert_json_success(result) messages = result.json()['messages'] @@ -1999,14 +1999,14 @@ class GetOldMessagesTest(ZulipTestCase): narrow = [dict(operator='sender', operand=cordelia.email)] result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=anchor, num_before=0, num_after=0, )) self.assertEqual(len(result['messages']), 1) narrow = [dict(operator='is', operand='mentioned')] - result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow), + result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode(), anchor=anchor, num_before=0, num_after=0)) self.assertEqual(len(result['messages']), 0) @@ -2332,7 +2332,7 @@ class GetOldMessagesTest(ZulipTestCase): self.login('hamlet') for operator in ['', 'foo', 'stream:verona', '__init__']: narrow = [dict(operator=operator, operand='')] - params = dict(anchor=0, num_before=0, num_after=0, narrow=ujson.dumps(narrow)) + params = dict(anchor=0, num_before=0, num_after=0, narrow=orjson.dumps(narrow).decode()) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, "Invalid narrow operator: unknown operator") @@ -2373,7 +2373,7 @@ class GetOldMessagesTest(ZulipTestCase): for operand in operands: narrow = [dict(operator=operator, operand=operand)] - params = dict(anchor=0, num_before=0, num_after=0, narrow=ujson.dumps(narrow)) + params = dict(anchor=0, num_before=0, num_after=0, narrow=orjson.dumps(narrow).decode()) result = self.client_get('/json/messages', params) self.assert_json_error_contains(result, error_msg) @@ -2383,7 +2383,7 @@ class GetOldMessagesTest(ZulipTestCase): other_params: List[Tuple[str, Any]] = [("anchor", 0), ("num_before", 0), ("num_after", 0)] for operand in operands: post_params = dict(other_params + [ - ("narrow", ujson.dumps([[operator, operand]]))]) + ("narrow", orjson.dumps([[operator, operand]]).decode())]) result = self.client_get("/json/messages", post_params) self.assert_json_error_contains(result, error_msg) @@ -2503,7 +2503,7 @@ class GetOldMessagesTest(ZulipTestCase): request = POSTRequestMock(query_params, user_profile) payload = get_messages_backend(request, user_profile) - result = ujson.loads(payload.content) + result = orjson.loads(payload.content) self.assertEqual(result['anchor'], first_message_id) self.assertEqual(result['found_newest'], True) self.assertEqual(result['found_oldest'], True) @@ -2947,7 +2947,7 @@ WHERE user_profile_id = {hamlet_id} AND (content ILIKE '%jumping%' OR subject IL dict(operator='search', operand=othello.email), ] result: Dict[str, Any] = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, )) @@ -2958,7 +2958,7 @@ WHERE user_profile_id = {hamlet_id} AND (content ILIKE '%jumping%' OR subject IL dict(operator='search', operand='othello'), ] result = self.get_and_check_messages(dict( - narrow=ujson.dumps(narrow), + narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, )) diff --git a/zerver/tests/test_message_flags.py b/zerver/tests/test_message_flags.py index 6d291ae20f..c22b11a782 100644 --- a/zerver/tests/test_message_flags.py +++ b/zerver/tests/test_message_flags.py @@ -1,7 +1,7 @@ from typing import Any, List, Mapping, Set from unittest import mock -import ujson +import orjson from django.db import connection from django.http import HttpResponse @@ -87,7 +87,7 @@ class FirstUnreadAnchorTests(ZulipTestCase): # Let's set this old message to be unread result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps([old_message_id]), + {"messages": orjson.dumps([old_message_id]).decode(), "op": "remove", "flag": "read"}) @@ -168,7 +168,7 @@ class UnreadCountTests(ZulipTestCase): self.login('hamlet') result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps(self.unread_msg_ids), + {"messages": orjson.dumps(self.unread_msg_ids).decode(), "op": "add", "flag": "read"}) self.assert_json_success(result) @@ -182,7 +182,7 @@ class UnreadCountTests(ZulipTestCase): self.assertEqual(found, 2) result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps([self.unread_msg_ids[1]]), + {"messages": orjson.dumps([self.unread_msg_ids[1]]).decode(), "op": "remove", "flag": "read"}) self.assert_json_success(result) @@ -432,13 +432,13 @@ class PushNotificationMarkReadFlowsTest(ZulipTestCase): property_name = "push_notifications" result = self.api_post(user_profile, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": True, - "stream_id": stream.id}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": True, + "stream_id": stream.id}]).decode()}) result = self.api_post(user_profile, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": True, - "stream_id": second_stream.id}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": True, + "stream_id": second_stream.id}]).decode()}) self.assert_json_success(result) self.assertEqual(self.get_mobile_push_notification_ids(user_profile), []) @@ -900,32 +900,32 @@ class MessageAccessTests(ZulipTestCase): self.login('hamlet') result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps([message]), + {"messages": orjson.dumps([message]).decode(), "op": "add", "flag": "invalid"}) self.assert_json_error(result, "Invalid flag: 'invalid'") result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps([message]), + {"messages": orjson.dumps([message]).decode(), "op": "add", "flag": "is_private"}) self.assert_json_error(result, "Invalid flag: 'is_private'") result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps([message]), + {"messages": orjson.dumps([message]).decode(), "op": "add", "flag": "active_mobile_push_notification"}) self.assert_json_error(result, "Invalid flag: 'active_mobile_push_notification'") result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps([message]), + {"messages": orjson.dumps([message]).decode(), "op": "add", "flag": "mentioned"}) self.assert_json_error(result, "Flag not editable: 'mentioned'") def change_star(self, messages: List[int], add: bool=True, **kwargs: Any) -> HttpResponse: return self.client_post("/json/messages/flags", - {"messages": ujson.dumps(messages), + {"messages": orjson.dumps(messages).decode(), "op": "add" if add else "remove", "flag": "starred"}, **kwargs) @@ -992,13 +992,13 @@ class MessageAccessTests(ZulipTestCase): ), ] result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps(sent_message_ids), + {"messages": orjson.dumps(sent_message_ids).decode(), "op": "add", "flag": "read"}) # We can't change flags other than "starred" on historical messages: result = self.client_post("/json/messages/flags", - {"messages": ujson.dumps(message_ids), + {"messages": orjson.dumps(message_ids).decode(), "op": "add", "flag": "read"}) self.assert_json_error(result, 'Invalid message(s)') diff --git a/zerver/tests/test_message_send.py b/zerver/tests/test_message_send.py index 797491b694..7b1f093356 100644 --- a/zerver/tests/test_message_send.py +++ b/zerver/tests/test_message_send.py @@ -2,7 +2,7 @@ import datetime from typing import Any, Optional, Set from unittest import mock -import ujson +import orjson from django.conf import settings from django.db.models import Q from django.http import HttpResponse @@ -112,7 +112,7 @@ class MessagePOSTTest(ZulipTestCase): bot, "/api/v1/messages", { "type": "stream", - "to": ujson.dumps([99999]), + "to": orjson.dumps([99999]).decode(), "client": "test suite", "content": "Stream message by ID.", "topic": "Test topic for stream ID message", @@ -134,7 +134,7 @@ class MessagePOSTTest(ZulipTestCase): realm = get_realm('zulip') stream = get_stream('Verona', realm) result = self.client_post("/json/messages", {"type": "stream", - "to": ujson.dumps([stream.id]), + "to": orjson.dumps([stream.id]).decode(), "client": "test suite", "content": "Stream message by ID.", "topic": "Test topic for stream ID message"}) @@ -314,7 +314,7 @@ class MessagePOSTTest(ZulipTestCase): "client": "test suite", "to": othello.email}) self.assert_json_success(result) - message_id = ujson.loads(result.content.decode())['id'] + message_id = orjson.loads(result.content.decode())['id'] recent_conversations = get_recent_private_conversations(user_profile) self.assertEqual(len(recent_conversations), 1) @@ -329,7 +329,7 @@ class MessagePOSTTest(ZulipTestCase): "client": "test suite", "to": user_profile.email}) self.assert_json_success(result) - self_message_id = ujson.loads(result.content.decode())['id'] + self_message_id = orjson.loads(result.content.decode())['id'] recent_conversations = get_recent_private_conversations(user_profile) self.assertEqual(len(recent_conversations), 2) @@ -355,7 +355,7 @@ class MessagePOSTTest(ZulipTestCase): "type": "private", "content": "Test message", "client": "test suite", - "to": ujson.dumps([self.example_user("othello").id]), + "to": orjson.dumps([self.example_user("othello").id]).decode(), }, ) self.assert_json_success(result) @@ -375,8 +375,8 @@ class MessagePOSTTest(ZulipTestCase): "type": "private", "content": "Test message", "client": "test suite", - "to": ujson.dumps([self.example_user("othello").id, - self.example_user("cordelia").id]), + "to": orjson.dumps([self.example_user("othello").id, + self.example_user("cordelia").id]).decode(), }, ) self.assert_json_success(result) @@ -401,7 +401,7 @@ class MessagePOSTTest(ZulipTestCase): "type": "private", "content": "Test message", "client": "test suite", - "to": ujson.dumps([hamlet.id, othello.id])}) + "to": orjson.dumps([hamlet.id, othello.id]).decode()}) self.assert_json_success(result) msg = self.get_last_message() # Verify that we're not actually on the "recipient list" @@ -431,14 +431,14 @@ class MessagePOSTTest(ZulipTestCase): "type": "private", "content": "Test message", "client": "test suite", - "to": ujson.dumps([othello.id])}) + "to": orjson.dumps([othello.id]).decode()}) self.assert_json_error(result, f"'{othello.email}' is no longer using Zulip.") result = self.client_post("/json/messages", { "type": "private", "content": "Test message", "client": "test suite", - "to": ujson.dumps([othello.id, cordelia.id])}) + "to": orjson.dumps([othello.id, cordelia.id]).decode()}) self.assert_json_error(result, f"'{othello.email}' is no longer using Zulip.") def test_invalid_type(self) -> None: @@ -520,8 +520,8 @@ class MessagePOSTTest(ZulipTestCase): "sender": self.mit_email("sipbtest"), "content": "Test message", "client": "zephyr_mirror", - "to": ujson.dumps([self.mit_email("starnine"), - self.mit_email("espuser")])}, + "to": orjson.dumps([self.mit_email("starnine"), + self.mit_email("espuser")]).decode()}, subdomain="zephyr") self.assert_json_success(result) @@ -574,8 +574,8 @@ class MessagePOSTTest(ZulipTestCase): "sender": self.mit_email("sipbtest"), "content": "Test message", "client": "zephyr_mirror", - "to": ujson.dumps([self.mit_email("espuser"), - self.mit_email("starnine")])} + "to": orjson.dumps([self.mit_email("espuser"), + self.mit_email("starnine")]).decode()} with mock.patch('DNS.dnslookup', return_value=[['starnine:*:84233:101:Athena Consulting Exchange User,,,:/mit/starnine:/bin/bash']]): result1 = self.api_post(self.mit_user("starnine"), "/api/v1/messages", msg, @@ -587,8 +587,8 @@ class MessagePOSTTest(ZulipTestCase): subdomain="zephyr") self.assert_json_success(result2) - self.assertEqual(ujson.loads(result1.content)['id'], - ujson.loads(result2.content)['id']) + self.assertEqual(orjson.loads(result1.content)['id'], + orjson.loads(result2.content)['id']) def test_message_with_null_bytes(self) -> None: """ @@ -740,7 +740,7 @@ class MessagePOSTTest(ZulipTestCase): "sender": self.mit_email("sipbtest"), "content": "Test message", "client": "zephyr_mirror", - "to": ujson.dumps([user.id])}, + "to": orjson.dumps([user.id]).decode()}, subdomain="zephyr") self.assert_json_error(result, "Mirroring not allowed with recipient user IDs") @@ -815,7 +815,7 @@ class MessagePOSTTest(ZulipTestCase): client=client, topic='whatever', content='whatever', - forged=ujson.dumps(forged), + forged=orjson.dumps(forged).decode(), ) # Only pass the 'sender' property when doing mirroring behavior. @@ -1536,7 +1536,7 @@ class ExtractTest(ZulipTestCase): def test_extract_private_recipients_emails(self) -> None: # JSON list w/dups, empties, and trailing whitespace - s = ujson.dumps([' alice@zulip.com ', ' bob@zulip.com ', ' ', 'bob@zulip.com']) + s = orjson.dumps([' alice@zulip.com ', ' bob@zulip.com ', ' ', 'bob@zulip.com']).decode() # sorted() gets confused by extract_private_recipients' return type # For testing, ignorance here is better than manual casting result = sorted(extract_private_recipients(s)) @@ -1561,11 +1561,11 @@ class ExtractTest(ZulipTestCase): self.assertEqual(result, ['alice@zulip.com', 'bob@zulip.com']) # Invalid data - s = ujson.dumps(dict(color='red')) + s = orjson.dumps(dict(color='red')).decode() with self.assertRaisesRegex(JsonableError, 'Invalid data type for recipients'): extract_private_recipients(s) - s = ujson.dumps([{}]) + s = orjson.dumps([{}]).decode() with self.assertRaisesRegex(JsonableError, 'Invalid data type for recipients'): extract_private_recipients(s) @@ -1573,23 +1573,23 @@ class ExtractTest(ZulipTestCase): self.assertEqual(extract_private_recipients('[]'), []) # Heterogeneous lists are not supported - mixed = ujson.dumps(['eeshan@example.com', 3, 4]) + mixed = orjson.dumps(['eeshan@example.com', 3, 4]).decode() with self.assertRaisesRegex(JsonableError, 'Recipient lists may contain emails or user IDs, but not both.'): extract_private_recipients(mixed) def test_extract_recipient_ids(self) -> None: # JSON list w/dups - s = ujson.dumps([3, 3, 12]) + s = orjson.dumps([3, 3, 12]).decode() result = sorted(extract_private_recipients(s)) self.assertEqual(result, [3, 12]) # Invalid data - ids = ujson.dumps(dict(recipient=12)) + ids = orjson.dumps(dict(recipient=12)).decode() with self.assertRaisesRegex(JsonableError, 'Invalid data type for recipients'): extract_private_recipients(ids) # Heterogeneous lists are not supported - mixed = ujson.dumps([3, 4, 'eeshan@example.com']) + mixed = orjson.dumps([3, 4, 'eeshan@example.com']).decode() with self.assertRaisesRegex(JsonableError, 'Recipient lists may contain emails or user IDs, but not both.'): extract_private_recipients(mixed) diff --git a/zerver/tests/test_outgoing_webhook_system.py b/zerver/tests/test_outgoing_webhook_system.py index cf2a30bded..19886e0983 100644 --- a/zerver/tests/test_outgoing_webhook_system.py +++ b/zerver/tests/test_outgoing_webhook_system.py @@ -1,8 +1,8 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict from unittest import mock +import orjson import requests -import ujson from version import ZULIP_VERSION from zerver.lib.actions import do_create_user @@ -18,10 +18,10 @@ from zerver.models import Recipient, Service, UserProfile, get_display_recipient class ResponseMock: - def __init__(self, status_code: int, content: Optional[Any]=None) -> None: + def __init__(self, status_code: int, content: bytes=b"") -> None: self.status_code = status_code self.content = content - self.text = ujson.dumps(content) + self.text = content.decode() def request_exception_error(http_method: Any, final_url: Any, data: Any, **request_kwargs: Any) -> Any: raise requests.exceptions.RequestException("I'm a generic exception :(") @@ -53,7 +53,7 @@ class DoRestCallTests(ZulipTestCase): mock_event = self.mock_event(bot_user) service_handler = GenericOutgoingWebhookService("token", bot_user, "service") - response = ResponseMock(200, dict(content='whatever')) + response = ResponseMock(200, orjson.dumps(dict(content='whatever'))) expect_200 = mock.patch('requests.request', return_value=response) expect_send_response = mock.patch('zerver.lib.outgoing_webhook.send_response_message') @@ -222,7 +222,7 @@ class TestOutgoingWebhookMessaging(ZulipTestCase): for item in m.call_args_list: args = item[0] base_url = args[0] - request_data = ujson.loads(args[1]) + request_data = orjson.loads(args[1]) tup = (base_url, request_data['token']) url_token_tups.add(tup) message_data = request_data['message'] @@ -237,7 +237,7 @@ class TestOutgoingWebhookMessaging(ZulipTestCase): }, ) - @mock.patch('requests.request', return_value=ResponseMock(200, {"response_string": "Hidley ho, I'm a webhook responding!"})) + @mock.patch('requests.request', return_value=ResponseMock(200, orjson.dumps({"response_string": "Hidley ho, I'm a webhook responding!"}))) def test_pm_to_outgoing_webhook_bot(self, mock_requests_request: mock.Mock) -> None: bot_owner = self.example_user("othello") bot = self.create_outgoing_bot(bot_owner) @@ -257,7 +257,7 @@ class TestOutgoingWebhookMessaging(ZulipTestCase): Recipient.PERSONAL, ) - @mock.patch('requests.request', return_value=ResponseMock(200, {"response_string": "Hidley ho, I'm a webhook responding!"})) + @mock.patch('requests.request', return_value=ResponseMock(200, orjson.dumps({"response_string": "Hidley ho, I'm a webhook responding!"}))) def test_stream_message_to_outgoing_webhook_bot(self, mock_requests_request: mock.Mock) -> None: bot_owner = self.example_user("othello") bot = self.create_outgoing_bot(bot_owner) diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index b5157d0ff3..04940e1287 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -8,8 +8,8 @@ from typing import Any, Dict, Iterator, List, Optional from unittest import mock from unittest.mock import call +import orjson import requests -import ujson from django.conf import settings from django.db import transaction from django.db.models import F @@ -500,9 +500,9 @@ class AnalyticsBouncerTest(BouncerTestCase): result = self.uuid_post( self.server_uuid, '/api/v1/remotes/server/analytics', - {'realm_counts': ujson.dumps(realm_count_data), - 'installation_counts': ujson.dumps(installation_count_data), - 'realmauditlog_rows': ujson.dumps(realmauditlog_data)}, + {'realm_counts': orjson.dumps(realm_count_data).decode(), + 'installation_counts': orjson.dumps(installation_count_data).decode(), + 'realmauditlog_rows': orjson.dumps(realmauditlog_data).decode()}, subdomain="") self.assert_json_error(result, "Data is out of order.") @@ -514,9 +514,9 @@ class AnalyticsBouncerTest(BouncerTestCase): result = self.uuid_post( self.server_uuid, '/api/v1/remotes/server/analytics', - {'realm_counts': ujson.dumps(realm_count_data), - 'installation_counts': ujson.dumps(installation_count_data), - 'realmauditlog_rows': ujson.dumps(realmauditlog_data)}, + {'realm_counts': orjson.dumps(realm_count_data).decode(), + 'installation_counts': orjson.dumps(installation_count_data).decode(), + 'realmauditlog_rows': orjson.dumps(realmauditlog_data).decode()}, subdomain="") self.assert_json_error(result, "Invalid data.") self.assertEqual(warn_log.output, [ @@ -1630,12 +1630,12 @@ class TestSendNotificationsToBouncer(ZulipTestCase): } mock_send.assert_called_with('POST', 'push/notify', - ujson.dumps(post_data), + orjson.dumps(post_data), extra_headers={'Content-type': 'application/json'}) class Result: - def __init__(self, status: int=200, content: str=ujson.dumps({'msg': 'error'})) -> None: + def __init__(self, status: int=200, content: bytes=orjson.dumps({'msg': 'error'})) -> None: self.status_code = status self.content = content @@ -1660,21 +1660,19 @@ class TestSendToPushBouncer(ZulipTestCase): error_obj = InvalidZulipServerError("testRole") with mock.patch('requests.request', return_value=Result(status=400, - content=ujson.dumps(error_obj.to_json()))): + content=orjson.dumps(error_obj.to_json()))): with self.assertRaises(PushNotificationBouncerException) as exc: send_to_push_bouncer('register', 'register', {'msg': 'true'}) self.assertEqual(str(exc.exception), 'Push notifications bouncer error: ' 'Zulip server auth failure: testRole is not registered') - @mock.patch('requests.request', return_value=Result(status=400, content='/')) + @mock.patch('requests.request', return_value=Result(status=400, content=b'/')) def test_400_error_when_content_is_not_serializable(self, mock_request: mock.MagicMock) -> None: - with self.assertRaises(ValueError) as exc: + with self.assertRaises(orjson.JSONDecodeError): send_to_push_bouncer('register', 'register', {'msg': 'true'}) - self.assertEqual(str(exc.exception), - 'Expected object or value') - @mock.patch('requests.request', return_value=Result(status=300, content='/')) + @mock.patch('requests.request', return_value=Result(status=300, content=b'/')) def test_300_error(self, mock_request: mock.MagicMock) -> None: with self.assertRaises(PushNotificationBouncerException) as exc: send_to_push_bouncer('register', 'register', {'msg': 'true'}) @@ -2090,7 +2088,7 @@ class TestReceivesNotificationsFunctions(ZulipTestCase): class TestPushNotificationsContent(ZulipTestCase): def test_fixtures(self) -> None: - fixtures = ujson.loads(self.fixture_data("markdown_test_cases.json")) + fixtures = orjson.loads(self.fixture_data("markdown_test_cases.json")) tests = fixtures["regular_tests"] for test in tests: if "text_content" in test: diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index 3ab78bb936..108978a861 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -5,7 +5,7 @@ import time from typing import Any, Callable, Dict, List, Mapping, Tuple from unittest.mock import MagicMock, patch -import ujson +import orjson from django.conf import settings from django.test import override_settings @@ -444,7 +444,7 @@ class WorkerTest(ZulipTestCase): fake_response = MagicMock() fake_response.status_code = 400 - fake_response.text = ujson.dumps({'title': ''}) + fake_response.content = orjson.dumps({'title': ''}) with simulated_queue_client(lambda: fake_client): worker = queue_processors.SignupWorker() worker.setup() @@ -471,7 +471,7 @@ class WorkerTest(ZulipTestCase): fake_response = MagicMock() fake_response.status_code = 400 - fake_response.text = ujson.dumps({'title': 'Member Exists'}) + fake_response.content = orjson.dumps({'title': 'Member Exists'}) with simulated_queue_client(lambda: fake_client): worker = queue_processors.SignupWorker() worker.setup() @@ -500,7 +500,7 @@ class WorkerTest(ZulipTestCase): fake_response = MagicMock() fake_response.status_code = 444 # Any non-400 bad request code. - fake_response.text = ujson.dumps({'title': 'Member Exists'}) + fake_response.content = orjson.dumps({'title': 'Member Exists'}) with simulated_queue_client(lambda: fake_client): worker = queue_processors.SignupWorker() worker.setup() @@ -576,7 +576,7 @@ class WorkerTest(ZulipTestCase): self.assertEqual(processed, ['good', 'fine', 'back to normal']) with open(fn) as f: line = f.readline().strip() - events = ujson.loads(line.split('\t')[1]) + events = orjson.loads(line.split('\t')[1]) self.assert_length(events, 1) event = events[0] self.assertEqual(event["type"], 'unexpected behaviour') @@ -616,7 +616,7 @@ class WorkerTest(ZulipTestCase): self.assertEqual(processed, ['good', 'fine']) with open(fn) as f: line = f.readline().strip() - events = ujson.loads(line.split('\t')[1]) + events = orjson.loads(line.split('\t')[1]) self.assert_length(events, 4) self.assertEqual([event["type"] for event in events], diff --git a/zerver/tests/test_reactions.py b/zerver/tests/test_reactions.py index 91cb2c517a..e747423b6f 100644 --- a/zerver/tests/test_reactions.py +++ b/zerver/tests/test_reactions.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Mapping from unittest import mock -import ujson +import orjson from django.http import HttpResponse from zerver.lib.cache import cache_get, to_dict_cache_key_id @@ -299,7 +299,7 @@ class ReactionTest(ZulipTestCase): "content": "Test message", "to": pm_recipient.email}) self.assert_json_success(pm) - content = ujson.loads(pm.content) + content = orjson.loads(pm.content) pm_id = content['id'] @@ -329,7 +329,7 @@ class ReactionTest(ZulipTestCase): "to": pm_recipient.email}) self.assert_json_success(pm) - content = ujson.loads(pm.content) + content = orjson.loads(pm.content) pm_id = content['id'] reaction_info = { 'emoji_name': 'smile', diff --git a/zerver/tests/test_realm.py b/zerver/tests/test_realm.py index be017dae4d..5c6bd24cf1 100644 --- a/zerver/tests/test_realm.py +++ b/zerver/tests/test_realm.py @@ -3,7 +3,7 @@ import re from typing import Any, Dict, List, Mapping from unittest import mock -import ujson +import orjson from django.conf import settings from confirmation.models import Confirmation, create_confirmation_link @@ -94,7 +94,7 @@ class RealmTest(ZulipTestCase): def test_update_realm_description(self) -> None: self.login('iago') new_description = 'zulip dev group' - data = dict(description=ujson.dumps(new_description)) + data = dict(description=orjson.dumps(new_description).decode()) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): result = self.client_patch('/json/realm', data) @@ -112,7 +112,7 @@ class RealmTest(ZulipTestCase): def test_realm_description_length(self) -> None: new_description = 'A' * 1001 - data = dict(description=ujson.dumps(new_description)) + data = dict(description=orjson.dumps(new_description).decode()) # create an admin user self.login('iago') @@ -124,7 +124,7 @@ class RealmTest(ZulipTestCase): def test_realm_name_length(self) -> None: new_name = 'A' * (Realm.MAX_REALM_NAME_LENGTH + 1) - data = dict(name=ujson.dumps(new_name)) + data = dict(name=orjson.dumps(new_name).decode()) # create an admin user self.login('iago') @@ -139,7 +139,7 @@ class RealmTest(ZulipTestCase): self.login('othello') - req = dict(name=ujson.dumps(new_name)) + req = dict(name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Must be an organization administrator') @@ -269,14 +269,14 @@ class RealmTest(ZulipTestCase): self.login('iago') disabled_notif_stream_id = -1 - req = dict(notifications_stream_id = ujson.dumps(disabled_notif_stream_id)) + req = dict(notifications_stream_id = orjson.dumps(disabled_notif_stream_id).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) realm = get_realm('zulip') self.assertEqual(realm.notifications_stream, None) new_notif_stream_id = 4 - req = dict(notifications_stream_id = ujson.dumps(new_notif_stream_id)) + req = dict(notifications_stream_id = orjson.dumps(new_notif_stream_id).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) realm = get_realm('zulip') @@ -284,7 +284,7 @@ class RealmTest(ZulipTestCase): self.assertEqual(realm.notifications_stream.id, new_notif_stream_id) invalid_notif_stream_id = 1234 - req = dict(notifications_stream_id = ujson.dumps(invalid_notif_stream_id)) + req = dict(notifications_stream_id = orjson.dumps(invalid_notif_stream_id).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid stream id') realm = get_realm('zulip') @@ -308,14 +308,14 @@ class RealmTest(ZulipTestCase): self.login('iago') disabled_signup_notifications_stream_id = -1 - req = dict(signup_notifications_stream_id = ujson.dumps(disabled_signup_notifications_stream_id)) + req = dict(signup_notifications_stream_id = orjson.dumps(disabled_signup_notifications_stream_id).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) realm = get_realm('zulip') self.assertEqual(realm.signup_notifications_stream, None) new_signup_notifications_stream_id = 4 - req = dict(signup_notifications_stream_id = ujson.dumps(new_signup_notifications_stream_id)) + req = dict(signup_notifications_stream_id = orjson.dumps(new_signup_notifications_stream_id).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) @@ -324,7 +324,7 @@ class RealmTest(ZulipTestCase): self.assertEqual(realm.signup_notifications_stream.id, new_signup_notifications_stream_id) invalid_signup_notifications_stream_id = 1234 - req = dict(signup_notifications_stream_id = ujson.dumps(invalid_signup_notifications_stream_id)) + req = dict(signup_notifications_stream_id = orjson.dumps(invalid_signup_notifications_stream_id).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid stream id') realm = get_realm('zulip') @@ -350,7 +350,7 @@ class RealmTest(ZulipTestCase): # we need an admin user. self.login('iago') - req = dict(default_language=ujson.dumps(new_lang)) + req = dict(default_language=orjson.dumps(new_lang).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) realm = get_realm('zulip') @@ -360,7 +360,7 @@ class RealmTest(ZulipTestCase): # as the default realm language, correct validation error is # raised and the invalid language is not saved in db invalid_lang = "invalid_lang" - req = dict(default_language=ujson.dumps(invalid_lang)) + req = dict(default_language=orjson.dumps(invalid_lang).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, f"Invalid language '{invalid_lang}'") realm = get_realm('zulip') @@ -389,12 +389,12 @@ class RealmTest(ZulipTestCase): def test_change_bot_creation_policy(self) -> None: # We need an admin user. self.login('iago') - req = dict(bot_creation_policy = ujson.dumps(Realm.BOT_CREATION_LIMIT_GENERIC_BOTS)) + req = dict(bot_creation_policy = orjson.dumps(Realm.BOT_CREATION_LIMIT_GENERIC_BOTS).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) invalid_add_bot_permission = 4 - req = dict(bot_creation_policy = ujson.dumps(invalid_add_bot_permission)) + req = dict(bot_creation_policy = orjson.dumps(invalid_add_bot_permission).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid bot_creation_policy') @@ -406,14 +406,14 @@ class RealmTest(ZulipTestCase): self.login_user(user_profile) invalid_value = 12 - req = dict(email_address_visibility = ujson.dumps(invalid_value)) + req = dict(email_address_visibility = orjson.dumps(invalid_value).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid email_address_visibility') reset_emails_in_zulip_realm() realm = get_realm("zulip") - req = dict(email_address_visibility = ujson.dumps(Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS)) + req = dict(email_address_visibility = orjson.dumps(Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) realm = get_realm("zulip") @@ -437,7 +437,7 @@ class RealmTest(ZulipTestCase): self.assertEqual(result.json()['user'].get('delivery_email'), hamlet.delivery_email) - req = dict(email_address_visibility = ujson.dumps(Realm.EMAIL_ADDRESS_VISIBILITY_NOBODY)) + req = dict(email_address_visibility = orjson.dumps(Realm.EMAIL_ADDRESS_VISIBILITY_NOBODY).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) @@ -457,48 +457,48 @@ class RealmTest(ZulipTestCase): def test_change_stream_creation_policy(self) -> None: # We need an admin user. self.login('iago') - req = dict(create_stream_policy = ujson.dumps(Realm.POLICY_ADMINS_ONLY)) + req = dict(create_stream_policy = orjson.dumps(Realm.POLICY_ADMINS_ONLY).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) invalid_value = 10 - req = dict(create_stream_policy = ujson.dumps(invalid_value)) + req = dict(create_stream_policy = orjson.dumps(invalid_value).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid create_stream_policy') def test_change_invite_to_stream_policy(self) -> None: # We need an admin user. self.login('iago') - req = dict(invite_to_stream_policy = ujson.dumps(Realm.POLICY_ADMINS_ONLY)) + req = dict(invite_to_stream_policy = orjson.dumps(Realm.POLICY_ADMINS_ONLY).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) invalid_value = 10 - req = dict(invite_to_stream_policy = ujson.dumps(invalid_value)) + req = dict(invite_to_stream_policy = orjson.dumps(invalid_value).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid invite_to_stream_policy') def test_user_group_edit_policy(self) -> None: # We need an admin user. self.login('iago') - req = dict(user_group_edit_policy = ujson.dumps(Realm.USER_GROUP_EDIT_POLICY_ADMINS)) + req = dict(user_group_edit_policy = orjson.dumps(Realm.USER_GROUP_EDIT_POLICY_ADMINS).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) invalid_value = 10 - req = dict(user_group_edit_policy = ujson.dumps(invalid_value)) + req = dict(user_group_edit_policy = orjson.dumps(invalid_value).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid user_group_edit_policy') def test_private_message_policy(self) -> None: # We need an admin user. self.login('iago') - req = dict(private_message_policy = ujson.dumps(Realm.PRIVATE_MESSAGE_POLICY_DISABLED)) + req = dict(private_message_policy = orjson.dumps(Realm.PRIVATE_MESSAGE_POLICY_DISABLED).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) invalid_value = 10 - req = dict(private_message_policy = ujson.dumps(invalid_value)) + req = dict(private_message_policy = orjson.dumps(invalid_value).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, 'Invalid private_message_policy') @@ -549,29 +549,29 @@ class RealmTest(ZulipTestCase): self.login('iago') invalid_video_chat_provider_value = 10 - req = {"video_chat_provider": ujson.dumps(invalid_video_chat_provider_value)} + req = {"video_chat_provider": orjson.dumps(invalid_video_chat_provider_value).decode()} result = self.client_patch('/json/realm', req) self.assert_json_error(result, ("Invalid video_chat_provider {}").format(invalid_video_chat_provider_value)) - req = {"video_chat_provider": ujson.dumps(Realm.VIDEO_CHAT_PROVIDERS['disabled']['id'])} + req = {"video_chat_provider": orjson.dumps(Realm.VIDEO_CHAT_PROVIDERS['disabled']['id']).decode()} result = self.client_patch('/json/realm', req) self.assert_json_success(result) self.assertEqual(get_realm('zulip').video_chat_provider, Realm.VIDEO_CHAT_PROVIDERS['disabled']['id']) - req = {"video_chat_provider": ujson.dumps(Realm.VIDEO_CHAT_PROVIDERS['jitsi_meet']['id'])} + req = {"video_chat_provider": orjson.dumps(Realm.VIDEO_CHAT_PROVIDERS['jitsi_meet']['id']).decode()} result = self.client_patch('/json/realm', req) self.assert_json_success(result) self.assertEqual(get_realm('zulip').video_chat_provider, Realm.VIDEO_CHAT_PROVIDERS['jitsi_meet']['id']) - req = {"video_chat_provider": ujson.dumps(Realm.VIDEO_CHAT_PROVIDERS['big_blue_button']['id'])} + req = {"video_chat_provider": orjson.dumps(Realm.VIDEO_CHAT_PROVIDERS['big_blue_button']['id']).decode()} result = self.client_patch('/json/realm', req) self.assert_json_success(result) self.assertEqual(get_realm('zulip').video_chat_provider, Realm.VIDEO_CHAT_PROVIDERS['big_blue_button']['id']) - req = {"video_chat_provider": ujson.dumps(Realm.VIDEO_CHAT_PROVIDERS['zoom']['id'])} + req = {"video_chat_provider": orjson.dumps(Realm.VIDEO_CHAT_PROVIDERS['zoom']['id']).decode()} result = self.client_patch('/json/realm', req) self.assert_json_success(result) @@ -629,45 +629,45 @@ class RealmTest(ZulipTestCase): realm = get_realm('zulip') self.assertEqual(realm.plan_type, Realm.SELF_HOSTED) - req = dict(message_retention_days=ujson.dumps(10)) + req = dict(message_retention_days=orjson.dumps(10).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, "Must be an organization owner") self.login('desdemona') - req = dict(message_retention_days=ujson.dumps(0)) + req = dict(message_retention_days=orjson.dumps(0).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, "Bad value for 'message_retention_days': 0") - req = dict(message_retention_days=ujson.dumps(-10)) + req = dict(message_retention_days=orjson.dumps(-10).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error( result, "Bad value for 'message_retention_days': -10") - req = dict(message_retention_days=ujson.dumps('invalid')) + req = dict(message_retention_days=orjson.dumps('invalid').decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, "Bad value for 'message_retention_days': invalid") - req = dict(message_retention_days=ujson.dumps(-1)) + req = dict(message_retention_days=orjson.dumps(-1).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error(result, "Bad value for 'message_retention_days': -1") - req = dict(message_retention_days=ujson.dumps('forever')) + req = dict(message_retention_days=orjson.dumps('forever').decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) - req = dict(message_retention_days=ujson.dumps(10)) + req = dict(message_retention_days=orjson.dumps(10).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) do_change_plan_type(realm, Realm.LIMITED) - req = dict(message_retention_days=ujson.dumps(10)) + req = dict(message_retention_days=orjson.dumps(10).decode()) result = self.client_patch('/json/realm', req) self.assert_json_error( result, "Available on Zulip Standard. Upgrade to access.") do_change_plan_type(realm, Realm.STANDARD) - req = dict(message_retention_days=ujson.dumps(10)) + req = dict(message_retention_days=orjson.dumps(10).decode()) result = self.client_patch('/json/realm', req) self.assert_json_success(result) @@ -683,7 +683,7 @@ class RealmAPITest(ZulipTestCase): realm.save(update_fields=[attr]) def update_with_api(self, name: str, value: int) -> Realm: - result = self.client_patch('/json/realm', {name: ujson.dumps(value)}) + result = self.client_patch('/json/realm', {name: orjson.dumps(value).decode()}) self.assert_json_success(result) return get_realm('zulip') # refresh data @@ -725,7 +725,7 @@ class RealmAPITest(ZulipTestCase): Realm.EMAIL_ADDRESS_VISIBILITY_NOBODY], video_chat_provider=[ dict( - video_chat_provider=ujson.dumps(Realm.VIDEO_CHAT_PROVIDERS['jitsi_meet']['id']), + video_chat_provider=orjson.dumps(Realm.VIDEO_CHAT_PROVIDERS['jitsi_meet']['id']).decode(), ), ], message_content_delete_limit_seconds=[1000, 1100, 1200] @@ -740,7 +740,7 @@ class RealmAPITest(ZulipTestCase): if name == 'video_chat_provider': self.set_up_db(name, vals[0][name]) realm = self.update_with_api_multiple_value(vals[0]) - self.assertEqual(getattr(realm, name), ujson.loads(vals[0][name])) + self.assertEqual(getattr(realm, name), orjson.loads(vals[0][name])) else: self.set_up_db(name, vals[0]) realm = self.update_with_api(name, vals[1]) diff --git a/zerver/tests/test_realm_domains.py b/zerver/tests/test_realm_domains.py index 65a64c8d7e..c9115ac954 100644 --- a/zerver/tests/test_realm_domains.py +++ b/zerver/tests/test_realm_domains.py @@ -1,4 +1,4 @@ -import ujson +import orjson from django.core.exceptions import ValidationError from django.db.utils import IntegrityError @@ -26,10 +26,10 @@ class RealmDomainTest(ZulipTestCase): RealmDomain.objects.create(realm=realm, domain='acme.com', allow_subdomains=True) result = self.client_get("/json/realm/domains") self.assert_json_success(result) - received = ujson.dumps(result.json()['domains'], sort_keys=True) - expected = ujson.dumps([{'domain': 'zulip.com', 'allow_subdomains': False}, - {'domain': 'acme.com', 'allow_subdomains': True}], - sort_keys=True) + received = orjson.dumps(result.json()['domains'], option=orjson.OPT_SORT_KEYS) + expected = orjson.dumps([{'domain': 'zulip.com', 'allow_subdomains': False}, + {'domain': 'acme.com', 'allow_subdomains': True}], + option=orjson.OPT_SORT_KEYS) self.assertEqual(received, expected) def test_not_realm_admin(self) -> None: @@ -43,12 +43,12 @@ class RealmDomainTest(ZulipTestCase): def test_create_realm_domain(self) -> None: self.login('iago') - data = {'domain': ujson.dumps(''), - 'allow_subdomains': ujson.dumps(True)} + data = {'domain': orjson.dumps('').decode(), + 'allow_subdomains': orjson.dumps(True).decode()} result = self.client_post("/json/realm/domains", info=data) self.assert_json_error(result, 'Invalid domain: Domain can\'t be empty.') - data['domain'] = ujson.dumps('acme.com') + data['domain'] = orjson.dumps('acme.com').decode() result = self.client_post("/json/realm/domains", info=data) self.assert_json_success(result) realm = get_realm('zulip') @@ -73,7 +73,7 @@ class RealmDomainTest(ZulipTestCase): RealmDomain.objects.create(realm=realm, domain='acme.com', allow_subdomains=False) data = { - 'allow_subdomains': ujson.dumps(True), + 'allow_subdomains': orjson.dumps(True).decode(), } url = "/json/realm/domains/acme.com" result = self.client_patch(url, data) diff --git a/zerver/tests/test_realm_export.py b/zerver/tests/test_realm_export.py index 17ae7de56b..6accd7bb63 100644 --- a/zerver/tests/test_realm_export.py +++ b/zerver/tests/test_realm_export.py @@ -2,7 +2,7 @@ import os from unittest.mock import patch import botocore.exceptions -import ujson +import orjson from django.conf import settings from django.utils.timezone import now as timezone_now @@ -63,7 +63,7 @@ class RealmExportTest(ZulipTestCase): self.assertEqual(audit_log_entry.acting_user_id, admin.id) # Test that the file is hosted, and the contents are as expected. - path_id = ujson.loads(audit_log_entry.extra_data).get('export_path') + path_id = orjson.loads(audit_log_entry.extra_data).get('export_path') self.assertIsNotNone(path_id) self.assertEqual(bucket.Object(path_id).get()['Body'].read(), b'zulip!') @@ -89,7 +89,7 @@ class RealmExportTest(ZulipTestCase): # Try to delete an export with a `deleted_timestamp` key. audit_log_entry.refresh_from_db() - export_data = ujson.loads(audit_log_entry.extra_data) + export_data = orjson.loads(audit_log_entry.extra_data) self.assertIn('deleted_timestamp', export_data) result = self.client_delete(f'/json/export/realm/{audit_log_entry.id}') self.assert_json_error(result, "Export already deleted") @@ -123,7 +123,7 @@ class RealmExportTest(ZulipTestCase): self.assertEqual(audit_log_entry.acting_user_id, admin.id) # Test that the file is hosted, and the contents are as expected. - path_id = ujson.loads(audit_log_entry.extra_data).get('export_path') + path_id = orjson.loads(audit_log_entry.extra_data).get('export_path') response = self.client_get(path_id) self.assertEqual(response.status_code, 200) self.assert_url_serves_contents_of_file(path_id, b'zulip!') @@ -149,7 +149,7 @@ class RealmExportTest(ZulipTestCase): # Try to delete an export with a `deleted_timestamp` key. audit_log_entry.refresh_from_db() - export_data = ujson.loads(audit_log_entry.extra_data) + export_data = orjson.loads(audit_log_entry.extra_data) self.assertIn('deleted_timestamp', export_data) result = self.client_delete(f'/json/export/realm/{audit_log_entry.id}') self.assert_json_error(result, "Export already deleted") diff --git a/zerver/tests/test_report.py b/zerver/tests/test_report.py index 5781d538e8..c41eefc643 100644 --- a/zerver/tests/test_report.py +++ b/zerver/tests/test_report.py @@ -1,7 +1,7 @@ from typing import Any, Callable, Dict, Iterable, List, Tuple from unittest import mock -import ujson +import orjson from django.test import override_settings from zerver.lib.test_classes import ZulipTestCase @@ -11,7 +11,7 @@ from zerver.lib.utils import statsd def fix_params(raw_params: Dict[str, Any]) -> Dict[str, str]: # A few of our few legacy endpoints need their # individual parameters serialized as JSON. - return {k: ujson.dumps(v) for k, v in raw_params.items()} + return {k: orjson.dumps(v).decode() for k, v in raw_params.items()} class StatsMock: def __init__(self, settings: Callable[..., Any]) -> None: diff --git a/zerver/tests/test_service_bot_system.py b/zerver/tests/test_service_bot_system.py index 2cf0d3a588..dd1179da14 100644 --- a/zerver/tests/test_service_bot_system.py +++ b/zerver/tests/test_service_bot_system.py @@ -1,7 +1,7 @@ from typing import Any, Mapping, Union from unittest import mock -import ujson +import orjson from django.conf import settings from django.test import override_settings @@ -241,14 +241,14 @@ class TestServiceBotStateHandler(ZulipTestCase): # Store some data. initial_dict = {'key 1': 'value 1', 'key 2': 'value 2', 'key 3': 'value 3'} params = { - 'storage': ujson.dumps(initial_dict), + 'storage': orjson.dumps(initial_dict).decode(), } result = self.client_put('/json/bot_storage', params) self.assert_json_success(result) # Assert the stored data for some keys. params = { - 'keys': ujson.dumps(['key 1', 'key 3']), + 'keys': orjson.dumps(['key 1', 'key 3']).decode(), } result = self.client_get('/json/bot_storage', params) self.assert_json_success(result) @@ -262,7 +262,7 @@ class TestServiceBotStateHandler(ZulipTestCase): # Store some more data; update an entry and store a new entry dict_update = {'key 1': 'new value', 'key 4': 'value 4'} params = { - 'storage': ujson.dumps(dict_update), + 'storage': orjson.dumps(dict_update).decode(), } result = self.client_put('/json/bot_storage', params) self.assert_json_success(result) @@ -282,13 +282,13 @@ class TestServiceBotStateHandler(ZulipTestCase): self.assert_json_error(result, 'Argument "keys" is not valid JSON.') params = { - 'keys': ujson.dumps(["key 1", "nonexistent key"]), + 'keys': orjson.dumps(["key 1", "nonexistent key"]).decode(), } result = self.client_get('/json/bot_storage', params) self.assert_json_error(result, "Key does not exist.") params = { - 'storage': ujson.dumps({'foo': [1, 2, 3]}), + 'storage': orjson.dumps({'foo': [1, 2, 3]}).decode(), } result = self.client_put('/json/bot_storage', params) self.assert_json_error(result, "storage contains a value that is not a string") @@ -296,7 +296,7 @@ class TestServiceBotStateHandler(ZulipTestCase): # Remove some entries. keys_to_remove = ['key 1', 'key 2'] params = { - 'keys': ujson.dumps(keys_to_remove), + 'keys': orjson.dumps(keys_to_remove).decode(), } result = self.client_delete('/json/bot_storage', params) self.assert_json_success(result) @@ -310,7 +310,7 @@ class TestServiceBotStateHandler(ZulipTestCase): # Try to remove an existing and a nonexistent key. params = { - 'keys': ujson.dumps(['key 3', 'nonexistent key']), + 'keys': orjson.dumps(['key 3', 'nonexistent key']).decode(), } result = self.client_delete('/json/bot_storage', params) self.assert_json_error(result, "Key does not exist.") diff --git a/zerver/tests/test_settings.py b/zerver/tests/test_settings.py index 0ae4090c8c..163830269b 100644 --- a/zerver/tests/test_settings.py +++ b/zerver/tests/test_settings.py @@ -2,7 +2,7 @@ import time from typing import Any, Dict from unittest import mock -import ujson +import orjson from django.http import HttpResponse from django.test import override_settings @@ -25,14 +25,14 @@ class ChangeSettingsTest(ZulipTestCase): self.login('hamlet') user_profile = self.example_user('hamlet') json_result = self.client_post(pattern, - {param: ujson.dumps(True)}) + {param: orjson.dumps(True).decode()}) self.assert_json_success(json_result) # refetch user_profile object to correctly handle caching user_profile = self.example_user('hamlet') self.assertEqual(getattr(user_profile, param), True) json_result = self.client_post(pattern, - {param: ujson.dumps(False)}) + {param: orjson.dumps(False).decode()}) self.assert_json_success(json_result) # refetch user_profile object to correctly handle caching user_profile = self.example_user('hamlet') @@ -44,14 +44,14 @@ class ChangeSettingsTest(ZulipTestCase): self.login('hamlet') user_profile = self.example_user('hamlet') json_result = self.client_patch(pattern, - {param: ujson.dumps(True)}) + {param: orjson.dumps(True).decode()}) self.assert_json_success(json_result) # refetch user_profile object to correctly handle caching user_profile = self.example_user('hamlet') self.assertEqual(getattr(user_profile, param), True) json_result = self.client_patch(pattern, - {param: ujson.dumps(False)}) + {param: orjson.dumps(False).decode()}) self.assert_json_success(json_result) # refetch user_profile object to correctly handle caching user_profile = self.example_user('hamlet') @@ -72,7 +72,7 @@ class ChangeSettingsTest(ZulipTestCase): new_password='foobar1', )) self.assert_json_success(json_result) - result = ujson.loads(json_result.content) + result = orjson.loads(json_result.content) self.check_well_formed_change_settings_response(result) user.refresh_from_db() @@ -177,11 +177,11 @@ class ChangeSettingsTest(ZulipTestCase): self.login_user(user_profile) json_result = self.client_patch(pattern, - {param: ujson.dumps("invalid")}) + {param: orjson.dumps("invalid").decode()}) self.assert_json_error(json_result, "Invalid notification sound 'invalid'") json_result = self.client_patch(pattern, - {param: ujson.dumps("ding")}) + {param: orjson.dumps("ding").decode()}) self.assert_json_success(json_result) # refetch user_profile object to correctly handle caching @@ -189,7 +189,7 @@ class ChangeSettingsTest(ZulipTestCase): self.assertEqual(getattr(user_profile, param), "ding") json_result = self.client_patch(pattern, - {param: ujson.dumps('zulip')}) + {param: orjson.dumps('zulip').decode()}) self.assert_json_success(json_result) # refetch user_profile object to correctly handle caching @@ -339,7 +339,7 @@ class ChangeSettingsTest(ZulipTestCase): invalid_value: Any = 100 else: invalid_value = 'invalid_' + setting_name - data = {setting_name: ujson.dumps(test_value)} + data = {setting_name: orjson.dumps(test_value).decode()} result = self.client_patch("/json/settings/display", data) self.assert_json_success(result) @@ -348,7 +348,7 @@ class ChangeSettingsTest(ZulipTestCase): # Test to make sure invalid settings are not accepted # and saved in the db. - data = {setting_name: ujson.dumps(invalid_value)} + data = {setting_name: orjson.dumps(invalid_value).decode()} result = self.client_patch("/json/settings/display", data) # the json error for multiple word setting names (ex: default_language) @@ -366,7 +366,7 @@ class ChangeSettingsTest(ZulipTestCase): def do_change_emojiset(self, emojiset: str) -> HttpResponse: self.login('hamlet') - data = {'emojiset': ujson.dumps(emojiset)} + data = {'emojiset': orjson.dumps(emojiset).decode()} result = self.client_patch("/json/settings/display", data) return result diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.py index 037134f21e..9eb94d821e 100644 --- a/zerver/tests/test_signup.py +++ b/zerver/tests/test_signup.py @@ -6,7 +6,7 @@ import urllib from typing import Any, List, Optional, Sequence from unittest.mock import MagicMock, patch -import ujson +import orjson from django.conf import settings from django.contrib.auth.views import INTERNAL_RESET_URL_TOKEN from django.contrib.contenttypes.models import ContentType @@ -816,7 +816,7 @@ class InviteUserBase(ZulipTestCase): stream_ids.append(self.get_stream_id(stream_name)) return self.client_post("/json/invites", {"invitee_emails": invitee_emails, - "stream_ids": ujson.dumps(stream_ids), + "stream_ids": orjson.dumps(stream_ids).decode(), "invite_as": invite_as}) class InviteUserTest(InviteUserBase): @@ -1706,7 +1706,7 @@ class InvitationsTestCase(InviteUserBase): result = self.client_get("/json/invites") self.assertEqual(result.status_code, 200) - invites = ujson.loads(result.content)["invites"] + invites = orjson.loads(result.content)["invites"] self.assertEqual(len(invites), 2) self.assertFalse(invites[0]["is_multiuse"]) @@ -2137,7 +2137,7 @@ class MultiuseInviteTest(ZulipTestCase): stream_ids = [stream.id for stream in streams] result = self.client_post('/json/invites/multiuse', - {"stream_ids": ujson.dumps(stream_ids)}) + {"stream_ids": orjson.dumps(stream_ids).decode()}) self.assert_json_success(result) invite_link = result.json()["invite_link"] @@ -2164,12 +2164,12 @@ class MultiuseInviteTest(ZulipTestCase): def test_multiuse_link_for_inviting_as_owner(self) -> None: self.login('iago') result = self.client_post('/json/invites/multiuse', - {"invite_as": ujson.dumps(PreregistrationUser.INVITE_AS['REALM_OWNER'])}) + {"invite_as": orjson.dumps(PreregistrationUser.INVITE_AS['REALM_OWNER']).decode()}) self.assert_json_error(result, "Must be an organization owner") self.login('desdemona') result = self.client_post('/json/invites/multiuse', - {"invite_as": ujson.dumps(PreregistrationUser.INVITE_AS['REALM_OWNER'])}) + {"invite_as": orjson.dumps(PreregistrationUser.INVITE_AS['REALM_OWNER']).decode()}) self.assert_json_success(result) invite_link = result.json()["invite_link"] @@ -2178,7 +2178,7 @@ class MultiuseInviteTest(ZulipTestCase): def test_create_multiuse_link_invalid_stream_api_call(self) -> None: self.login('iago') result = self.client_post('/json/invites/multiuse', - {"stream_ids": ujson.dumps([54321])}) + {"stream_ids": orjson.dumps([54321]).decode()}) self.assert_json_error(result, "Invalid stream id 54321. No invites were sent.") class EmailUnsubscribeTests(ZulipTestCase): diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index d4969fe20c..53a36f232d 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Iterator, List, Optional, Set, Tuple from unittest import mock from unittest.mock import ANY, call -import ujson +import orjson from django.conf import settings from django.utils.timezone import now as timezone_now @@ -723,13 +723,13 @@ class SlackImporter(ZulipTestCase): messages_file_2 = os.path.join(output_dir, 'messages-000002.json') self.assertTrue(os.path.exists(messages_file_2)) - with open(messages_file_1) as f: - message_json = ujson.load(f) + with open(messages_file_1, "rb") as f: + message_json = orjson.loads(f.read()) self.assertEqual(message_json['zerver_message'], zerver_message[:1]) self.assertEqual(message_json['zerver_usermessage'], zerver_usermessage[:2]) - with open(messages_file_2) as f: - message_json = ujson.load(f) + with open(messages_file_2, "rb") as f: + message_json = orjson.loads(f.read()) self.assertEqual(message_json['zerver_message'], zerver_message[1:2]) self.assertEqual(message_json['zerver_usermessage'], zerver_usermessage[2:5]) @@ -764,8 +764,8 @@ class SlackImporter(ZulipTestCase): # Also the unzipped data file should be removed if the test fails at 'do_convert_data' self.rm_tree(test_slack_unzipped_file) - user_data_fixture = ujson.loads(self.fixture_data('user_data.json', type='slack_fixtures')) - team_info_fixture = ujson.loads(self.fixture_data('team_info.json', type='slack_fixtures')) + user_data_fixture = orjson.loads(self.fixture_data('user_data.json', type='slack_fixtures')) + team_info_fixture = orjson.loads(self.fixture_data('team_info.json', type='slack_fixtures')) mock_get_slack_api_data.side_effect = [user_data_fixture['members'], {}, team_info_fixture["team"]] mock_requests_get.return_value.raw = get_test_image_file("img.png") @@ -779,8 +779,8 @@ class SlackImporter(ZulipTestCase): realm_icon_records_path = os.path.join(realm_icons_path, 'records.json') self.assertTrue(os.path.exists(realm_icon_records_path)) - with open(realm_icon_records_path) as f: - records = ujson.load(f) + with open(realm_icon_records_path, "rb") as f: + records = orjson.loads(f.read()) self.assertEqual(len(records), 2) self.assertEqual(records[0]["path"], "0/icon.original") self.assertTrue(os.path.exists(os.path.join(realm_icons_path, records[0]["path"]))) diff --git a/zerver/tests/test_slack_message_conversion.py b/zerver/tests/test_slack_message_conversion.py index 870723eecd..103e9e32f0 100644 --- a/zerver/tests/test_slack_message_conversion.py +++ b/zerver/tests/test_slack_message_conversion.py @@ -1,7 +1,7 @@ import os from typing import Any, Dict, List, Tuple -import ujson +import orjson from zerver.data_import.slack_message_conversion import ( convert_to_zulip_markdown, @@ -23,8 +23,8 @@ class SlackMessageConversion(ZulipTestCase): def load_slack_message_conversion_tests(self) -> Dict[Any, Any]: test_fixtures = {} with open(os.path.join(os.path.dirname(__file__), - 'fixtures/slack_message_conversion.json')) as f: - data = ujson.load(f) + 'fixtures/slack_message_conversion.json'), "rb") as f: + data = orjson.loads(f.read()) for test in data['regular_tests']: test_fixtures[test['name']] = test diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py index d4a2a21eee..406111e660 100644 --- a/zerver/tests/test_subs.py +++ b/zerver/tests/test_subs.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Union from unittest import mock from urllib.parse import urlencode -import ujson +import orjson from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse @@ -185,9 +185,9 @@ class TestCreateStreams(ZulipTestCase): user = self.example_user("hamlet") realm = user.realm self.login_user(user) - post_data = {'subscriptions': ujson.dumps([{"name": 'new_stream', - "description": "multi\nline\ndescription"}]), - 'invite_only': ujson.dumps(False)} + post_data = {'subscriptions': orjson.dumps([{"name": 'new_stream', + "description": "multi\nline\ndescription"}]).decode(), + 'invite_only': orjson.dumps(False).decode()} result = self.api_post(user, "/api/v1/users/me/subscriptions", post_data, subdomain="zulip") self.assert_json_success(result) @@ -284,7 +284,7 @@ class TestCreateStreams(ZulipTestCase): "history_public_to_subscribers": 'true', "invite_only": 'false', "announce": 'true', - "principals": ujson.dumps([iago.id, aaron.id, cordelia.id, hamlet.id]), + "principals": orjson.dumps([iago.id, aaron.id, cordelia.id, hamlet.id]).decode(), "stream_post_policy": '1', } @@ -305,7 +305,7 @@ class TestCreateStreams(ZulipTestCase): "already_subscribed": {}, } self.assertEqual(response.status_code, 200) - self.assertEqual(ujson.loads(response.content.decode()), expected_response) + self.assertEqual(orjson.loads(response.content.decode()), expected_response) # 2 messages should be created, one in announce and one in the new stream itself. self.assertEqual(final_message_count - initial_message_count, 2) @@ -353,8 +353,8 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) params = { - 'stream_name': ujson.dumps('private_stream'), - 'is_private': ujson.dumps(False), + 'stream_name': orjson.dumps('private_stream').decode(), + 'is_private': orjson.dumps(False).decode(), } stream_id = get_stream('private_stream', user_profile.realm).id result = self.client_patch(f"/json/streams/{stream_id}", params) @@ -365,8 +365,8 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) params = { - 'stream_name': ujson.dumps('private_stream'), - 'is_private': ujson.dumps(False), + 'stream_name': orjson.dumps('private_stream').decode(), + 'is_private': orjson.dumps(False).decode(), } result = self.client_patch(f"/json/streams/{stream_id}", params) self.assert_json_success(result) @@ -384,8 +384,8 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) params = { - 'stream_name': ujson.dumps('public_stream'), - 'is_private': ujson.dumps(True), + 'stream_name': orjson.dumps('public_stream').decode(), + 'is_private': orjson.dumps(True).decode(), } stream_id = get_stream('public_stream', realm).id result = self.client_patch(f"/json/streams/{stream_id}", params) @@ -403,8 +403,8 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) params = { - 'stream_name': ujson.dumps('target_stream'), - 'is_private': ujson.dumps(False), + 'stream_name': orjson.dumps('target_stream').decode(), + 'is_private': orjson.dumps(False).decode(), } stream_id = get_stream('target_stream', realm).id result = self.client_patch(f"/json/streams/{stream_id}", params, @@ -422,9 +422,9 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) params = { - 'stream_name': ujson.dumps('public_history_stream'), - 'is_private': ujson.dumps(True), - 'history_public_to_subscribers': ujson.dumps(True), + 'stream_name': orjson.dumps('public_history_stream').decode(), + 'is_private': orjson.dumps(True).decode(), + 'history_public_to_subscribers': orjson.dumps(True).decode(), } stream_id = get_stream('public_history_stream', realm).id result = self.client_patch(f"/json/streams/{stream_id}", params) @@ -441,9 +441,9 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) params = { - 'stream_name': ujson.dumps('public_stream'), - 'is_private': ujson.dumps(False), - 'history_public_to_subscribers': ujson.dumps(False), + 'stream_name': orjson.dumps('public_stream').decode(), + 'is_private': orjson.dumps(False).decode(), + 'history_public_to_subscribers': orjson.dumps(False).decode(), } stream_id = get_stream('public_stream', realm).id result = self.client_patch(f"/json/streams/{stream_id}", params) @@ -541,7 +541,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = get_stream('private_stream', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'description': ujson.dumps('Test description')}) + {'description': orjson.dumps('Test description').decode()}) self.assert_json_success(result) # Should be just a description change event self.assert_length(events, 1) @@ -558,7 +558,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = get_stream('private_stream', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('whatever')}) + {'new_name': orjson.dumps('whatever').decode()}) self.assert_json_success(result) # Should be a name event, an email address event and a notification event self.assert_length(events, 3) @@ -584,25 +584,25 @@ class StreamAdminTest(ZulipTestCase): do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) result = self.client_patch(f'/json/streams/{stream.id}', - {'new_name': ujson.dumps('stream_name1')}) + {'new_name': orjson.dumps('stream_name1').decode()}) self.assert_json_error(result, "Stream already has that name!") result = self.client_patch(f'/json/streams/{stream.id}', - {'new_name': ujson.dumps('Denmark')}) + {'new_name': orjson.dumps('Denmark').decode()}) self.assert_json_error(result, "Stream name 'Denmark' is already taken.") result = self.client_patch(f'/json/streams/{stream.id}', - {'new_name': ujson.dumps('denmark ')}) + {'new_name': orjson.dumps('denmark ').decode()}) self.assert_json_error(result, "Stream name 'denmark' is already taken.") # Do a rename that is case-only--this should succeed. result = self.client_patch(f'/json/streams/{stream.id}', - {'new_name': ujson.dumps('sTREAm_name1')}) + {'new_name': orjson.dumps('sTREAm_name1').decode()}) self.assert_json_success(result) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): stream_id = get_stream('stream_name1', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('stream_name2')}) + {'new_name': orjson.dumps('stream_name2').decode()}) self.assert_json_success(result) event = events[1]['event'] self.assertEqual(event, dict( @@ -633,7 +633,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = stream_name2_exists.id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('नया नाम')}) + {'new_name': orjson.dumps('नया नाम').decode()}) self.assert_json_success(result) # While querying, system can handle unicode strings. stream_name_uni_exists = get_stream('नया नाम', realm) @@ -645,7 +645,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = stream_name_uni_exists.id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('नाम में क्या रक्खा हे')}) + {'new_name': orjson.dumps('नाम में क्या रक्खा हे').decode()}) self.assert_json_success(result) # While querying, system can handle unicode strings. self.assertRaises(Stream.DoesNotExist, get_stream, 'नया नाम', realm) @@ -657,7 +657,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = stream_name_new_uni_exists.id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('français')}) + {'new_name': orjson.dumps('français').decode()}) self.assert_json_success(result) stream_name_fr_exists = get_stream('français', realm) self.assertTrue(stream_name_fr_exists) @@ -666,7 +666,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = stream_name_fr_exists.id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('français name')}) + {'new_name': orjson.dumps('français name').decode()}) self.assert_json_success(result) stream_name_mixed_exists = get_stream('français name', realm) self.assertTrue(stream_name_mixed_exists) @@ -678,7 +678,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = get_stream('stream_private_name1', realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('stream_private_name2')}) + {'new_name': orjson.dumps('stream_private_name2').decode()}) self.assert_json_success(result) notified_user_ids = set(events[1]['users']) self.assertEqual(notified_user_ids, can_access_stream_user_ids(stream_private)) @@ -697,7 +697,7 @@ class StreamAdminTest(ZulipTestCase): stream_id = get_stream('stream_name1', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('stream_name2')}) + {'new_name': orjson.dumps('stream_name2').decode()}) self.assert_json_error(result, 'Must be an organization administrator') def test_notify_on_stream_rename(self) -> None: @@ -708,7 +708,7 @@ class StreamAdminTest(ZulipTestCase): stream = self.subscribe(user_profile, 'stream_name1') do_change_user_role(user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR) result = self.client_patch(f'/json/streams/{stream.id}', - {'new_name': ujson.dumps('stream_name2')}) + {'new_name': orjson.dumps('stream_name2').decode()}) self.assert_json_success(result) # Inspect the notification message sent @@ -728,23 +728,23 @@ class StreamAdminTest(ZulipTestCase): self.login_user(iago) result = self.common_subscribe_to_streams(iago, ["private_stream"], - dict(principals=ujson.dumps([hamlet.id])), + dict(principals=orjson.dumps([hamlet.id]).decode()), invite_only=True) self.assert_json_success(result) stream_id = get_stream('private_stream', iago.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'new_name': ujson.dumps('new_private_stream')}) + {'new_name': orjson.dumps('new_private_stream').decode()}) self.assert_json_success(result) result = self.client_patch(f'/json/streams/{stream_id}', - {'new_description': ujson.dumps('new description')}) + {'new_description': orjson.dumps('new description').decode()}) self.assert_json_success(result) # But cannot change stream type. result = self.client_patch(f'/json/streams/{stream_id}', - {'stream_name': ujson.dumps('private_stream'), - 'is_private': ujson.dumps(True)}) + {'stream_name': orjson.dumps('private_stream').decode(), + 'is_private': orjson.dumps(True).decode()}) self.assert_json_error(result, "Invalid stream id") def test_change_stream_description(self) -> None: @@ -757,7 +757,7 @@ class StreamAdminTest(ZulipTestCase): with tornado_redirected_to_list(events): stream_id = get_stream('stream_name1', realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'description': ujson.dumps('Test description')}) + {'description': orjson.dumps('Test description').decode()}) self.assert_json_success(result) event = events[0]['event'] @@ -784,14 +784,14 @@ class StreamAdminTest(ZulipTestCase): self.assertEqual('Test description', stream.description) result = self.client_patch(f'/json/streams/{stream_id}', - {'description': ujson.dumps('a' * 1025)}) + {'description': orjson.dumps('a' * 1025).decode()}) self.assert_json_error( result, f"description is too long (limit: {Stream.MAX_DESCRIPTION_LENGTH} characters)", ) result = self.client_patch(f'/json/streams/{stream_id}', - {'description': ujson.dumps('a\nmulti\nline\ndescription')}) + {'description': orjson.dumps('a\nmulti\nline\ndescription').decode()}) self.assert_json_success(result) stream = get_stream('stream_name1', realm) self.assertEqual(stream.description, 'a multi line description') @@ -799,7 +799,7 @@ class StreamAdminTest(ZulipTestCase): # Verify that we don't render inline URL previews in this code path. with self.settings(INLINE_URL_EMBED_PREVIEW=True): result = self.client_patch(f'/json/streams/{stream_id}', - {'description': ujson.dumps('See https://zulip.com/team')}) + {'description': orjson.dumps('See https://zulip.com/team').decode()}) self.assert_json_success(result) stream = get_stream('stream_name1', realm) self.assertEqual( @@ -816,7 +816,7 @@ class StreamAdminTest(ZulipTestCase): stream_id = get_stream('stream_name1', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'description': ujson.dumps('Test description')}) + {'description': orjson.dumps('Test description').decode()}) self.assert_json_error(result, 'Must be an organization administrator') def test_change_to_stream_post_policy_admins(self) -> None: @@ -828,7 +828,7 @@ class StreamAdminTest(ZulipTestCase): stream_id = get_stream('stream_name1', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'is_announcement_only': ujson.dumps(True)}) + {'is_announcement_only': orjson.dumps(True).decode()}) self.assert_json_success(result) stream = get_stream('stream_name1', user_profile.realm) self.assertTrue(stream.stream_post_policy == Stream.STREAM_POST_POLICY_ADMINS) @@ -848,7 +848,7 @@ class StreamAdminTest(ZulipTestCase): self.assertEqual(user_profile.is_new_member, is_new) stream_id = get_stream('stream_name1', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'stream_post_policy': ujson.dumps(policy)}) + {'stream_post_policy': orjson.dumps(policy).decode()}) self.assert_json_error(result, 'Must be an organization administrator') policies = [Stream.STREAM_POST_POLICY_ADMINS, Stream.STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS] @@ -862,7 +862,7 @@ class StreamAdminTest(ZulipTestCase): for policy in policies: stream_id = get_stream('stream_name1', user_profile.realm).id result = self.client_patch(f'/json/streams/{stream_id}', - {'stream_post_policy': ujson.dumps(policy)}) + {'stream_post_policy': orjson.dumps(policy).decode()}) self.assert_json_success(result) stream = get_stream('stream_name1', user_profile.realm) self.assertEqual(stream.stream_post_policy, policy) @@ -875,14 +875,14 @@ class StreamAdminTest(ZulipTestCase): stream = self.subscribe(user_profile, 'stream_name1') result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps(2)}) + {'message_retention_days': orjson.dumps(2).decode()}) self.assert_json_error(result, "Available on Zulip Standard. Upgrade to access.") do_change_plan_type(realm, Realm.SELF_HOSTED) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps(2)}) + {'message_retention_days': orjson.dumps(2).decode()}) self.assert_json_success(result) event = events[0]['event'] @@ -906,7 +906,7 @@ class StreamAdminTest(ZulipTestCase): events = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps("forever")}) + {'message_retention_days': orjson.dumps("forever").decode()}) self.assert_json_success(result) event = events[0]['event'] self.assertEqual(event, dict( @@ -924,7 +924,7 @@ class StreamAdminTest(ZulipTestCase): events = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps("realm_default")}) + {'message_retention_days': orjson.dumps("realm_default").decode()}) self.assert_json_success(result) event = events[0]['event'] self.assertEqual(event, dict( @@ -939,15 +939,15 @@ class StreamAdminTest(ZulipTestCase): self.assertEqual(stream.message_retention_days, None) result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps("invalid")}) + {'message_retention_days': orjson.dumps("invalid").decode()}) self.assert_json_error(result, "Bad value for 'message_retention_days': invalid") result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps(-1)}) + {'message_retention_days': orjson.dumps(-1).decode()}) self.assert_json_error(result, "Bad value for 'message_retention_days': -1") result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps(0)}) + {'message_retention_days': orjson.dumps(0).decode()}) self.assert_json_error(result, "Bad value for 'message_retention_days': 0") def test_change_stream_message_retention_days_requires_realm_owner(self) -> None: @@ -957,12 +957,12 @@ class StreamAdminTest(ZulipTestCase): stream = self.subscribe(user_profile, 'stream_name1') result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps(2)}) + {'message_retention_days': orjson.dumps(2).decode()}) self.assert_json_error(result, "Must be an organization owner") do_change_user_role(user_profile, UserProfile.ROLE_REALM_OWNER) result = self.client_patch(f'/json/streams/{stream.id}', - {'message_retention_days': ujson.dumps(2)}) + {'message_retention_days': orjson.dumps(2).decode()}) self.assert_json_success(result) stream = get_stream('stream_name1', realm) self.assertEqual(stream.message_retention_days, 2) @@ -1090,7 +1090,7 @@ class StreamAdminTest(ZulipTestCase): # Even if you could guess the new name, you can't subscribe to it. result = self.client_post( "/json/users/me/subscriptions", - {"subscriptions": ujson.dumps([{"name": deactivated_stream_name}])}) + {"subscriptions": orjson.dumps([{"name": deactivated_stream_name}]).decode()}) self.assert_json_error( result, f"Unable to access stream ({deactivated_stream_name}).") @@ -1178,8 +1178,8 @@ class StreamAdminTest(ZulipTestCase): with queries_captured() as queries: result = self.client_delete( "/json/users/me/subscriptions", - {"subscriptions": ujson.dumps([stream_name]), - "principals": ujson.dumps(principals)}) + {"subscriptions": orjson.dumps([stream_name]).decode(), + "principals": orjson.dumps(principals).decode()}) self.assert_length(queries, query_count) # If the removal succeeded, then assert that Cordelia is no longer subscribed. @@ -1373,7 +1373,7 @@ class StreamAdminTest(ZulipTestCase): result = self.common_subscribe_to_streams( hamlet_user, stream_name, - {"principals": ujson.dumps([cordelia_user_id])}, + {"principals": orjson.dumps([cordelia_user_id]).decode()}, allow_fail=True, ) self.assert_json_error(result, @@ -1383,7 +1383,7 @@ class StreamAdminTest(ZulipTestCase): do_set_realm_property(hamlet_user.realm, 'waiting_period_threshold', 0) # Attempt and succeed to invite Cordelia to the stream.. - self.common_subscribe_to_streams(hamlet_user, stream_name, {"principals": ujson.dumps([cordelia_user_id])}) + self.common_subscribe_to_streams(hamlet_user, stream_name, {"principals": orjson.dumps([cordelia_user_id]).decode()}) # Set threshold to 20 days.. do_set_realm_property(hamlet_user.realm, 'waiting_period_threshold', 20) @@ -1394,7 +1394,7 @@ class StreamAdminTest(ZulipTestCase): self.unsubscribe(cordelia_user, stream_name[0]) # Attempt and succeed to invite Aaron to the stream.. - self.common_subscribe_to_streams(hamlet_user, stream_name, {"principals": ujson.dumps([cordelia_user_id])}) + self.common_subscribe_to_streams(hamlet_user, stream_name, {"principals": orjson.dumps([cordelia_user_id]).decode()}) def test_remove_already_not_subbed(self) -> None: """ @@ -1420,8 +1420,8 @@ class StreamAdminTest(ZulipTestCase): self.make_stream(stream_name) result = self.client_delete("/json/users/me/subscriptions", - {"subscriptions": ujson.dumps([stream_name]), - "principals": ujson.dumps([99])}) + {"subscriptions": orjson.dumps([stream_name]).decode(), + "principals": orjson.dumps([99]).decode()}) self.assert_json_error( result, "User not authorized to execute queries on behalf of '99'", @@ -1625,7 +1625,7 @@ class DefaultStreamGroupTest(ZulipTestCase): result = self.client_post('/json/default_stream_groups/create', {"group_name": group_name, "description": description, - "stream_names": ujson.dumps(stream_names)}) + "stream_names": orjson.dumps(stream_names).decode()}) self.assert_json_success(result) default_stream_groups = get_default_stream_groups(realm) self.assert_length(default_stream_groups, 1) @@ -1636,7 +1636,7 @@ class DefaultStreamGroupTest(ZulipTestCase): # Try adding the same streams to the group. result = self.client_post('/json/default_stream_groups/create', {"group_name": group_name, "description": description, - "stream_names": ujson.dumps(stream_names)}) + "stream_names": orjson.dumps(stream_names).decode()}) self.assert_json_error(result, "Default stream group 'group1' already exists") # Test adding streams to existing default stream group @@ -1649,15 +1649,15 @@ class DefaultStreamGroupTest(ZulipTestCase): streams.append(new_stream) result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"stream_names": ujson.dumps(new_stream_names)}) + {"stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, "Missing 'op' argument") result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "invalid", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "invalid", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, 'Invalid value for "op". Specify one of "add" or "remove".') result = self.client_patch("/json/default_stream_groups/12345/streams", - {"op": "add", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "add", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, "Default stream group with id '12345' does not exist.") result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", {"op": "add"}) @@ -1665,12 +1665,12 @@ class DefaultStreamGroupTest(ZulipTestCase): do_add_default_stream(new_streams[0]) result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "add", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "add", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, "'stream4' is a default stream and cannot be added to 'group1'") do_remove_default_stream(new_streams[0]) result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "add", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "add", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_success(result) default_stream_groups = get_default_stream_groups(realm) self.assert_length(default_stream_groups, 1) @@ -1678,22 +1678,22 @@ class DefaultStreamGroupTest(ZulipTestCase): self.assertEqual(list(default_stream_groups[0].streams.all().order_by('name')), streams) result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "add", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "add", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, "Stream 'stream4' is already present in default stream group 'group1'") # Test removing streams from default stream group result = self.client_patch("/json/default_stream_groups/12345/streams", - {"op": "remove", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "remove", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, "Default stream group with id '12345' does not exist.") result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "remove", "stream_names": ujson.dumps(["random stream name"])}) + {"op": "remove", "stream_names": orjson.dumps(["random stream name"]).decode()}) self.assert_json_error(result, "Invalid stream name 'random stream name'") streams.remove(new_streams[0]) result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "remove", "stream_names": ujson.dumps([new_stream_names[0]])}) + {"op": "remove", "stream_names": orjson.dumps([new_stream_names[0]]).decode()}) self.assert_json_success(result) default_stream_groups = get_default_stream_groups(realm) self.assert_length(default_stream_groups, 1) @@ -1701,7 +1701,7 @@ class DefaultStreamGroupTest(ZulipTestCase): self.assertEqual(list(default_stream_groups[0].streams.all().order_by('name')), streams) result = self.client_patch(f"/json/default_stream_groups/{group_id}/streams", - {"op": "remove", "stream_names": ujson.dumps(new_stream_names)}) + {"op": "remove", "stream_names": orjson.dumps(new_stream_names).decode()}) self.assert_json_error(result, "Stream 'stream4' is not present in default stream group 'group1'") # Test changing description of default stream group @@ -1712,13 +1712,13 @@ class DefaultStreamGroupTest(ZulipTestCase): self.assert_json_error(result, 'You must pass "new_description" or "new_group_name".') result = self.client_patch("/json/default_stream_groups/12345", - {"op": "change", "new_description": ujson.dumps(new_description)}) + {"op": "change", "new_description": orjson.dumps(new_description).decode()}) self.assert_json_error(result, "Default stream group with id '12345' does not exist.") result = self.client_patch(f"/json/default_stream_groups/{group_id}", {"group_name": group_name, "op": "change", - "new_description": ujson.dumps(new_description)}) + "new_description": orjson.dumps(new_description).decode()}) self.assert_json_success(result) default_stream_groups = get_default_stream_groups(realm) self.assert_length(default_stream_groups, 1) @@ -1729,17 +1729,17 @@ class DefaultStreamGroupTest(ZulipTestCase): new_group_name = "new group1" do_create_default_stream_group(realm, "group2", "", []) result = self.client_patch(f"/json/default_stream_groups/{group_id}", - {"op": "change", "new_group_name": ujson.dumps("group2")}) + {"op": "change", "new_group_name": orjson.dumps("group2").decode()}) self.assert_json_error(result, "Default stream group 'group2' already exists") new_group = lookup_default_stream_groups(["group2"], realm)[0] do_remove_default_stream_group(realm, new_group) result = self.client_patch(f"/json/default_stream_groups/{group_id}", - {"op": "change", "new_group_name": ujson.dumps(group_name)}) + {"op": "change", "new_group_name": orjson.dumps(group_name).decode()}) self.assert_json_error(result, "This default stream group is already named 'group1'") result = self.client_patch(f"/json/default_stream_groups/{group_id}", - {"op": "change", "new_group_name": ujson.dumps(new_group_name)}) + {"op": "change", "new_group_name": orjson.dumps(new_group_name).decode()}) self.assert_json_success(result) default_stream_groups = get_default_stream_groups(realm) self.assert_length(default_stream_groups, 1) @@ -1770,18 +1770,18 @@ class DefaultStreamGroupTest(ZulipTestCase): result = self.client_post('/json/default_stream_groups/create', {"group_name": "", "description": description, - "stream_names": ujson.dumps(stream_names)}) + "stream_names": orjson.dumps(stream_names).decode()}) self.assert_json_error(result, "Invalid default stream group name ''") result = self.client_post('/json/default_stream_groups/create', {"group_name": 'x'*100, "description": description, - "stream_names": ujson.dumps(stream_names)}) + "stream_names": orjson.dumps(stream_names).decode()}) self.assert_json_error(result, "Default stream group name too long (limit: {} characters)" .format(DefaultStreamGroup.MAX_NAME_LENGTH)) result = self.client_post('/json/default_stream_groups/create', {"group_name": "abc\000", "description": description, - "stream_names": ujson.dumps(stream_names)}) + "stream_names": orjson.dumps(stream_names).decode()}) self.assert_json_error(result, "Default stream group name 'abc\000' contains NULL (0x00) characters.") # Also test that lookup_default_stream_groups raises an @@ -1806,9 +1806,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): stream_id = sub['stream_id'] new_color = "#ffffff" # TODO: ensure that this is different from old_color result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "color", - "stream_id": stream_id, - "value": "#ffffff"}])}) + {"subscription_data": orjson.dumps([{"property": "color", + "stream_id": stream_id, + "value": "#ffffff"}]).decode()}) self.assert_json_success(result) new_subs = gather_subscriptions(test_user)[0] @@ -1831,9 +1831,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): invalid_color = "3ffrff" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "color", - "stream_id": stream_id, - "value": invalid_color}])}) + {"subscription_data": orjson.dumps([{"property": "color", + "stream_id": stream_id, + "value": invalid_color}]).decode()}) self.assert_json_error(result, "color is not a valid hex color code") def test_set_color_missing_stream_id(self) -> None: @@ -1843,8 +1843,8 @@ class SubscriptionPropertiesTest(ZulipTestCase): test_user = self.example_user('hamlet') self.login_user(test_user) result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "color", - "value": "#ffffff"}])}) + {"subscription_data": orjson.dumps([{"property": "color", + "value": "#ffffff"}]).decode()}) self.assert_json_error( result, "stream_id key is missing from subscription_data[0]") @@ -1858,9 +1858,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): subscribed, unsubscribed, never_subscribed = gather_subscriptions_helper(test_user) not_subbed = unsubscribed + never_subscribed result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "color", - "stream_id": not_subbed[0]["stream_id"], - "value": "#ffffff"}])}) + {"subscription_data": orjson.dumps([{"property": "color", + "stream_id": not_subbed[0]["stream_id"], + "value": "#ffffff"}]).decode()}) self.assert_json_error( result, "Not subscribed to stream id {}".format(not_subbed[0]["stream_id"])) @@ -1872,8 +1872,8 @@ class SubscriptionPropertiesTest(ZulipTestCase): self.login_user(test_user) subs = gather_subscriptions(test_user)[0] result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "color", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": "color", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error( result, "value key is missing from subscription_data[0]") @@ -1888,9 +1888,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): subs = gather_subscriptions(test_user)[0] sub = subs[0] result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "wildcard_mentions_notify", - "stream_id": sub["stream_id"], - "value": True}])}) + {"subscription_data": orjson.dumps([{"property": "wildcard_mentions_notify", + "stream_id": sub["stream_id"], + "value": True}]).decode()}) self.assert_json_success(result) @@ -1911,9 +1911,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): stream_id = sub['stream_id'] new_pin_to_top = not sub['pin_to_top'] result = self.api_post(user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "pin_to_top", - "stream_id": stream_id, - "value": new_pin_to_top}])}) + {"subscription_data": orjson.dumps([{"property": "pin_to_top", + "stream_id": stream_id, + "value": new_pin_to_top}]).decode()}) self.assert_json_success(result) updated_sub = get_subscription(sub['name'], user) @@ -1935,9 +1935,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): property_name = "is_muted" with tornado_redirected_to_list(events): result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": True, - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": True, + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_success(result) self.assert_length(events, 1) self.assertEqual(events[0]['event']['property'], 'in_home_view') @@ -1951,9 +1951,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): legacy_property_name = 'in_home_view' with tornado_redirected_to_list(events): result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": legacy_property_name, - "value": True, - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": legacy_property_name, + "value": True, + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_success(result) self.assert_length(events, 1) self.assertEqual(events[0]['event']['property'], 'in_home_view') @@ -1967,9 +1967,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): events = [] with tornado_redirected_to_list(events): result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": legacy_property_name, - "value": False, - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": legacy_property_name, + "value": False, + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_success(result) self.assert_length(events, 1) self.assertEqual(events[0]['event']['property'], 'in_home_view') @@ -1990,66 +1990,66 @@ class SubscriptionPropertiesTest(ZulipTestCase): property_name = "is_muted" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a boolean') property_name = "in_home_view" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a boolean') property_name = "desktop_notifications" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a boolean') property_name = "audible_notifications" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a boolean') property_name = "push_notifications" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a boolean') property_name = "email_notifications" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a boolean') property_name = "wildcard_mentions_notify" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f"{property_name} is not a boolean") property_name = "color" result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": property_name, - "value": False, - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": property_name, + "value": False, + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, f'{property_name} is not a string') @@ -2059,9 +2059,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): stream_id = 1000 result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "is_muted", - "stream_id": stream_id, - "value": False}])}) + {"subscription_data": orjson.dumps([{"property": "is_muted", + "stream_id": stream_id, + "value": False}]).decode()}) self.assert_json_error(result, "Invalid stream id") def test_set_invalid_property(self) -> None: @@ -2072,9 +2072,9 @@ class SubscriptionPropertiesTest(ZulipTestCase): self.login_user(test_user) subs = gather_subscriptions(test_user)[0] result = self.api_post(test_user, "/api/v1/users/me/subscriptions/properties", - {"subscription_data": ujson.dumps([{"property": "bad", - "value": "bad", - "stream_id": subs[0]["stream_id"]}])}) + {"subscription_data": orjson.dumps([{"property": "bad", + "value": "bad", + "stream_id": subs[0]["stream_id"]}]).decode()}) self.assert_json_error(result, "Unknown subscription property: bad") @@ -2085,7 +2085,7 @@ class SubscriptionRestApiTest(ZulipTestCase): # add request = { - 'add': ujson.dumps([{'name': 'my_test_stream_1'}]), + 'add': orjson.dumps([{'name': 'my_test_stream_1'}]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_success(result) @@ -2094,7 +2094,7 @@ class SubscriptionRestApiTest(ZulipTestCase): # now delete the same stream request = { - 'delete': ujson.dumps(['my_test_stream_1']), + 'delete': orjson.dumps(['my_test_stream_1']).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_success(result) @@ -2107,14 +2107,14 @@ class SubscriptionRestApiTest(ZulipTestCase): # add with color proposition request = { - 'add': ujson.dumps([{'name': 'my_test_stream_2', 'color': '#afafaf'}]), + 'add': orjson.dumps([{'name': 'my_test_stream_2', 'color': '#afafaf'}]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_success(result) # incorrect color format request = { - 'subscriptions': ujson.dumps([{'name': 'my_test_stream_3', 'color': '#0g0g0g'}]), + 'subscriptions': orjson.dumps([{'name': 'my_test_stream_3', 'color': '#0g0g0g'}]).decode(), } result = self.api_post(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, 'subscriptions[0]["color"] is not a valid hex color code') @@ -2163,7 +2163,7 @@ class SubscriptionRestApiTest(ZulipTestCase): def check_for_error(val: Any, expected_message: str) -> None: request = { - 'add': ujson.dumps(val), + 'add': orjson.dumps(val).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, expected_message) @@ -2177,8 +2177,8 @@ class SubscriptionRestApiTest(ZulipTestCase): self.login_user(user) request = { - 'add': ujson.dumps([{'name': 'my_new_stream'}]), - 'principals': ujson.dumps([{}]), + 'add': orjson.dumps([{'name': 'my_new_stream'}]).decode(), + 'principals': orjson.dumps([{}]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, 'principals is not an allowed_type') @@ -2188,7 +2188,7 @@ class SubscriptionRestApiTest(ZulipTestCase): self.login_user(user) request = { - 'delete': ujson.dumps([{'name': 'my_test_stream_1'}]), + 'delete': orjson.dumps([{'name': 'my_test_stream_1'}]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, "delete[0] is not a string") @@ -2210,7 +2210,7 @@ class SubscriptionRestApiTest(ZulipTestCase): invalid_stream_name = "" request = { - 'delete': ujson.dumps([invalid_stream_name]), + 'delete': orjson.dumps([invalid_stream_name]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, @@ -2222,7 +2222,7 @@ class SubscriptionRestApiTest(ZulipTestCase): long_stream_name = "a" * 61 request = { - 'delete': ujson.dumps([long_stream_name]), + 'delete': orjson.dumps([long_stream_name]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, @@ -2234,7 +2234,7 @@ class SubscriptionRestApiTest(ZulipTestCase): stream_name = "abc\000" request = { - 'delete': ujson.dumps([stream_name]), + 'delete': orjson.dumps([stream_name]).decode(), } result = self.api_patch(user, "/api/v1/users/me/subscriptions", request) self.assert_json_error(result, @@ -2404,7 +2404,7 @@ class SubscriptionAPITest(ZulipTestCase): invite_streams, extra_post_data={ 'announce': 'true', - 'principals': ujson.dumps([self.user_profile.id]), + 'principals': orjson.dumps([self.user_profile.id]).decode(), }, ) @@ -2431,7 +2431,7 @@ class SubscriptionAPITest(ZulipTestCase): invite_streams, extra_post_data=dict( announce='true', - principals= ujson.dumps([self.user_profile.id]), + principals= orjson.dumps([self.user_profile.id]).decode(), ), ) @@ -2496,7 +2496,7 @@ class SubscriptionAPITest(ZulipTestCase): invite_streams, extra_post_data={ 'announce': 'true', - 'principals': ujson.dumps([self.user_profile.id]), + 'principals': orjson.dumps([self.user_profile.id]).decode(), }, ) @@ -2583,7 +2583,7 @@ class SubscriptionAPITest(ZulipTestCase): result = self.common_subscribe_to_streams( self.test_user, ['stream1'], - {"principals": ujson.dumps([invitee_user_id])}, + {"principals": orjson.dumps([invitee_user_id]).decode()}, allow_fail=True, ) self.assert_json_error( @@ -2592,8 +2592,8 @@ class SubscriptionAPITest(ZulipTestCase): do_set_realm_property(realm, "invite_to_stream_policy", Realm.POLICY_MEMBERS_ONLY) self.common_subscribe_to_streams( - self.test_user, ['stream2'], {"principals": ujson.dumps([ - self.test_user.id, invitee_user_id])}) + self.test_user, ['stream2'], {"principals": orjson.dumps([ + self.test_user.id, invitee_user_id]).decode()}) self.unsubscribe(user_profile, "stream2") do_set_realm_property(realm, "invite_to_stream_policy", @@ -2602,7 +2602,7 @@ class SubscriptionAPITest(ZulipTestCase): result = self.common_subscribe_to_streams( self.test_user, ['stream2'], - {"principals": ujson.dumps([invitee_user_id])}, + {"principals": orjson.dumps([invitee_user_id]).decode()}, allow_fail=True, ) self.assert_json_error( @@ -2610,7 +2610,7 @@ class SubscriptionAPITest(ZulipTestCase): do_set_realm_property(realm, "waiting_period_threshold", 0) self.common_subscribe_to_streams( - self.test_user, ['stream2'], {"principals": ujson.dumps([invitee_user_id])}) + self.test_user, ['stream2'], {"principals": orjson.dumps([invitee_user_id]).decode()}) def test_can_subscribe_other_users(self) -> None: """ @@ -2666,7 +2666,7 @@ class SubscriptionAPITest(ZulipTestCase): streams_to_sub = streams[:1] # just add one, to make the message easier to check streams_to_sub.extend(current_streams) self.helper_check_subs_before_and_after_add(streams_to_sub, - {"principals": ujson.dumps([invitee_data])}, streams[:1], + {"principals": orjson.dumps([invitee_data]).decode()}, streams[:1], current_streams, other_profile.email, streams_to_sub, invitee_realm, invite_only=invite_only) @@ -2689,7 +2689,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, streams_to_sub, - dict(principals=ujson.dumps([user1.id, user2.id])), + dict(principals=orjson.dumps([user1.id, user2.id]).decode()), ) self.assert_length(queries, 40) @@ -2716,7 +2716,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, streams_to_sub, - dict(principals=ujson.dumps([self.test_user.id])), + dict(principals=orjson.dumps([self.test_user.id]).decode()), ) self.assert_length(queries, 14) @@ -2921,7 +2921,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, streams_to_sub, - dict(principals=ujson.dumps(orig_user_ids_to_subscribe))) + dict(principals=orjson.dumps(orig_user_ids_to_subscribe).decode())) new_user_ids_to_subscribe = [iago.id, cordelia.id] events: List[Mapping[str, Any]] = [] @@ -2929,7 +2929,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, streams_to_sub, - dict(principals=ujson.dumps(new_user_ids_to_subscribe)), + dict(principals=orjson.dumps(new_user_ids_to_subscribe).decode()), ) add_peer_events = [events[2], events[3]] @@ -3039,7 +3039,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( mit_user, stream_names, - dict(principals=ujson.dumps([mit_user.id])), + dict(principals=orjson.dumps([mit_user.id]).decode()), subdomain="zephyr", allow_fail=True, ) @@ -3069,7 +3069,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, streams, - dict(principals=ujson.dumps([self.test_user.id])), + dict(principals=orjson.dumps([self.test_user.id]).decode()), ) # Make sure we don't make O(streams) queries self.assert_length(queries, 16) @@ -3095,7 +3095,7 @@ class SubscriptionAPITest(ZulipTestCase): """ target_profile = self.example_user("cordelia") post_data = dict( - principals=ujson.dumps([target_profile.id]), + principals=orjson.dumps([target_profile.id]).decode(), ) self.common_subscribe_to_streams(self.test_user, "Verona", post_data) @@ -3135,7 +3135,7 @@ class SubscriptionAPITest(ZulipTestCase): with self.assertRaises(UserProfile.DoesNotExist): get_user(invalid_principal, invalid_principal_realm) result = self.common_subscribe_to_streams(self.test_user, self.streams, - {"principals": ujson.dumps([invalid_principal])}, + {"principals": orjson.dumps([invalid_principal]).decode()}, allow_fail=True) self.assert_json_error( result, @@ -3149,7 +3149,7 @@ class SubscriptionAPITest(ZulipTestCase): with self.assertRaises(UserProfile.DoesNotExist): get_user_profile_by_id_in_realm(invalid_principal, invalid_principal_realm) result = self.common_subscribe_to_streams(self.test_user, self.streams, - {"principals": ujson.dumps([invalid_principal])}, + {"principals": orjson.dumps([invalid_principal]).decode()}, allow_fail=True) self.assert_json_error( result, @@ -3167,7 +3167,7 @@ class SubscriptionAPITest(ZulipTestCase): # verify that principal exists (thus, the reason for the error is the cross-realming) self.assertIsInstance(profile, UserProfile) result = self.common_subscribe_to_streams(self.test_user, self.streams, - {"principals": ujson.dumps([principal])}, + {"principals": orjson.dumps([principal]).decode()}, allow_fail=True) self.assert_json_error( result, @@ -3190,7 +3190,7 @@ class SubscriptionAPITest(ZulipTestCase): "not_removed": ["Rome"], "result": "success"} """ result = self.client_delete("/json/users/me/subscriptions", - {"subscriptions": ujson.dumps(subscriptions)}) + {"subscriptions": orjson.dumps(subscriptions).decode()}) self.assert_json_success(result) json = result.json() for key, val in json_dict.items(): @@ -3229,7 +3229,7 @@ class SubscriptionAPITest(ZulipTestCase): self.assertNotEqual(len(random_streams), 0) # necessary for full test coverage streams_to_remove = random_streams[:1] # pick only one fake stream, to make checking the error message easy result = self.client_delete("/json/users/me/subscriptions", - {"subscriptions": ujson.dumps(streams_to_remove)}) + {"subscriptions": orjson.dumps(streams_to_remove).decode()}) self.assert_json_error(result, f"Stream(s) ({random_streams[0]}) do not exist") def helper_subscriptions_exists(self, stream: str, expect_success: bool, subscribed: bool) -> None: @@ -3489,7 +3489,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, [new_streams[0]], - dict(principals=ujson.dumps([user1.id, user2.id])), + dict(principals=orjson.dumps([user1.id, user2.id]).decode()), ) self.assert_length(queries, 40) @@ -3498,7 +3498,7 @@ class SubscriptionAPITest(ZulipTestCase): self.common_subscribe_to_streams( self.test_user, [new_streams[1]], - dict(principals=ujson.dumps([user1.id, user2.id])), + dict(principals=orjson.dumps([user1.id, user2.id]).decode()), invite_only=True, ) self.assert_length(queries, 40) @@ -3513,7 +3513,7 @@ class SubscriptionAPITest(ZulipTestCase): [new_streams[2]], dict( announce='true', - principals=ujson.dumps([user1.id, user2.id]), + principals=orjson.dumps([user1.id, user2.id]).decode(), ), ) self.assert_length(queries, 52) @@ -3545,7 +3545,7 @@ class GetStreamsTest(ZulipTestCase): self.assertIsInstance(json["streams"], list) self.assert_json_success(owner_subs) - owner_subs_json = ujson.loads(owner_subs.content) + owner_subs_json = orjson.loads(owner_subs.content) self.assertEqual(sorted([s["name"] for s in json["streams"]]), sorted([s["name"] for s in owner_subs_json["subscriptions"]])) @@ -3678,7 +3678,7 @@ class GetStreamsTest(ZulipTestCase): self.assertIsInstance(json["streams"], list) self.assert_json_success(result2) - json2 = ujson.loads(result2.content) + json2 = orjson.loads(result2.content) self.assertEqual(sorted([s["name"] for s in json["streams"]]), sorted([s["name"] for s in json2["subscriptions"]])) @@ -3771,7 +3771,7 @@ class InviteOnlyStreamTest(ZulipTestCase): # authorization_errors_fatal=False works self.login_user(othello) result = self.common_subscribe_to_streams(othello, [stream_name], - extra_post_data={'authorization_errors_fatal': ujson.dumps(False)}) + extra_post_data={'authorization_errors_fatal': orjson.dumps(False).decode()}) json = result.json() self.assertEqual(json["unauthorized"], [stream_name]) self.assertEqual(json["subscribed"], {}) @@ -3781,7 +3781,7 @@ class InviteOnlyStreamTest(ZulipTestCase): self.login_user(hamlet) result = self.common_subscribe_to_streams( hamlet, [stream_name], - extra_post_data={'principals': ujson.dumps([othello.id])}) + extra_post_data={'principals': orjson.dumps([othello.id]).decode()}) json = result.json() self.assertEqual(json["subscribed"], {othello.email: [stream_name]}) self.assertEqual(json["already_subscribed"], {}) @@ -3866,7 +3866,7 @@ class GetSubscribersTest(ZulipTestCase): self.common_subscribe_to_streams( self.user_profile, streams, - dict(principals=ujson.dumps(users_to_subscribe))) + dict(principals=orjson.dumps(users_to_subscribe).decode())) msg = ''' @**King Hamlet** subscribed you to the following streams: @@ -3889,7 +3889,7 @@ class GetSubscribersTest(ZulipTestCase): self.common_subscribe_to_streams( self.user_profile, ["stream_invite_only_1"], - dict(principals=ujson.dumps([self.user_profile.id])), + dict(principals=orjson.dumps([self.user_profile.id]).decode()), invite_only=True) # Now add in other users, and this should trigger messages @@ -3897,7 +3897,7 @@ class GetSubscribersTest(ZulipTestCase): self.common_subscribe_to_streams( self.user_profile, ["stream_invite_only_1"], - dict(principals=ujson.dumps(users_to_subscribe)), + dict(principals=orjson.dumps(users_to_subscribe).decode()), invite_only=True) msg = ''' @@ -3951,7 +3951,7 @@ class GetSubscribersTest(ZulipTestCase): self.common_subscribe_to_streams( self.user_profile, public_streams, - dict(principals=ujson.dumps(users_to_subscribe)), + dict(principals=orjson.dumps(users_to_subscribe).decode()), ) create_public_streams() @@ -3963,7 +3963,7 @@ class GetSubscribersTest(ZulipTestCase): ret = self.common_subscribe_to_streams( self.user_profile, web_public_streams, - dict(principals=ujson.dumps(users_to_subscribe)) + dict(principals=orjson.dumps(users_to_subscribe).decode()) ) self.assert_json_success(ret) @@ -3973,7 +3973,7 @@ class GetSubscribersTest(ZulipTestCase): self.common_subscribe_to_streams( self.user_profile, private_streams, - dict(principals=ujson.dumps(users_to_subscribe)), + dict(principals=orjson.dumps(users_to_subscribe).decode()), invite_only=True, ) @@ -4126,7 +4126,7 @@ class GetSubscribersTest(ZulipTestCase): self.common_subscribe_to_streams( mit_user_profile, ["mit_invite_only"], - dict(principals=ujson.dumps(users_to_subscribe)), + dict(principals=orjson.dumps(users_to_subscribe).decode()), invite_only=True, subdomain="zephyr") diff --git a/zerver/tests/test_thumbnail.py b/zerver/tests/test_thumbnail.py index 967b859dda..7f440c7184 100644 --- a/zerver/tests/test_thumbnail.py +++ b/zerver/tests/test_thumbnail.py @@ -2,7 +2,7 @@ import base64 import urllib from io import StringIO -import ujson +import orjson from django.conf import settings from zerver.lib.test_classes import ZulipTestCase @@ -40,7 +40,7 @@ class ThumbnailTest(ZulipTestCase): result = self.client_post("/json/user_uploads", {'file': fp}) self.assert_json_success(result) - json = ujson.loads(result.content) + json = orjson.loads(result.content) self.assertIn("uri", json) uri = json["uri"] base = '/user_uploads/' @@ -165,7 +165,7 @@ class ThumbnailTest(ZulipTestCase): result = self.client_post("/json/user_uploads", {'file': fp}) self.assert_json_success(result) - json = ujson.loads(result.content) + json = orjson.loads(result.content) self.assertIn("uri", json) uri = json["uri"] base = '/user_uploads/' @@ -192,7 +192,7 @@ class ThumbnailTest(ZulipTestCase): result = self.client_post("/json/user_uploads", {'file': fp}) self.assert_json_success(result) - json = ujson.loads(result.content) + json = orjson.loads(result.content) self.assertIn("uri", json) uri = json["uri"] @@ -262,7 +262,7 @@ class ThumbnailTest(ZulipTestCase): result = self.client_post("/json/user_uploads", {'file': fp}) self.assert_json_success(result) - json = ujson.loads(result.content) + json = orjson.loads(result.content) self.assertIn("uri", json) uri = json["uri"] base = '/user_uploads/' @@ -306,7 +306,7 @@ class ThumbnailTest(ZulipTestCase): result = self.client_post("/json/user_uploads", {'file': fp}) self.assert_json_success(result) - json = ujson.loads(result.content) + json = orjson.loads(result.content) self.assertIn("uri", json) uri = json["uri"] base = '/user_uploads/' @@ -338,7 +338,7 @@ class ThumbnailTest(ZulipTestCase): result = self.client_post("/json/user_uploads", {'file': fp}) self.assert_json_success(result) - json = ujson.loads(result.content) + json = orjson.loads(result.content) self.assertIn("uri", json) uri = json["uri"] base = '/user_uploads/' diff --git a/zerver/tests/test_tornado.py b/zerver/tests/test_tornado.py index d5849f2a35..69a2347b65 100644 --- a/zerver/tests/test_tornado.py +++ b/zerver/tests/test_tornado.py @@ -1,7 +1,7 @@ import urllib.parse from typing import Any, Dict, Optional -import ujson +import orjson from django.conf import settings from django.core import signals from django.db import close_old_connections @@ -83,7 +83,7 @@ class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase): skip_user_agent=True, ) self.assertEqual(response.code, 200) - body = ujson.loads(response.body) + body = orjson.loads(response.body) self.assertEqual(body['events'], []) self.assertIn('queue_id', body) return body['queue_id'] @@ -116,7 +116,7 @@ class EventsTestCase(TornadoWebTestCase): self.io_loop.call_later(0.1, process_events) response = self.wait() - data = ujson.loads(response.body) + data = orjson.loads(response.body) self.assertEqual(data['events'], [ {'type': 'test', 'data': 'test data', 'id': 0}, ]) diff --git a/zerver/tests/test_tutorial.py b/zerver/tests/test_tutorial.py index 7332a25ee5..bc0ac902c3 100644 --- a/zerver/tests/test_tutorial.py +++ b/zerver/tests/test_tutorial.py @@ -1,4 +1,4 @@ -import ujson +import orjson from django.conf import settings from zerver.lib.actions import internal_send_private_message @@ -27,7 +27,7 @@ class TutorialTests(ZulipTestCase): ('finished', UserProfile.TUTORIAL_FINISHED), ] for incoming_status, expected_db_status in cases: - params = dict(status=ujson.dumps(incoming_status)) + params = dict(status=orjson.dumps(incoming_status).decode()) result = self.client_post('/json/users/me/tutorial_status', params) self.assert_json_success(result) user = self.example_user('hamlet') diff --git a/zerver/tests/test_typing.py b/zerver/tests/test_typing.py index 750b293b7a..d95f7ce1e7 100644 --- a/zerver/tests/test_typing.py +++ b/zerver/tests/test_typing.py @@ -1,6 +1,6 @@ from typing import Any, List, Mapping -import ujson +import orjson from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured, tornado_redirected_to_list @@ -14,7 +14,7 @@ class TypingValidateOperatorTest(ZulipTestCase): """ sender = self.example_user("hamlet") params = dict( - to=ujson.dumps([sender.id]), + to=orjson.dumps([sender.id]).decode(), ) result = self.api_post(sender, '/api/v1/typing', params) self.assert_json_error(result, 'Missing \'op\' argument') @@ -25,7 +25,7 @@ class TypingValidateOperatorTest(ZulipTestCase): """ sender = self.example_user("hamlet") params = dict( - to=ujson.dumps([sender.id]), + to=orjson.dumps([sender.id]).decode(), op='foo', ) result = self.api_post(sender, '/api/v1/typing', params) @@ -75,7 +75,7 @@ class TypingHappyPathTest(ZulipTestCase): expected_recipient_ids = {user.id for user in expected_recipients} params = dict( - to=ujson.dumps([recipient_user.id]), + to=orjson.dumps([recipient_user.id]).decode(), op='start', ) @@ -113,7 +113,7 @@ class TypingHappyPathTest(ZulipTestCase): events: List[Mapping[str, Any]] = [] params = dict( - to=ujson.dumps([user.id for user in recipient_users]), + to=orjson.dumps([user.id for user in recipient_users]).decode(), op='start', ) @@ -156,7 +156,7 @@ class TypingHappyPathTest(ZulipTestCase): user, '/api/v1/typing', { - 'to': ujson.dumps([user.id]), + 'to': orjson.dumps([user.id]).decode(), 'op': 'start', }, ) @@ -187,7 +187,7 @@ class TypingHappyPathTest(ZulipTestCase): expected_recipient_ids = {user.id for user in expected_recipients} params = dict( - to=ujson.dumps([recipient.id]), + to=orjson.dumps([recipient.id]).decode(), op='start', ) @@ -223,7 +223,7 @@ class TypingHappyPathTest(ZulipTestCase): events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): params = dict( - to=ujson.dumps([user.id]), + to=orjson.dumps([user.id]).decode(), op='stop', ) result = self.api_post(user, '/api/v1/typing', params) @@ -257,7 +257,7 @@ class TypingHappyPathTest(ZulipTestCase): events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): params = dict( - to=ujson.dumps([recipient.id]), + to=orjson.dumps([recipient.id]).decode(), op='stop', ) result = self.api_post(sender, '/api/v1/typing', params) diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.py index 2e532d839d..c98247147d 100644 --- a/zerver/tests/test_upload.py +++ b/zerver/tests/test_upload.py @@ -10,7 +10,7 @@ from unittest import mock from unittest.mock import patch import botocore.exceptions -import ujson +import orjson from django.conf import settings from django.utils.timezone import now as timezone_now from django_sendfile.utils import _get_sendfile @@ -1353,7 +1353,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): with get_test_image_file('img.png') as fp1, \ get_test_image_file('img.png') as fp2: result = self.client_post("/json/realm/logo", {'f1': fp1, 'f2': fp2, - 'night': ujson.dumps(self.night)}) + 'night': orjson.dumps(self.night).decode()}) self.assert_json_error(result, "You must upload exactly one logo.") def test_no_file_upload_failure(self) -> None: @@ -1362,7 +1362,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): """ self.login('iago') - result = self.client_post("/json/realm/logo", {'night': ujson.dumps(self.night)}) + result = self.client_post("/json/realm/logo", {'night': orjson.dumps(self.night).decode()}) self.assert_json_error(result, "You must upload exactly one logo.") correct_files = [ @@ -1377,7 +1377,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): def test_no_admin_user_upload(self) -> None: self.login('hamlet') with get_test_image_file(self.correct_files[0][0]) as fp: - result = self.client_post("/json/realm/logo", {'file': fp, 'night': ujson.dumps(self.night)}) + result = self.client_post("/json/realm/logo", {'file': fp, 'night': orjson.dumps(self.night).decode()}) self.assert_json_error(result, 'Must be an organization administrator') def test_upload_limited_plan_type(self) -> None: @@ -1385,7 +1385,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): do_change_plan_type(user_profile.realm, Realm.LIMITED) self.login_user(user_profile) with get_test_image_file(self.correct_files[0][0]) as fp: - result = self.client_post("/json/realm/logo", {'file': fp, 'night': ujson.dumps(self.night)}) + result = self.client_post("/json/realm/logo", {'file': fp, 'night': orjson.dumps(self.night).decode()}) self.assert_json_error(result, 'Available on Zulip Standard. Upgrade to access.') def test_get_default_logo(self) -> None: @@ -1393,7 +1393,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): self.login_user(user_profile) realm = user_profile.realm do_change_logo_source(realm, Realm.LOGO_DEFAULT, self.night, acting_user=user_profile) - response = self.client_get("/json/realm/logo", {'night': ujson.dumps(self.night)}) + response = self.client_get("/json/realm/logo", {'night': orjson.dumps(self.night).decode()}) redirect_url = response['Location'] is_night_str = str(self.night).lower() self.assertEqual(redirect_url, f"/static/images/logo/zulip-org-logo.svg?version=0&night={is_night_str}") @@ -1403,7 +1403,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): self.login_user(user_profile) realm = user_profile.realm do_change_logo_source(realm, Realm.LOGO_UPLOADED, self.night, acting_user=user_profile) - response = self.client_get("/json/realm/logo", {'night': ujson.dumps(self.night)}) + response = self.client_get("/json/realm/logo", {'night': orjson.dumps(self.night).decode()}) redirect_url = response['Location'] self.assertTrue(redirect_url.endswith(get_realm_logo_url(realm, self.night) + f'&night={str(self.night).lower()}')) @@ -1421,7 +1421,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): self.assertEqual(realm.night_logo_source, Realm.LOGO_UPLOADED) else: self.assertEqual(realm.logo_source, Realm.LOGO_UPLOADED) - response = self.client_get("/json/realm/logo", {'night': ujson.dumps(self.night)}) + response = self.client_get("/json/realm/logo", {'night': orjson.dumps(self.night).decode()}) redirect_url = response['Location'] self.assertEqual(redirect_url, f"/static/images/logo/zulip-org-logo.svg?version=0&night={is_night_str}") @@ -1435,7 +1435,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): # with self.subTest(fname=fname): self.login('iago') with get_test_image_file(fname) as fp: - result = self.client_post("/json/realm/logo", {'file': fp, 'night': ujson.dumps(self.night)}) + result = self.client_post("/json/realm/logo", {'file': fp, 'night': orjson.dumps(self.night).decode()}) realm = get_realm('zulip') self.assert_json_success(result) logo_url = get_realm_logo_url(realm, self.night) @@ -1455,7 +1455,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): # with self.subTest(fname=fname): self.login('iago') with get_test_image_file(fname) as fp: - result = self.client_post("/json/realm/logo", {'file': fp, 'night': ujson.dumps(self.night)}) + result = self.client_post("/json/realm/logo", {'file': fp, 'night': orjson.dumps(self.night).decode()}) self.assert_json_error(result, "Could not decode image; did you upload an image file?") @@ -1467,7 +1467,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): self.login_user(user_profile) realm = user_profile.realm do_change_logo_source(realm, Realm.LOGO_UPLOADED, self.night, acting_user=user_profile) - result = self.client_delete("/json/realm/logo", {'night': ujson.dumps(self.night)}) + result = self.client_delete("/json/realm/logo", {'night': orjson.dumps(self.night).decode()}) self.assert_json_success(result) realm = get_realm('zulip') if self.night: @@ -1484,7 +1484,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): version = realm.logo_version self.assertEqual(version, 1) with get_test_image_file(self.correct_files[0][0]) as fp: - self.client_post("/json/realm/logo", {'file': fp, 'night': ujson.dumps(self.night)}) + self.client_post("/json/realm/logo", {'file': fp, 'night': orjson.dumps(self.night).decode()}) realm = get_realm('zulip') if self.night: self.assertEqual(realm.night_logo_version, version + 1) @@ -1496,7 +1496,7 @@ class RealmLogoTest(UploadSerializeMixin, ZulipTestCase): with get_test_image_file(self.correct_files[0][0]) as fp: with self.settings(MAX_LOGO_FILE_SIZE=0): result = self.client_post("/json/realm/logo", {'file': fp, 'night': - ujson.dumps(self.night)}) + orjson.dumps(self.night).decode()}) self.assert_json_error(result, "Uploaded file is larger than the allowed limit of 0 MiB") def tearDown(self) -> None: @@ -2009,7 +2009,7 @@ class DecompressionBombTests(ZulipTestCase): for url, error_string in self.test_urls.items(): fp.seek(0, 0) if (url == "/json/realm/logo"): - result = self.client_post(url, {'f1': fp, 'night': ujson.dumps(False)}) + result = self.client_post(url, {'f1': fp, 'night': orjson.dumps(False).decode()}) else: result = self.client_post(url, {'f1': fp}) self.assert_json_error(result, error_string) diff --git a/zerver/tests/test_urls.py b/zerver/tests/test_urls.py index 8523a54f20..db65a75cad 100644 --- a/zerver/tests/test_urls.py +++ b/zerver/tests/test_urls.py @@ -3,7 +3,7 @@ import os from typing import List, Optional import django.urls.resolvers -import ujson +import orjson from django.test import Client from zerver.lib.test_classes import ZulipTestCase @@ -93,7 +93,7 @@ class PublicURLTest(ZulipTestCase): resp = self.client_get("/api/v1/fetch_google_client_id") self.assertEqual(200, resp.status_code, msg=f"Expected 200, received {resp.status_code} for GET /api/v1/fetch_google_client_id") - data = ujson.loads(resp.content) + data = orjson.loads(resp.content) self.assertEqual('success', data['result']) self.assertEqual('ABCD', data['google_client_id']) diff --git a/zerver/tests/test_user_groups.py b/zerver/tests/test_user_groups.py index 3ac492c780..24ed19a7c7 100644 --- a/zerver/tests/test_user_groups.py +++ b/zerver/tests/test_user_groups.py @@ -1,6 +1,6 @@ from unittest import mock -import ujson +import orjson from zerver.lib.actions import do_set_realm_property, ensure_stream from zerver.lib.test_classes import ZulipTestCase @@ -82,7 +82,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login('hamlet') params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Support team', } result = self.client_post('/json/user_groups/create', info=params) @@ -92,7 +92,7 @@ class UserGroupAPITestCase(ZulipTestCase): # Test invalid member error params = { 'name': 'backend', - 'members': ujson.dumps([1111]), + 'members': orjson.dumps([1111]).decode(), 'description': 'Backend team', } result = self.client_post('/json/user_groups/create', info=params) @@ -102,7 +102,7 @@ class UserGroupAPITestCase(ZulipTestCase): # Test we cannot add hamlet again params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Support team', } result = self.client_post('/json/user_groups/create', info=params) @@ -124,7 +124,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login_user(guest_user) params = { 'name': 'support', - 'members': ujson.dumps([guest_user.id]), + 'members': orjson.dumps([guest_user.id]).decode(), 'description': 'Support team', } result = self.client_post('/json/user_groups/create', info=params) @@ -135,7 +135,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login('hamlet') params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Support team', } self.client_post('/json/user_groups/create', info=params) @@ -185,7 +185,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login_user(hamlet) params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id, guest_user.id]), + 'members': orjson.dumps([hamlet.id, guest_user.id]).decode(), 'description': 'Support team', } result = self.client_post('/json/user_groups/create', info=params) @@ -220,7 +220,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login('hamlet') params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Support team', } self.client_post('/json/user_groups/create', info=params) @@ -240,7 +240,7 @@ class UserGroupAPITestCase(ZulipTestCase): # Test when user not a member of user group tries to delete it params = { 'name': 'Development', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Development team', } self.client_post('/json/user_groups/create', info=params) @@ -270,7 +270,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login_user(hamlet) params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id, guest_user.id]), + 'members': orjson.dumps([hamlet.id, guest_user.id]).decode(), 'description': 'Support team', } result = self.client_post('/json/user_groups/create', info=params) @@ -287,7 +287,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login('hamlet') params = { 'name': 'support', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Support team', } self.client_post('/json/user_groups/create', info=params) @@ -297,7 +297,7 @@ class UserGroupAPITestCase(ZulipTestCase): othello = self.example_user('othello') add = [othello.id] - params = {'add': ujson.dumps(add)} + params = {'add': orjson.dumps(add).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_success(result) @@ -318,7 +318,7 @@ class UserGroupAPITestCase(ZulipTestCase): cordelia = self.example_user('cordelia') self.login_user(cordelia) add = [cordelia.id] - params = {'add': ujson.dumps(add)} + params = {'add': orjson.dumps(add).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_error(result, "Only group members and organization administrators can administer this group.") @@ -330,7 +330,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.login_user(iago) aaron = self.example_user('aaron') add = [aaron.id] - params = {'add': ujson.dumps(add)} + params = {'add': orjson.dumps(add).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_success(result) @@ -342,7 +342,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.logout() self.login_user(hamlet) # Test remove members - params = {'delete': ujson.dumps([othello.id])} + params = {'delete': orjson.dumps([othello.id]).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_success(result) @@ -351,7 +351,7 @@ class UserGroupAPITestCase(ZulipTestCase): self.assertEqual(len(members), 2) # Test remove a member that's already removed - params = {'delete': ujson.dumps([othello.id])} + params = {'delete': orjson.dumps([othello.id]).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_error(result, f"There is no member '{othello.id}' in this user group") @@ -368,7 +368,7 @@ class UserGroupAPITestCase(ZulipTestCase): # Test when user not a member of user group tries to remove members self.logout() self.login_user(cordelia) - params = {'delete': ujson.dumps([hamlet.id])} + params = {'delete': orjson.dumps([hamlet.id]).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_error(result, "Only group members and organization administrators can administer this group.") @@ -445,7 +445,7 @@ class UserGroupAPITestCase(ZulipTestCase): params = { 'name': 'support', - 'members': ujson.dumps([iago.id, hamlet.id]), + 'members': orjson.dumps([iago.id, hamlet.id]).decode(), 'description': 'Support team', } @@ -454,13 +454,13 @@ class UserGroupAPITestCase(ZulipTestCase): user_group = UserGroup.objects.get(name='support') # Test add member - params = {'add': ujson.dumps([cordelia.id])} + params = {'add': orjson.dumps([cordelia.id]).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_success(result) # Test remove member - params = {'delete': ujson.dumps([cordelia.id])} + params = {'delete': orjson.dumps([cordelia.id]).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_success(result) @@ -489,14 +489,14 @@ class UserGroupAPITestCase(ZulipTestCase): # Test creating a group params = { 'name': 'support2', - 'members': ujson.dumps([hamlet.id]), + 'members': orjson.dumps([hamlet.id]).decode(), 'description': 'Support team', } result = self.client_post('/json/user_groups/create', info=params) self.assert_json_error(result, "Must be an organization administrator") # Test add member - params = {'add': ujson.dumps([cordelia.id])} + params = {'add': orjson.dumps([cordelia.id]).decode()} result = self.client_post(f'/json/user_groups/{user_group.id}/members', info=params) self.assert_json_error(result, "Must be an organization administrator") diff --git a/zerver/tests/test_user_status.py b/zerver/tests/test_user_status.py index 9b1a2fb63a..7917a0f64d 100644 --- a/zerver/tests/test_user_status.py +++ b/zerver/tests/test_user_status.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Set -import ujson +import orjson from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import EventInfo, capture_event @@ -159,7 +159,7 @@ class UserStatusTest(ZulipTestCase): self.assert_json_error(result, "status_text is too long (limit: 60 characters)") payload = dict( - away=ujson.dumps(True), + away=orjson.dumps(True).decode(), status_text='on vacation', ) @@ -179,7 +179,7 @@ class UserStatusTest(ZulipTestCase): ) # Now revoke "away" status. - payload = dict(away=ujson.dumps(False)) + payload = dict(away=orjson.dumps(False).decode()) event_info = EventInfo() with capture_event(event_info): diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py index 2aeba1dcd8..cb397a9612 100644 --- a/zerver/tests/test_users.py +++ b/zerver/tests/test_users.py @@ -3,7 +3,7 @@ from email.headerregistry import Address from typing import Any, Dict, Iterable, List, Mapping, Optional, TypeVar, Union from unittest import mock -import ujson +import orjson from django.conf import settings from django.core.exceptions import ValidationError from django.test import override_settings @@ -205,7 +205,7 @@ class PermissionTest(ZulipTestCase): self.assertFalse(othello_dict['is_admin']) # Giveth - req = dict(role=ujson.dumps(UserProfile.ROLE_REALM_ADMINISTRATOR)) + req = dict(role=orjson.dumps(UserProfile.ROLE_REALM_ADMINISTRATOR).decode()) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): @@ -218,7 +218,7 @@ class PermissionTest(ZulipTestCase): self.assertEqual(person['role'], UserProfile.ROLE_REALM_ADMINISTRATOR) # Taketh away - req = dict(role=ujson.dumps(UserProfile.ROLE_MEMBER)) + req = dict(role=orjson.dumps(UserProfile.ROLE_MEMBER).decode()) events = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/users/{othello.id}', req) @@ -303,7 +303,7 @@ class PermissionTest(ZulipTestCase): def test_user_cannot_promote_to_admin(self) -> None: self.login('hamlet') - req = dict(role=ujson.dumps(UserProfile.ROLE_REALM_ADMINISTRATOR)) + req = dict(role=orjson.dumps(UserProfile.ROLE_REALM_ADMINISTRATOR).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_error(result, 'Insufficient permission') @@ -311,7 +311,7 @@ class PermissionTest(ZulipTestCase): new_name = 'new name' self.login('iago') hamlet = self.example_user('hamlet') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch(f'/json/users/{hamlet.id}', req) self.assert_json_success(result) hamlet = self.example_user('hamlet') @@ -319,21 +319,21 @@ class PermissionTest(ZulipTestCase): def test_non_admin_cannot_change_full_name(self) -> None: self.login('hamlet') - req = dict(full_name=ujson.dumps('new name')) + req = dict(full_name=orjson.dumps('new name').decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('othello').id), req) self.assert_json_error(result, 'Insufficient permission') def test_admin_cannot_set_long_full_name(self) -> None: new_name = 'a' * (UserProfile.MAX_NAME_LENGTH + 1) self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_error(result, 'Name too long!') def test_admin_cannot_set_short_full_name(self) -> None: new_name = 'a' self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_error(result, 'Name too short!') @@ -341,7 +341,7 @@ class PermissionTest(ZulipTestCase): # Name of format "Alice|999" breaks in Markdown new_name = 'iago|72' self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_error(result, 'Invalid format!') @@ -349,21 +349,21 @@ class PermissionTest(ZulipTestCase): # Adding characters after r'|d+' doesn't break Markdown new_name = 'Hello- 12iago|72k' self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_success(result) def test_not_allowed_format_complex(self) -> None: new_name = 'Hello- 12iago|72' self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_error(result, 'Invalid format!') def test_admin_cannot_set_full_name_with_invalid_characters(self) -> None: new_name = 'Opheli*' self.login('iago') - req = dict(full_name=ujson.dumps(new_name)) + req = dict(full_name=orjson.dumps(new_name).decode()) result = self.client_patch('/json/users/{}'.format(self.example_user('hamlet').id), req) self.assert_json_error(result, 'Invalid characters in name!') @@ -403,7 +403,7 @@ class PermissionTest(ZulipTestCase): hamlet = self.example_user("hamlet") self.assertFalse(hamlet.is_guest) - req = dict(role=ujson.dumps(UserProfile.ROLE_GUEST)) + req = dict(role=orjson.dumps(UserProfile.ROLE_GUEST).decode()) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/users/{hamlet.id}', req) @@ -422,7 +422,7 @@ class PermissionTest(ZulipTestCase): polonius = self.example_user("polonius") self.assertTrue(polonius.is_guest) - req = dict(role=ujson.dumps(UserProfile.ROLE_MEMBER)) + req = dict(role=orjson.dumps(UserProfile.ROLE_MEMBER).decode()) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/users/{polonius.id}', req) @@ -445,7 +445,7 @@ class PermissionTest(ZulipTestCase): # Test changing a user from admin to guest and revoking admin status hamlet = self.example_user("hamlet") self.assertFalse(hamlet.is_guest) - req = dict(role=ujson.dumps(UserProfile.ROLE_GUEST)) + req = dict(role=orjson.dumps(UserProfile.ROLE_GUEST).decode()) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/users/{hamlet.id}', req) @@ -469,7 +469,7 @@ class PermissionTest(ZulipTestCase): # Test changing a user from guest to admin and revoking guest status polonius = self.example_user("polonius") self.assertFalse(polonius.is_realm_admin) - req = dict(role=ujson.dumps(UserProfile.ROLE_REALM_ADMINISTRATOR)) + req = dict(role=orjson.dumps(UserProfile.ROLE_REALM_ADMINISTRATOR).decode()) events: List[Mapping[str, Any]] = [] with tornado_redirected_to_list(events): result = self.client_patch(f'/json/users/{polonius.id}', req) @@ -603,7 +603,7 @@ class PermissionTest(ZulipTestCase): }) result = self.client_patch(f'/json/users/{cordelia.id}', - {'profile_data': ujson.dumps(new_profile_data)}) + {'profile_data': orjson.dumps(new_profile_data).decode()}) self.assert_json_success(result) cordelia = self.example_user("cordelia") @@ -628,7 +628,7 @@ class PermissionTest(ZulipTestCase): }) result = self.client_patch(f'/json/users/{cordelia.id}', - {'profile_data': ujson.dumps(new_profile_data)}) + {'profile_data': orjson.dumps(new_profile_data).decode()}) self.assert_json_error(result, error_msg) # non-existent field and no data @@ -637,7 +637,7 @@ class PermissionTest(ZulipTestCase): 'value': '', }] result = self.client_patch(f'/json/users/{cordelia.id}', - {'profile_data': ujson.dumps(invalid_profile_data)}) + {'profile_data': orjson.dumps(invalid_profile_data).decode()}) self.assert_json_error(result, 'Field id 9001 not found.') # non-existent field and data @@ -646,7 +646,7 @@ class PermissionTest(ZulipTestCase): 'value': 'some data', }] result = self.client_patch(f'/json/users/{cordelia.id}', - {'profile_data': ujson.dumps(invalid_profile_data)}) + {'profile_data': orjson.dumps(invalid_profile_data).decode()}) self.assert_json_error(result, 'Field id 9001 not found.') # Test for clearing/resetting field values. @@ -661,7 +661,7 @@ class PermissionTest(ZulipTestCase): 'value': value, }) result = self.client_patch(f'/json/users/{cordelia.id}', - {'profile_data': ujson.dumps(empty_profile_data)}) + {'profile_data': orjson.dumps(empty_profile_data).decode()}) self.assert_json_success(result) for field_dict in cordelia.profile_data: with self.subTest(field_name=field_dict['name']): @@ -690,7 +690,7 @@ class PermissionTest(ZulipTestCase): 'value': value, }) result = self.client_patch(f'/json/users/{cordelia.id}', - {'profile_data': ujson.dumps(new_profile_data)}) + {'profile_data': orjson.dumps(new_profile_data).decode()}) self.assert_json_success(result) for field_dict in cordelia.profile_data: with self.subTest(field_name=field_dict['name']): @@ -708,11 +708,11 @@ class PermissionTest(ZulipTestCase): 'value': "New hamlet Biography", }) result = self.client_patch(f'/json/users/{hamlet.id}', - {'profile_data': ujson.dumps(new_profile_data)}) + {'profile_data': orjson.dumps(new_profile_data).decode()}) self.assert_json_error(result, 'Insufficient permission') result = self.client_patch('/json/users/{}'.format(self.example_user("cordelia").id), - {'profile_data': ujson.dumps(new_profile_data)}) + {'profile_data': orjson.dumps(new_profile_data).decode()}) self.assert_json_error(result, 'Insufficient permission') class BulkCreateUserTest(ZulipTestCase): @@ -1132,13 +1132,13 @@ class UserProfileTest(ZulipTestCase): result = self.client_get(f"/json/users/{iago.id}/subscriptions/25") self.assert_json_error(result, "Invalid stream id") - result = ujson.loads(self.client_get(f"/json/users/{iago.id}/subscriptions/{stream.id}").content) + result = orjson.loads(self.client_get(f"/json/users/{iago.id}/subscriptions/{stream.id}").content) self.assertFalse(result['is_subscribed']) # Subscribe to the stream. self.subscribe(iago, stream.name) with queries_captured() as queries: - result = ujson.loads(self.client_get(f"/json/users/{iago.id}/subscriptions/{stream.id}").content) + result = orjson.loads(self.client_get(f"/json/users/{iago.id}/subscriptions/{stream.id}").content) self.assert_length(queries, 7) self.assertTrue(result['is_subscribed']) @@ -1149,7 +1149,7 @@ class UserProfileTest(ZulipTestCase): self.assertTrue(polonius.is_guest) self.assertTrue(stream.is_web_public) - result = ujson.loads(self.client_get(f"/json/users/{iago.id}/subscriptions/{stream.id}").content) + result = orjson.loads(self.client_get(f"/json/users/{iago.id}/subscriptions/{stream.id}").content) self.assertTrue(result['is_subscribed']) class ActivateTest(ZulipTestCase): @@ -1499,7 +1499,7 @@ class BulkUsersTest(ZulipTestCase): hamlet = self.example_user('hamlet') def get_hamlet_avatar(client_gravatar: bool) -> Optional[str]: - data = dict(client_gravatar=ujson.dumps(client_gravatar)) + data = dict(client_gravatar=orjson.dumps(client_gravatar).decode()) result = self.client_get('/json/users', data) self.assert_json_success(result) rows = result.json()['members'] @@ -1548,7 +1548,7 @@ class GetProfileTest(ZulipTestCase): desdemona = self.example_user('desdemona') self.login('hamlet') - result = ujson.loads(self.client_get('/json/users/me').content) + result = orjson.loads(self.client_get('/json/users/me').content) self.assertEqual(result['email'], hamlet.email) self.assertEqual(result['full_name'], 'King Hamlet') self.assertIn("user_id", result) @@ -1558,7 +1558,7 @@ class GetProfileTest(ZulipTestCase): self.assertFalse(result['is_guest']) self.assertFalse('delivery_email' in result) self.login('iago') - result = ujson.loads(self.client_get('/json/users/me').content) + result = orjson.loads(self.client_get('/json/users/me').content) self.assertEqual(result['email'], iago.email) self.assertEqual(result['full_name'], 'Iago') self.assertFalse(result['is_bot']) @@ -1566,7 +1566,7 @@ class GetProfileTest(ZulipTestCase): self.assertFalse(result['is_owner']) self.assertFalse(result['is_guest']) self.login('desdemona') - result = ujson.loads(self.client_get('/json/users/me').content) + result = orjson.loads(self.client_get('/json/users/me').content) self.assertEqual(result['email'], desdemona.email) self.assertFalse(result['is_bot']) self.assertTrue(result['is_admin']) @@ -1575,7 +1575,7 @@ class GetProfileTest(ZulipTestCase): # Tests the GET ../users/{id} api endpoint. user = self.example_user('hamlet') - result = ujson.loads(self.client_get(f'/json/users/{user.id}').content) + result = orjson.loads(self.client_get(f'/json/users/{user.id}').content) self.assertEqual(result['user']['email'], user.email) self.assertEqual(result['user']['full_name'], user.full_name) self.assertIn("user_id", result['user']) @@ -1584,14 +1584,14 @@ class GetProfileTest(ZulipTestCase): self.assertFalse(result['user']['is_admin']) self.assertFalse(result['user']['is_owner']) - result = ujson.loads(self.client_get(f'/json/users/{user.id}?include_custom_profile_fields=true').content) + result = orjson.loads(self.client_get(f'/json/users/{user.id}?include_custom_profile_fields=true').content) self.assertIn('profile_data', result['user']) result = self.client_get(f'/json/users/{30}?') self.assert_json_error(result, "No such user") bot = self.example_user("default_bot") - result = ujson.loads(self.client_get(f'/json/users/{bot.id}').content) + result = orjson.loads(self.client_get(f'/json/users/{bot.id}').content) self.assertEqual(result['user']['email'], bot.email) self.assertTrue(result['user']['is_bot']) diff --git a/zerver/tests/test_widgets.py b/zerver/tests/test_widgets.py index 14eadb2975..7a89259103 100644 --- a/zerver/tests/test_widgets.py +++ b/zerver/tests/test_widgets.py @@ -1,7 +1,7 @@ import re from typing import Any, Dict -import ujson +import orjson from django.core.exceptions import ValidationError from zerver.lib.test_classes import ZulipTestCase @@ -79,7 +79,7 @@ class WidgetContentTestCase(ZulipTestCase): self.assert_json_error_contains(result, 'Widgets: API programmer sent invalid JSON') bogus_data = dict(color='red', foo='bar', x=2) - payload['widget_content'] = ujson.dumps(bogus_data) + payload['widget_content'] = orjson.dumps(bogus_data).decode() result = self.api_post(sender, "/api/v1/messages", payload) self.assert_json_error_contains(result, 'Widgets: widget_type is not in widget_content') @@ -129,7 +129,7 @@ class WidgetContentTestCase(ZulipTestCase): client='test suite', topic='whatever', content=content, - widget_content=ujson.dumps(widget_content), + widget_content=orjson.dumps(widget_content).decode(), ) result = self.api_post(sender, "/api/v1/messages", payload) self.assert_json_success(result) @@ -144,7 +144,7 @@ class WidgetContentTestCase(ZulipTestCase): submessage = SubMessage.objects.get(message_id=message.id) self.assertEqual(submessage.msg_type, 'widget') - self.assertEqual(ujson.loads(submessage.content), expected_submessage_content) + self.assertEqual(orjson.loads(submessage.content), expected_submessage_content) def test_tictactoe(self) -> None: # The tictactoe widget is mostly useful as a code sample, @@ -175,7 +175,7 @@ class WidgetContentTestCase(ZulipTestCase): submessage = SubMessage.objects.get(message_id=message.id) self.assertEqual(submessage.msg_type, 'widget') - self.assertEqual(ujson.loads(submessage.content), expected_submessage_content) + self.assertEqual(orjson.loads(submessage.content), expected_submessage_content) def test_poll_command_extra_data(self) -> None: sender = self.example_user('cordelia') @@ -207,7 +207,7 @@ class WidgetContentTestCase(ZulipTestCase): submessage = SubMessage.objects.get(message_id=message.id) self.assertEqual(submessage.msg_type, 'widget') - self.assertEqual(ujson.loads(submessage.content), expected_submessage_content) + self.assertEqual(orjson.loads(submessage.content), expected_submessage_content) # Now don't supply a question. @@ -228,4 +228,4 @@ class WidgetContentTestCase(ZulipTestCase): self.assertEqual(message.content, content) submessage = SubMessage.objects.get(message_id=message.id) self.assertEqual(submessage.msg_type, 'widget') - self.assertEqual(ujson.loads(submessage.content), expected_submessage_content) + self.assertEqual(orjson.loads(submessage.content), expected_submessage_content) diff --git a/zerver/tests/test_zephyr.py b/zerver/tests/test_zephyr.py index b95c2ed332..4b1ac76496 100644 --- a/zerver/tests/test_zephyr.py +++ b/zerver/tests/test_zephyr.py @@ -1,7 +1,7 @@ from typing import Any from unittest.mock import patch -import ujson +import orjson from django.http import HttpResponse from zerver.lib.test_classes import ZulipTestCase @@ -15,7 +15,7 @@ class ZephyrTest(ZulipTestCase): self.login_user(user) def post(subdomain: Any, **kwargs: Any) -> HttpResponse: - params = {k: ujson.dumps(v) for k, v in kwargs.items()} + params = {k: orjson.dumps(v).decode() for k, v in kwargs.items()} return self.client_post('/accounts/webathena_kerberos_login/', params, subdomain=subdomain) diff --git a/zerver/tornado/django_api.py b/zerver/tornado/django_api.py index 118b2b6689..5ac87dbdb1 100644 --- a/zerver/tornado/django_api.py +++ b/zerver/tornado/django_api.py @@ -1,8 +1,8 @@ from functools import lru_cache from typing import Any, Container, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union +import orjson import requests -import ujson from django.conf import settings from requests.adapters import ConnectionError, HTTPAdapter from requests.models import PreparedRequest, Response @@ -61,20 +61,20 @@ def request_event_queue(user_profile: UserProfile, user_client: Client, apply_ma tornado_uri = get_tornado_uri(user_profile.realm) req = {'dont_block': 'true', - 'apply_markdown': ujson.dumps(apply_markdown), - 'client_gravatar': ujson.dumps(client_gravatar), - 'slim_presence': ujson.dumps(slim_presence), - 'all_public_streams': ujson.dumps(all_public_streams), + 'apply_markdown': orjson.dumps(apply_markdown), + 'client_gravatar': orjson.dumps(client_gravatar), + 'slim_presence': orjson.dumps(slim_presence), + 'all_public_streams': orjson.dumps(all_public_streams), 'client': 'internal', 'user_profile_id': user_profile.id, 'user_client': user_client.name, - 'narrow': ujson.dumps(narrow), + 'narrow': orjson.dumps(narrow), 'secret': settings.SHARED_SECRET, 'lifespan_secs': queue_lifespan_secs, - 'bulk_message_deletion': ujson.dumps(bulk_message_deletion)} + 'bulk_message_deletion': orjson.dumps(bulk_message_deletion)} if event_types is not None: - req['event_types'] = ujson.dumps(event_types) + req['event_types'] = orjson.dumps(event_types) resp = requests_client().post( tornado_uri + '/api/v1/events/internal', @@ -108,7 +108,7 @@ def send_notification_http(realm: Realm, data: Mapping[str, Any]) -> None: tornado_uri = get_tornado_uri(realm) requests_client().post( tornado_uri + "/notify_tornado", - data=dict(data=ujson.dumps(data), secret=settings.SHARED_SECRET), + data=dict(data=orjson.dumps(data), secret=settings.SHARED_SECRET), ) def send_event(realm: Realm, event: Mapping[str, Any], diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.py index 52f9da35b0..8914adc922 100644 --- a/zerver/tornado/event_queue.py +++ b/zerver/tornado/event_queue.py @@ -27,8 +27,8 @@ from typing import ( cast, ) +import orjson import tornado.ioloop -import ujson from django.conf import settings from django.utils.translation import ugettext as _ from typing_extensions import TypedDict @@ -465,9 +465,10 @@ def persistent_queue_filename(port: int, last: bool=False) -> str: def dump_event_queues(port: int) -> None: start = time.time() - with open(persistent_queue_filename(port), "w") as stored_queues: - ujson.dump([(qid, client.to_dict()) for (qid, client) in clients.items()], - stored_queues) + with open(persistent_queue_filename(port), "wb") as stored_queues: + stored_queues.write( + orjson.dumps([(qid, client.to_dict()) for (qid, client) in clients.items()]) + ) logging.info('Tornado %d dumped %d event queues in %.3fs', port, len(clients), time.time() - start) @@ -477,8 +478,8 @@ def load_event_queues(port: int) -> None: start = time.time() try: - with open(persistent_queue_filename(port)) as stored_queues: - data = ujson.load(stored_queues) + with open(persistent_queue_filename(port), "rb") as stored_queues: + data = orjson.loads(stored_queues.read()) except FileNotFoundError: pass except ValueError: diff --git a/zerver/tornado/views.py b/zerver/tornado/views.py index 5bc7f0a5f7..70c71c348e 100644 --- a/zerver/tornado/views.py +++ b/zerver/tornado/views.py @@ -1,7 +1,7 @@ import time from typing import Iterable, Optional, Sequence -import ujson +import orjson from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -22,7 +22,7 @@ from zerver.tornado.handlers import AsyncDjangoHandler @internal_notify_view(True) def notify(request: HttpRequest) -> HttpResponse: - process_notification(ujson.loads(request.POST['data'])) + process_notification(orjson.loads(request.POST['data'])) return json_success() @has_request_variables diff --git a/zerver/views/custom_profile_fields.py b/zerver/views/custom_profile_fields.py index 6d2143afea..02a25df9bf 100644 --- a/zerver/views/custom_profile_fields.py +++ b/zerver/views/custom_profile_fields.py @@ -1,6 +1,6 @@ from typing import Dict, List, Union -import ujson +import orjson from django.core.exceptions import ValidationError from django.db import IntegrityError from django.http import HttpRequest, HttpResponse @@ -93,7 +93,7 @@ def create_realm_custom_profile_field(request: HttpRequest, name: str=REQ(default=''), hint: str=REQ(default=''), field_data: ProfileFieldData=REQ(default={}, - converter=ujson.loads), + converter=orjson.loads), field_type: int=REQ(validator=check_int)) -> HttpResponse: validate_custom_profile_field(name, hint, field_type, field_data) try: @@ -136,7 +136,7 @@ def update_realm_custom_profile_field(request: HttpRequest, user_profile: UserPr name: str=REQ(default=''), hint: str=REQ(default=''), field_data: ProfileFieldData=REQ(default={}, - converter=ujson.loads), + converter=orjson.loads), ) -> HttpResponse: realm = user_profile.realm try: @@ -145,7 +145,7 @@ def update_realm_custom_profile_field(request: HttpRequest, user_profile: UserPr return json_error(_('Field id {id} not found.').format(id=field_id)) if field.field_type == CustomProfileField.EXTERNAL_ACCOUNT: - if is_default_external_field(field.field_type, ujson.loads(field.field_data)): + if is_default_external_field(field.field_type, orjson.loads(field.field_data)): return json_error(_("Default custom field cannot be updated.")) validate_custom_profile_field(name, hint, field.field_type, field_data) diff --git a/zerver/views/development/email_log.py b/zerver/views/development/email_log.py index 40f8a9bc73..457773b3e6 100755 --- a/zerver/views/development/email_log.py +++ b/zerver/views/development/email_log.py @@ -2,7 +2,7 @@ import os import subprocess import urllib -import ujson +import orjson from django.conf import settings from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect, render @@ -97,7 +97,7 @@ def generate_all_emails(request: HttpRequest) -> HttpResponse: stream = get_realm_stream("Denmark", user.realm.id) result = client.post("/json/invites", {"invitee_emails": unregistered_email_2, - "stream_ids": ujson.dumps([stream.id])}, + "stream_ids": orjson.dumps([stream.id]).decode()}, **host_kwargs) assert result.status_code == 200 diff --git a/zerver/views/development/integrations.py b/zerver/views/development/integrations.py index 235de84a29..aa557f0445 100644 --- a/zerver/views/development/integrations.py +++ b/zerver/views/development/integrations.py @@ -1,7 +1,7 @@ import os from typing import Any, Dict, List, Optional -import ujson +import orjson from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.test import Client @@ -65,7 +65,7 @@ def get_fixtures(request: HttpResponse, with open(fixture_path) as f: body = f.read() try: - body = ujson.loads(body) + body = orjson.loads(body) except ValueError: pass # The file extension will be used to determine the type. @@ -90,7 +90,7 @@ def check_send_webhook_fixture_message(request: HttpRequest, is_json: bool=REQ(validator=check_bool), custom_headers: str=REQ()) -> HttpResponse: try: - custom_headers_dict = ujson.loads(custom_headers) + custom_headers_dict = orjson.loads(custom_headers) except ValueError as ve: return json_error(f"Custom HTTP headers are not in a valid JSON format. {ve}") # nolint diff --git a/zerver/views/message_edit.py b/zerver/views/message_edit.py index 14dff89874..fa8df66d88 100644 --- a/zerver/views/message_edit.py +++ b/zerver/views/message_edit.py @@ -1,7 +1,7 @@ import datetime from typing import Any, Dict, List, Optional, Set -import ujson +import orjson from django.db import IntegrityError from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now @@ -84,7 +84,7 @@ def get_message_edit_history(request: HttpRequest, user_profile: UserProfile, # Extract the message edit history from the message if message.edit_history is not None: - message_edit_history = ujson.loads(message.edit_history) + message_edit_history = orjson.loads(message.edit_history) else: message_edit_history = [] diff --git a/zerver/views/message_fetch.py b/zerver/views/message_fetch.py index 5566acf0f6..03f258dddf 100644 --- a/zerver/views/message_fetch.py +++ b/zerver/views/message_fetch.py @@ -1,7 +1,7 @@ import re from typing import Any, Dict, Iterable, List, Optional, Tuple, Union -import ujson +import orjson from django.conf import settings from django.core.exceptions import ValidationError from django.db import connection @@ -520,7 +520,7 @@ def get_search_fields(rendered_content: str, topic_name: str, content_matches: I def narrow_parameter(json: str) -> OptionalNarrowListT: - data = ujson.loads(json) + data = orjson.loads(json) if not isinstance(data, list): raise ValueError("argument is not a list") if len(data) == 0: diff --git a/zerver/views/portico.py b/zerver/views/portico.py index c6e97bf275..c854bd7040 100644 --- a/zerver/views/portico.py +++ b/zerver/views/portico.py @@ -1,4 +1,4 @@ -import ujson +import orjson from django.conf import settings from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.template.response import TemplateResponse @@ -56,8 +56,8 @@ def team_view(request: HttpRequest) -> HttpResponse: return HttpResponseRedirect('https://zulip.com/team/', status=301) try: - with open(settings.CONTRIBUTOR_DATA_FILE_PATH) as f: - data = ujson.load(f) + with open(settings.CONTRIBUTOR_DATA_FILE_PATH, "rb") as f: + data = orjson.loads(f.read()) except FileNotFoundError: data = {'contributors': {}, 'date': "Never ran."} diff --git a/zerver/views/realm_export.py b/zerver/views/realm_export.py index cf3aefb297..09086cfc38 100644 --- a/zerver/views/realm_export.py +++ b/zerver/views/realm_export.py @@ -1,6 +1,6 @@ from datetime import timedelta -import ujson +import orjson from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now @@ -79,7 +79,7 @@ def delete_realm_export(request: HttpRequest, user: UserProfile, export_id: int) except RealmAuditLog.DoesNotExist: return json_error(_("Invalid data export ID")) - export_data = ujson.loads(audit_log_entry.extra_data) + export_data = orjson.loads(audit_log_entry.extra_data) if 'deleted_timestamp' in export_data: return json_error(_("Export already deleted")) do_delete_realm_export(user, audit_log_entry) diff --git a/zerver/views/streams.py b/zerver/views/streams.py index 68f8b485be..2f388af907 100644 --- a/zerver/views/streams.py +++ b/zerver/views/streams.py @@ -13,7 +13,7 @@ from typing import ( Union, ) -import ujson +import orjson from django.conf import settings from django.core.exceptions import ValidationError from django.db import transaction @@ -340,7 +340,7 @@ def compose_views( response = method(request, user_profile, **kwargs) if response.status_code != 200: raise JsonableError(response.content) - json_dict.update(ujson.loads(response.content)) + json_dict.update(orjson.loads(response.content)) return json_success(json_dict) check_principals: Validator[Union[List[str], List[int]]] = check_union( diff --git a/zerver/views/submessage.py b/zerver/views/submessage.py index 0c0346949a..1f6ddf421d 100644 --- a/zerver/views/submessage.py +++ b/zerver/views/submessage.py @@ -1,4 +1,4 @@ -import ujson +import orjson from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -20,7 +20,7 @@ def process_submessage(request: HttpRequest, message, user_message = access_message(user_profile, message_id) try: - ujson.loads(content) + orjson.loads(content) except Exception: return json_error(_("Invalid json for submessage")) diff --git a/zerver/views/zephyr.py b/zerver/views/zephyr.py index b2045d5b82..7d73399590 100644 --- a/zerver/views/zephyr.py +++ b/zerver/views/zephyr.py @@ -4,7 +4,7 @@ import re import subprocess from typing import Optional -import ujson +import orjson from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -34,7 +34,7 @@ def webathena_kerberos_login(request: HttpRequest, user_profile: UserProfile, return json_error(_("Webathena login not enabled")) try: - parsed_cred = ujson.loads(cred) + parsed_cred = orjson.loads(cred) user = parsed_cred["cname"]["nameString"][0] if user in kerberos_alter_egos: user = kerberos_alter_egos[user] diff --git a/zerver/webhooks/front/tests.py b/zerver/webhooks/front/tests.py index 14f48b8957..67cb7ca931 100644 --- a/zerver/webhooks/front/tests.py +++ b/zerver/webhooks/front/tests.py @@ -1,4 +1,4 @@ -import ujson +import orjson from zerver.lib.test_classes import WebhookTestCase @@ -172,9 +172,9 @@ class FrontHookTests(WebhookTestCase): def test_unknown_webhook_request(self) -> None: payload = self.get_body('conversation_assigned') - payload_json = ujson.loads(payload) + payload_json = orjson.loads(payload) payload_json['type'] = 'qwerty' - result = self.client_post(self.url, ujson.dumps(payload_json), + result = self.client_post(self.url, orjson.dumps(payload_json), content_type="application/x-www-form-urlencoded") self.assert_json_error(result, "Unknown webhook request") diff --git a/zerver/webhooks/hellosign/view.py b/zerver/webhooks/hellosign/view.py index 27018dd6b3..e60005a189 100644 --- a/zerver/webhooks/hellosign/view.py +++ b/zerver/webhooks/hellosign/view.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List -import ujson +import orjson from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view @@ -56,7 +56,7 @@ def get_recipients_text(recipients: List[str]) -> str: @has_request_variables def api_hellosign_webhook(request: HttpRequest, user_profile: UserProfile, payload: Dict[str, Dict[str, Any]]=REQ( - whence='json', converter=ujson.loads)) -> HttpResponse: + whence='json', converter=orjson.loads)) -> HttpResponse: if "signature_request" in payload: body = get_message_body(payload) topic = payload['signature_request']['title'] diff --git a/zerver/webhooks/librato/view.py b/zerver/webhooks/librato/view.py index d962911bdc..27aa4a7c26 100644 --- a/zerver/webhooks/librato/view.py +++ b/zerver/webhooks/librato/view.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Tuple -import ujson +import orjson from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -143,9 +143,9 @@ class LibratoWebhookHandler(LibratoWebhookParser): @api_key_only_webhook_view('Librato') @has_request_variables def api_librato_webhook(request: HttpRequest, user_profile: UserProfile, - payload: Dict[str, Any]=REQ(converter=ujson.loads, default={})) -> HttpResponse: + payload: Dict[str, Any]=REQ(converter=orjson.loads, default={})) -> HttpResponse: try: - attachments = ujson.loads(request.body).get('attachments', []) + attachments = orjson.loads(request.body).get('attachments', []) except ValueError: attachments = [] diff --git a/zerver/webhooks/pivotal/view.py b/zerver/webhooks/pivotal/view.py index 6fd45a8275..88a82e64c9 100644 --- a/zerver/webhooks/pivotal/view.py +++ b/zerver/webhooks/pivotal/view.py @@ -2,7 +2,7 @@ import re from typing import Any, Dict, List, Optional, Tuple -import ujson +import orjson from defusedxml.ElementTree import fromstring as xml_fromstring from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -73,7 +73,7 @@ UNSUPPORTED_EVENT_TYPES = [ ] def api_pivotal_webhook_v5(request: HttpRequest, user_profile: UserProfile) -> Tuple[str, str]: - payload = ujson.loads(request.body) + payload = orjson.loads(request.body) event_type = payload["kind"] diff --git a/zerver/webhooks/semaphore/tests.py b/zerver/webhooks/semaphore/tests.py index 83356b5601..58d4ac914f 100644 --- a/zerver/webhooks/semaphore/tests.py +++ b/zerver/webhooks/semaphore/tests.py @@ -1,6 +1,6 @@ from unittest.mock import patch -import ujson +import orjson from zerver.lib.test_classes import WebhookTestCase @@ -118,6 +118,6 @@ class SemaphoreHookTests(WebhookTestCase): def get_unknown_event(self, fixture_name: str) -> str: """Return modified payload with revision.reference_type changed""" - fixture_data = ujson.loads(self.webhook_fixture_data("semaphore", fixture_name, file_type="json")) + fixture_data = orjson.loads(self.webhook_fixture_data("semaphore", fixture_name, file_type="json")) fixture_data['revision']['reference_type'] = 'unknown' return fixture_data diff --git a/zerver/webhooks/slack_incoming/view.py b/zerver/webhooks/slack_incoming/view.py index 67eb9d9003..24c9615e13 100644 --- a/zerver/webhooks/slack_incoming/view.py +++ b/zerver/webhooks/slack_incoming/view.py @@ -2,7 +2,7 @@ import re from typing import Any, Dict, Optional -import ujson +import orjson from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ @@ -20,7 +20,7 @@ def api_slack_incoming_webhook(request: HttpRequest, user_profile: UserProfile, user_specified_topic: Optional[str]=REQ("topic", default=None), payload: Optional[Dict[str, Any]] = REQ( 'payload', - converter=ujson.loads, + converter=orjson.loads, default=None)) -> HttpResponse: # Slack accepts webhook payloads as payload="encoded json" as @@ -30,7 +30,7 @@ def api_slack_incoming_webhook(request: HttpRequest, user_profile: UserProfile, # # we were given JSON. if payload is None: try: - payload = ujson.loads(request.body) + payload = orjson.loads(request.body) except ValueError: # nocoverage raise InvalidJSONError(_("Malformed JSON")) diff --git a/zerver/webhooks/teamcity/tests.py b/zerver/webhooks/teamcity/tests.py index db3e8acee7..7d5b3d1335 100644 --- a/zerver/webhooks/teamcity/tests.py +++ b/zerver/webhooks/teamcity/tests.py @@ -1,4 +1,4 @@ -import ujson +import orjson from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import WebhookTestCase @@ -35,7 +35,7 @@ class TeamcityHookTests(WebhookTestCase): def test_teamcity_personal(self) -> None: expected_message = "Your personal build for Project :: Compile build 5535 - CL 123456 is broken with status Exit code 1 (new)! :thumbs_down: See [changes](http://teamcity/viewLog.html?buildTypeId=Project_Compile&buildId=19952&tab=buildChangesDiv) and [build log](http://teamcity/viewLog.html?buildTypeId=Project_Compile&buildId=19952)." - payload = ujson.dumps(ujson.loads(self.webhook_fixture_data(self.FIXTURE_DIR_NAME, 'personal'))) + payload = orjson.dumps(orjson.loads(self.webhook_fixture_data(self.FIXTURE_DIR_NAME, 'personal'))) self.client_post(self.url, payload, content_type="application/json") msg = self.get_last_message() diff --git a/zerver/webhooks/trello/view/__init__.py b/zerver/webhooks/trello/view/__init__.py index e6a01d0c3c..64f5afb4ec 100644 --- a/zerver/webhooks/trello/view/__init__.py +++ b/zerver/webhooks/trello/view/__init__.py @@ -1,7 +1,7 @@ # Webhooks for external integrations. from typing import Any, Mapping, Optional, Tuple -import ujson +import orjson from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view, return_success_on_head_request @@ -20,7 +20,7 @@ from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, process_ def api_trello_webhook(request: HttpRequest, user_profile: UserProfile, payload: Mapping[str, Any]=REQ(argument_type='body')) -> HttpResponse: - payload = ujson.loads(request.body) + payload = orjson.loads(request.body) action_type = payload['action'].get('type') try: message = get_subject_and_body(payload, action_type) diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index 41065e7cb2..9accd7df61 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -31,8 +31,8 @@ from typing import ( cast, ) +import orjson import requests -import ujson from django.conf import settings from django.db import connection from django.utils.timezone import now as timezone_now @@ -200,10 +200,10 @@ class QueueProcessingWorker(ABC): fn = os.path.join(settings.QUEUE_STATS_DIR, fname) with lockfile(fn + '.lock'): tmp_fn = fn + '.tmp' - with open(tmp_fn, 'w') as f: - serialized_dict = ujson.dumps(stats_dict, indent=2) - serialized_dict += '\n' - f.write(serialized_dict) + with open(tmp_fn, 'wb') as f: + f.write( + orjson.dumps(stats_dict, option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2) + ) os.rename(tmp_fn, fn) @abstractmethod @@ -254,7 +254,7 @@ class QueueProcessingWorker(ABC): # flow. 'queue_name' is always a constant string. fname = mark_sanitized(f'{self.queue_name}.errors') fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) - line = f'{time.asctime()}\t{ujson.dumps(events)}\n' + line = f'{time.asctime()}\t{orjson.dumps(events).decode()}\n' lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'ab') as f: @@ -319,7 +319,7 @@ class SignupWorker(QueueProcessingWorker): params['list_id'] = settings.ZULIP_FRIENDS_LIST_ID params['status'] = 'subscribed' r = requests.post(endpoint, auth=('apikey', settings.MAILCHIMP_API_KEY), json=params, timeout=10) - if r.status_code == 400 and ujson.loads(r.text)['title'] == 'Member Exists': + if r.status_code == 400 and orjson.loads(r.content)['title'] == 'Member Exists': logging.warning("Attempted to sign up already existing email to list: %s", data['email_address']) elif r.status_code == 400: @@ -595,10 +595,10 @@ class TestWorker(QueueProcessingWorker): # and appends it to a file in /tmp. def consume(self, event: Mapping[str, Any]) -> None: # nocoverage fn = settings.ZULIP_WORKER_TEST_FILE - message = ujson.dumps(event) - logging.info("TestWorker should append this message to %s: %s", fn, message) - with open(fn, 'a') as f: - f.write(message + '\n') + message = orjson.dumps(event) + logging.info("TestWorker should append this message to %s: %s", fn, message.decode()) + with open(fn, 'ab') as f: + f.write(message + b'\n') @assign_queue('embed_links') class FetchLinksEmbedData(QueueProcessingWorker): @@ -721,9 +721,9 @@ class DeferredWorker(QueueProcessingWorker): threads=6, upload=True, public_only=True, delete_after_upload=True) except Exception: - export_event.extra_data = ujson.dumps(dict( + export_event.extra_data = orjson.dumps(dict( failed_timestamp=timezone_now().timestamp(), - )) + )).decode() export_event.save(update_fields=['extra_data']) logging.error( "Data export for %s failed after %s", @@ -735,9 +735,9 @@ class DeferredWorker(QueueProcessingWorker): assert public_url is not None # Update the extra_data field now that the export is complete. - export_event.extra_data = ujson.dumps(dict( + export_event.extra_data = orjson.dumps(dict( export_path=urllib.parse.urlparse(public_url).path, - )) + )).decode() export_event.save(update_fields=['extra_data']) # Send a private message notification letting the user who diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 95cf39afbc..e60588926e 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple import bmemcached -import ujson +import orjson from django.conf import settings from django.contrib.sessions.models import Session from django.core.management import call_command @@ -563,8 +563,8 @@ class Command(BaseCommand): # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. - with open("zerver/tests/fixtures/docs_url_preview_data.json") as f: - urls_with_preview_data = ujson.load(f) + with open("zerver/tests/fixtures/docs_url_preview_data.json", "rb") as f: + urls_with_preview_data = orjson.loads(f.read()) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) @@ -709,8 +709,8 @@ def generate_and_send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping random.seed(random_seed) with open(os.path.join(get_or_create_dev_uuid_var_path('test-backend'), - "test_messages.json")) as infile: - dialog = ujson.load(infile) + "test_messages.json"), "rb") as infile: + dialog = orjson.loads(infile.read()) random.shuffle(dialog) texts = itertools.cycle(dialog) diff --git a/zilencer/management/commands/render_messages.py b/zilencer/management/commands/render_messages.py index b01e6364d0..67a7249e4a 100644 --- a/zilencer/management/commands/render_messages.py +++ b/zilencer/management/commands/render_messages.py @@ -1,7 +1,7 @@ import os from typing import Any, Iterator -import ujson +import orjson from django.core.management.base import BaseCommand, CommandParser from django.db.models import QuerySet @@ -37,8 +37,8 @@ class Command(BaseCommand): if not os.path.exists(dest_dir): os.makedirs(dest_dir) - with open(options['destination'], 'w') as result: - result.write('[') + with open(options['destination'], 'wb') as result: + result.write(b'[') messages = Message.objects.filter(id__gt=latest - amount, id__lte=latest).order_by('id') for message in queryset_iterator(messages): content = message.content @@ -48,16 +48,16 @@ class Command(BaseCommand): # version, extracting it from the edit history if # necessary. if message.edit_history: - history = ujson.loads(message.edit_history) + history = orjson.loads(message.edit_history) history = sorted(history, key=lambda i: i['timestamp']) for entry in history: if 'prev_content' in entry: content = entry['prev_content'] break - result.write(ujson.dumps({ + result.write(orjson.dumps({ 'id': message.id, 'content': render_markdown(message, content), })) if message.id != latest: - result.write(',') - result.write(']') + result.write(b',') + result.write(b']') diff --git a/zproject/backends.py b/zproject/backends.py index 6ef1efcb4e..8c29540517 100644 --- a/zproject/backends.py +++ b/zproject/backends.py @@ -19,7 +19,7 @@ from abc import ABC, abstractmethod from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, cast import magic -import ujson +import orjson from decorator import decorator from django.conf import settings from django.contrib.auth import authenticate, get_backends @@ -1737,7 +1737,7 @@ class SAMLAuthBackend(SocialAuthMixin, SAMLAuth): data_to_relay = { key: request_data[key] for key in params_to_relay if key in request_data } - relay_state = ujson.dumps({"state_token": self.put_data_in_redis(data_to_relay)}) + relay_state = orjson.dumps({"state_token": self.put_data_in_redis(data_to_relay)}).decode() return auth.login(return_to=relay_state) @@ -1788,7 +1788,7 @@ class SAMLAuthBackend(SocialAuthMixin, SAMLAuth): relay_state = request_data['RelayState'] try: - data = ujson.loads(relay_state) + data = orjson.loads(relay_state) if 'state_token' in data: # SP-initiated sign in. We stored relevant information in the first # step of the flow