diff --git a/analytics/lib/counts.py b/analytics/lib/counts.py index f83346a8e1..a3f9ee8406 100644 --- a/analytics/lib/counts.py +++ b/analytics/lib/counts.py @@ -1,23 +1,35 @@ +import logging import time from collections import OrderedDict, defaultdict from datetime import datetime, timedelta -import logging -from typing import Callable, Dict, List, \ - Optional, Tuple, Type, Union +from typing import Callable, Dict, List, Optional, Tuple, Type, Union from django.conf import settings from django.db import connection from django.db.models import F -from psycopg2.sql import Composable, Identifier, Literal, SQL +from psycopg2.sql import SQL, Composable, Identifier, Literal -from analytics.models import BaseCount, \ - FillState, InstallationCount, RealmCount, StreamCount, \ - UserCount, installation_epoch, last_successful_fill +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, + last_successful_fill, +) from zerver.lib.logging_util import log_to_file -from zerver.lib.timestamp import ceiling_to_day, \ - ceiling_to_hour, floor_to_hour, verify_UTC -from zerver.models import Message, Realm, RealmAuditLog, \ - Stream, UserActivityInterval, UserProfile, models +from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Stream, + UserActivityInterval, + UserProfile, + models, +) ## Logging setup ## diff --git a/analytics/lib/fixtures.py b/analytics/lib/fixtures.py index a439b56d95..c707220a19 100644 --- a/analytics/lib/fixtures.py +++ b/analytics/lib/fixtures.py @@ -4,6 +4,7 @@ from typing import List from analytics.lib.counts import CountStat + def generate_time_series_data(days: int=100, business_hours_base: float=10, non_business_hours_base: float=10, growth: float=1, autocorrelation: float=0, spikiness: float=1, diff --git a/analytics/lib/time_utils.py b/analytics/lib/time_utils.py index 48d8f54223..b990ae252d 100644 --- a/analytics/lib/time_utils.py +++ b/analytics/lib/time_utils.py @@ -4,6 +4,7 @@ from typing import List, Optional from analytics.lib.counts import CountStat from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC + # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive. # If min_length is greater than 0, pads the list to the left. # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22], diff --git a/analytics/management/commands/check_analytics_state.py b/analytics/management/commands/check_analytics_state.py index c1a14f0341..698317b6f5 100644 --- a/analytics/management/commands/check_analytics_state.py +++ b/analytics/management/commands/check_analytics_state.py @@ -8,8 +8,7 @@ from django.utils.timezone import now as timezone_now from analytics.lib.counts import COUNT_STATS, CountStat from analytics.models import installation_epoch, last_successful_fill -from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, \ - floor_to_hour, verify_UTC +from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC from zerver.models import Realm states = { diff --git a/analytics/management/commands/populate_analytics_db.py b/analytics/management/commands/populate_analytics_db.py index 462f41447b..70857dc630 100644 --- a/analytics/management/commands/populate_analytics_db.py +++ b/analytics/management/commands/populate_analytics_db.py @@ -1,21 +1,25 @@ from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Type - from unittest import mock + from django.core.management.base import BaseCommand from django.utils.timezone import now as timezone_now -from analytics.lib.counts import COUNT_STATS, CountStat, \ - do_drop_all_analytics_tables +from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables from analytics.lib.fixtures import generate_time_series_data from analytics.lib.time_utils import time_range -from analytics.models import BaseCount, FillState, InstallationCount, \ - RealmCount, StreamCount, UserCount +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, +) from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, do_change_user_role from zerver.lib.create_user import create_user from zerver.lib.timestamp import floor_to_day -from zerver.models import Client, Realm, Recipient, Stream, Subscription, \ - UserProfile +from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserProfile class Command(BaseCommand): diff --git a/analytics/management/commands/realm_stats.py b/analytics/management/commands/realm_stats.py index 8730a3a217..336a092f6c 100644 --- a/analytics/management/commands/realm_stats.py +++ b/analytics/management/commands/realm_stats.py @@ -6,8 +6,17 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import Count from django.utils.timezone import now as timezone_now -from zerver.models import Message, Realm, Recipient, Stream, Subscription, \ - UserActivity, UserMessage, UserProfile, get_realm +from zerver.models import ( + Message, + Realm, + Recipient, + Stream, + Subscription, + UserActivity, + UserMessage, + UserProfile, + get_realm, +) MOBILE_CLIENT_LIST = ["Android", "ios"] HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"] diff --git a/analytics/management/commands/stream_stats.py b/analytics/management/commands/stream_stats.py index b6bde4bdc2..4bb861c00c 100644 --- a/analytics/management/commands/stream_stats.py +++ b/analytics/management/commands/stream_stats.py @@ -4,8 +4,7 @@ from typing import Any from django.core.management.base import BaseCommand, CommandError from django.db.models import Q -from zerver.models import Message, Realm, Recipient, Stream, Subscription, \ - get_realm +from zerver.models import Message, Realm, Recipient, Stream, Subscription, get_realm class Command(BaseCommand): diff --git a/analytics/management/commands/update_analytics_counts.py b/analytics/management/commands/update_analytics_counts.py index 9e89bb2246..2f66409563 100644 --- a/analytics/management/commands/update_analytics_counts.py +++ b/analytics/management/commands/update_analytics_counts.py @@ -1,8 +1,8 @@ import os import time from argparse import ArgumentParser -from typing import Any, Dict from datetime import timezone +from typing import Any, Dict from django.conf import settings from django.core.management.base import BaseCommand diff --git a/analytics/migrations/0015_clear_duplicate_counts.py b/analytics/migrations/0015_clear_duplicate_counts.py index b97f8265fa..9533f21bde 100644 --- a/analytics/migrations/0015_clear_duplicate_counts.py +++ b/analytics/migrations/0015_clear_duplicate_counts.py @@ -3,6 +3,7 @@ from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models import Count, Sum + def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """This is a preparatory migration for our Analytics tables. diff --git a/analytics/models.py b/analytics/models.py index b81ea5cff1..20bcc8b6a6 100644 --- a/analytics/models.py +++ b/analytics/models.py @@ -7,6 +7,7 @@ from django.db.models import Q, UniqueConstraint from zerver.lib.timestamp import floor_to_day from zerver.models import Realm, Stream, UserProfile + class FillState(models.Model): property: str = models.CharField(max_length=40, unique=True) end_time: datetime.datetime = models.DateTimeField() diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index 7992da7af6..52f58f6f5a 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple, Type - from unittest import mock + import ujson from django.apps import apps from django.db import models @@ -10,24 +10,58 @@ from django.test import TestCase from django.utils.timezone import now as timezone_now from psycopg2.sql import SQL, Literal -from analytics.lib.counts import COUNT_STATS, CountStat, get_count_stats, \ - DependentCountStat, LoggingCountStat, do_aggregate_to_summary_table, \ - do_drop_all_analytics_tables, do_drop_single_stat, \ - do_fill_count_stat_at_hour, do_increment_logging_stat, \ - process_count_stat, sql_data_collector -from analytics.models import BaseCount, \ - FillState, InstallationCount, RealmCount, StreamCount, \ - UserCount, installation_epoch -from zerver.lib.actions import do_activate_user, do_create_user, \ - do_deactivate_user, do_reactivate_user, update_user_activity_interval, \ - do_invite_users, do_revoke_user_invite, do_resend_user_invite_email, \ - InvitationError +from analytics.lib.counts import ( + COUNT_STATS, + CountStat, + DependentCountStat, + LoggingCountStat, + do_aggregate_to_summary_table, + do_drop_all_analytics_tables, + do_drop_single_stat, + do_fill_count_stat_at_hour, + do_increment_logging_stat, + get_count_stats, + process_count_stat, + sql_data_collector, +) +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, +) +from zerver.lib.actions import ( + InvitationError, + do_activate_user, + do_create_user, + do_deactivate_user, + do_invite_users, + do_reactivate_user, + do_resend_user_invite_email, + do_revoke_user_invite, + update_user_activity_interval, +) from zerver.lib.create_user import create_user from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day from zerver.lib.topic import DB_TOPIC_NAME -from zerver.models import Client, Huddle, Message, Realm, \ - RealmAuditLog, Recipient, Stream, UserActivityInterval, \ - UserProfile, get_client, get_user, PreregistrationUser +from zerver.models import ( + Client, + Huddle, + Message, + PreregistrationUser, + Realm, + RealmAuditLog, + Recipient, + Stream, + UserActivityInterval, + UserProfile, + get_client, + get_user, +) + class AnalyticsTestCase(TestCase): MINUTE = timedelta(seconds = 60) diff --git a/analytics/tests/test_fixtures.py b/analytics/tests/test_fixtures.py index d1e31142e6..a5e7f47125 100644 --- a/analytics/tests/test_fixtures.py +++ b/analytics/tests/test_fixtures.py @@ -2,6 +2,7 @@ from analytics.lib.counts import CountStat from analytics.lib.fixtures import generate_time_series_data from zerver.lib.test_classes import ZulipTestCase + # A very light test suite; the code being tested is not run in production. class TestFixtures(ZulipTestCase): def test_deterministic_settings(self) -> None: diff --git a/analytics/tests/test_views.py b/analytics/tests/test_views.py index 1b8d1465aa..2ec434b456 100644 --- a/analytics/tests/test_views.py +++ b/analytics/tests/test_views.py @@ -1,24 +1,21 @@ from datetime import datetime, timedelta, timezone from typing import List, Optional - from unittest import mock -from django.http import HttpResponse + import ujson +from django.http import HttpResponse from django.utils.timezone import now as timezone_now from analytics.lib.counts import COUNT_STATS, CountStat from analytics.lib.time_utils import time_range -from analytics.models import FillState, \ - RealmCount, UserCount, last_successful_fill -from analytics.views import rewrite_client_arrays, \ - sort_by_totals, sort_client_labels -from zerver.lib.test_helpers import reset_emails_in_zulip_realm +from analytics.models import FillState, RealmCount, UserCount, last_successful_fill +from analytics.views import rewrite_client_arrays, sort_by_totals, sort_client_labels +from zerver.lib.actions import do_create_multiuse_invite_link, do_send_realm_reactivation_email from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.timestamp import ceiling_to_day, \ - ceiling_to_hour, datetime_to_timestamp -from zerver.lib.actions import do_create_multiuse_invite_link, \ - do_send_realm_reactivation_email -from zerver.models import Client, get_realm, MultiuseInvite +from zerver.lib.test_helpers import reset_emails_in_zulip_realm +from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp +from zerver.models import Client, MultiuseInvite, get_realm + class TestStatsEndpoint(ZulipTestCase): def test_stats(self) -> None: diff --git a/analytics/views.py b/analytics/views.py index 702bb34dc6..3444db001e 100644 --- a/analytics/views.py +++ b/analytics/views.py @@ -6,54 +6,75 @@ import urllib from collections import defaultdict from datetime import datetime, timedelta, timezone from decimal import Decimal - -from typing import Any, Callable, Dict, List, \ - Optional, Set, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union import pytz from django.conf import settings -from django.urls import reverse +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator from django.db import connection from django.db.models.query import QuerySet from django.http import HttpRequest, HttpResponse, HttpResponseNotFound from django.shortcuts import render from django.template import loader +from django.urls import reverse +from django.utils.timesince import timesince from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.utils.timesince import timesince -from django.core.validators import URLValidator -from django.core.exceptions import ValidationError from jinja2 import Markup as mark_safe -from psycopg2.sql import Composable, Literal, SQL +from psycopg2.sql import SQL, Composable, Literal from analytics.lib.counts import COUNT_STATS, CountStat from analytics.lib.time_utils import time_range -from analytics.models import BaseCount, InstallationCount, \ - RealmCount, StreamCount, UserCount, last_successful_fill, installation_epoch -from confirmation.models import Confirmation, confirmation_url, _properties -from zerver.decorator import require_server_admin, require_server_admin_api, \ - to_utc_datetime, zulip_login_required, require_non_guest_user +from analytics.models import ( + BaseCount, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, + last_successful_fill, +) +from confirmation.models import Confirmation, _properties, confirmation_url +from confirmation.settings import STATUS_ACTIVE +from zerver.decorator import ( + require_non_guest_user, + require_server_admin, + require_server_admin_api, + to_utc_datetime, + zulip_login_required, +) +from zerver.lib.actions import ( + do_change_plan_type, + do_deactivate_realm, + do_scrub_realm, + do_send_realm_reactivation_email, +) from zerver.lib.exceptions import JsonableError +from zerver.lib.realm_icon import realm_icon_url from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.timestamp import convert_to_UTC, timestamp_to_datetime -from zerver.lib.realm_icon import realm_icon_url -from zerver.views.invite import get_invitee_emails_set from zerver.lib.subdomains import get_subdomain_from_hostname -from zerver.lib.actions import do_change_plan_type, do_deactivate_realm, \ - do_send_realm_reactivation_email, do_scrub_realm +from zerver.lib.timestamp import convert_to_UTC, timestamp_to_datetime from zerver.lib.validator import to_non_negative_int -from confirmation.settings import STATUS_ACTIVE +from zerver.views.invite import get_invitee_emails_set if settings.BILLING_ENABLED: from corporate.lib.stripe import attach_discount_to_realm, get_discount_for_realm -from zerver.models import Client, get_realm, Realm, UserActivity, UserActivityInterval, \ - UserProfile, PreregistrationUser, MultiuseInvite +from zerver.models import ( + Client, + MultiuseInvite, + PreregistrationUser, + Realm, + UserActivity, + UserActivityInterval, + UserProfile, + get_realm, +) if settings.ZILENCER_ENABLED: - from zilencer.models import RemoteInstallationCount, RemoteRealmCount, \ - RemoteZulipServer + from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer else: from unittest.mock import Mock RemoteInstallationCount = Mock() # type: ignore[misc] # https://github.com/JukkaL/mypy/issues/1188 diff --git a/confirmation/migrations/0001_initial.py b/confirmation/migrations/0001_initial.py index af6fed33e7..75b8fb2f97 100644 --- a/confirmation/migrations/0001_initial.py +++ b/confirmation/migrations/0001_initial.py @@ -1,5 +1,5 @@ -from django.db import models, migrations import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/migrations/0002_realmcreationkey.py b/confirmation/migrations/0002_realmcreationkey.py index 2053e28340..fac60f8186 100644 --- a/confirmation/migrations/0002_realmcreationkey.py +++ b/confirmation/migrations/0002_realmcreationkey.py @@ -1,5 +1,5 @@ -from django.db import models, migrations import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/migrations/0005_confirmation_realm.py b/confirmation/migrations/0005_confirmation_realm.py index 2dc1088e9d..75179cb797 100644 --- a/confirmation/migrations/0005_confirmation_realm.py +++ b/confirmation/migrations/0005_confirmation_realm.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.6 on 2017-11-30 00:13 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/models.py b/confirmation/models.py index c958e57584..8471ed8940 100644 --- a/confirmation/models.py +++ b/confirmation/models.py @@ -1,24 +1,23 @@ # Copyright: (c) 2008, Jarek Zgoda __revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $' - import datetime +import string +from random import SystemRandom +from typing import Dict, Optional, Union +from django.conf import settings +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models import CASCADE -from django.urls import reverse -from django.conf import settings -from django.contrib.contenttypes.models import ContentType -from django.contrib.contenttypes.fields import GenericForeignKey from django.http import HttpRequest, HttpResponse from django.shortcuts import render +from django.urls import reverse from django.utils.timezone import now as timezone_now -from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite, \ - UserProfile, Realm -from random import SystemRandom -import string -from typing import Dict, Optional, Union +from zerver.models import EmailChangeStatus, MultiuseInvite, PreregistrationUser, Realm, UserProfile + class ConfirmationKeyException(Exception): WRONG_LENGTH = 1 diff --git a/corporate/lib/stripe.py b/corporate/lib/stripe.py index 54ee6d3610..2eabbcabc3 100644 --- a/corporate/lib/stripe.py +++ b/corporate/lib/stripe.py @@ -1,26 +1,31 @@ -from datetime import datetime, timedelta -from decimal import Decimal -from functools import wraps import logging import math import os -from typing import Any, Callable, Dict, Optional, TypeVar, Tuple, cast -import ujson +from datetime import datetime, timedelta +from decimal import Decimal +from functools import wraps +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, cast -from django.conf import settings -from django.db import transaction -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now -from django.core.signing import Signer import stripe +import ujson +from django.conf import settings +from django.core.signing import Signer +from django.db import transaction +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from corporate.models import ( + Customer, + CustomerPlan, + LicenseLedger, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) from zerver.lib.logging_util import log_to_file from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.utils import generate_random_token -from zerver.models import Realm, UserProfile, RealmAuditLog -from corporate.models import Customer, CustomerPlan, LicenseLedger, \ - get_current_plan_by_customer, get_customer_by_realm, \ - get_current_plan_by_realm +from zerver.models import Realm, RealmAuditLog, UserProfile from zproject.config import get_secret STRIPE_PUBLISHABLE_KEY = get_secret('stripe_publishable_key') diff --git a/corporate/models.py b/corporate/models.py index bb6ea75fe7..715f10d7e3 100644 --- a/corporate/models.py +++ b/corporate/models.py @@ -7,6 +7,7 @@ from django.db.models import CASCADE from zerver.models import Realm + class Customer(models.Model): realm: Realm = models.OneToOneField(Realm, on_delete=CASCADE) stripe_customer_id: str = models.CharField(max_length=255, null=True, unique=True) diff --git a/corporate/tests/test_stripe.py b/corporate/tests/test_stripe.py index 1be9d5be65..cfd9586917 100644 --- a/corporate/tests/test_stripe.py +++ b/corporate/tests/test_stripe.py @@ -1,43 +1,66 @@ -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from functools import wraps -from unittest.mock import Mock, patch +import json import operator import os import re import sys -from typing import Any, Callable, Dict, List, Optional, TypeVar, Tuple, cast -import ujson -import json -import responses +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast +from unittest.mock import Mock, patch -from django.core import signing -from django.urls.resolvers import get_resolver -from django.http import HttpResponse +import responses +import stripe +import ujson from django.conf import settings +from django.core import signing +from django.http import HttpResponse +from django.urls.resolvers import get_resolver from django.utils.timezone import now as timezone_now -import stripe - -from zerver.lib.actions import do_deactivate_user, do_create_user, \ - do_activate_user, do_reactivate_user, do_deactivate_realm, \ - do_reactivate_realm +from corporate.lib.stripe import ( + MAX_INVOICED_LICENSES, + MIN_INVOICED_LICENSES, + BillingError, + StripeCardError, + add_months, + attach_discount_to_realm, + catch_stripe_errors, + compute_plan_parameters, + get_discount_for_realm, + get_latest_seat_count, + invoice_plan, + invoice_plans_as_needed, + make_end_of_cycle_updates_if_needed, + next_month, + process_initial_upgrade, + sign_string, + stripe_get_customer, + unsign_string, + update_license_ledger_for_automanaged_plan, + update_license_ledger_if_needed, + update_or_create_stripe_customer, +) +from corporate.models import ( + Customer, + CustomerPlan, + LicenseLedger, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) +from zerver.lib.actions import ( + do_activate_user, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import reset_emails_in_zulip_realm -from zerver.lib.timestamp import timestamp_to_datetime, datetime_to_timestamp -from zerver.models import Realm, UserProfile, get_realm, RealmAuditLog -from corporate.lib.stripe import catch_stripe_errors, attach_discount_to_realm, \ - get_latest_seat_count, sign_string, unsign_string, \ - BillingError, StripeCardError, stripe_get_customer, \ - MIN_INVOICED_LICENSES, MAX_INVOICED_LICENSES, \ - add_months, next_month, \ - compute_plan_parameters, update_or_create_stripe_customer, \ - process_initial_upgrade, make_end_of_cycle_updates_if_needed, \ - update_license_ledger_if_needed, update_license_ledger_for_automanaged_plan, \ - invoice_plan, invoice_plans_as_needed, get_discount_for_realm -from corporate.models import Customer, CustomerPlan, LicenseLedger, \ - get_customer_by_realm, get_current_plan_by_customer, \ - get_current_plan_by_realm +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.models import Realm, RealmAuditLog, UserProfile, get_realm CallableT = TypeVar('CallableT', bound=Callable[..., Any]) diff --git a/corporate/urls.py b/corporate/urls.py index c799f7edf2..4ebb7c2e6b 100644 --- a/corporate/urls.py +++ b/corporate/urls.py @@ -1,8 +1,8 @@ from typing import Any -from django.views.generic import TemplateView from django.conf.urls import include from django.urls import path +from django.views.generic import TemplateView import corporate.views from zerver.lib.rest import rest_dispatch diff --git a/corporate/views.py b/corporate/views.py index 0f8c8f3ffc..57f6e8d33b 100644 --- a/corporate/views.py +++ b/corporate/views.py @@ -1,30 +1,45 @@ import logging from decimal import Decimal -import stripe -from typing import Any, Dict, cast, Optional, Union +from typing import Any, Dict, Optional, Union, cast +import stripe +from django.conf import settings from django.core import signing from django.http import HttpRequest, HttpResponse, HttpResponseRedirect -from django.utils.timezone import now as timezone_now -from django.utils.translation import ugettext as _ from django.shortcuts import render from django.urls import reverse -from django.conf import settings +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ -from zerver.decorator import zulip_login_required, require_billing_access +from corporate.lib.stripe import ( + DEFAULT_INVOICE_DAYS_UNTIL_DUE, + MAX_INVOICED_LICENSES, + MIN_INVOICED_LICENSES, + STRIPE_PUBLISHABLE_KEY, + BillingError, + do_change_plan_status, + do_replace_payment_source, + downgrade_now, + get_latest_seat_count, + make_end_of_cycle_updates_if_needed, + process_initial_upgrade, + renewal_amount, + sign_string, + start_of_next_billing_cycle, + stripe_get_customer, + unsign_string, +) +from corporate.models import ( + CustomerPlan, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) +from zerver.decorator import require_billing_access, zulip_login_required from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_string, check_int +from zerver.lib.validator import check_int, check_string from zerver.models import UserProfile -from corporate.lib.stripe import STRIPE_PUBLISHABLE_KEY, \ - stripe_get_customer, get_latest_seat_count, \ - process_initial_upgrade, sign_string, \ - unsign_string, BillingError, do_change_plan_status, do_replace_payment_source, \ - MIN_INVOICED_LICENSES, MAX_INVOICED_LICENSES, DEFAULT_INVOICE_DAYS_UNTIL_DUE, \ - start_of_next_billing_cycle, renewal_amount, \ - make_end_of_cycle_updates_if_needed, downgrade_now -from corporate.models import CustomerPlan, get_current_plan_by_customer, \ - get_customer_by_realm, get_current_plan_by_realm billing_logger = logging.getLogger('corporate.stripe') diff --git a/docs/conf.py b/docs/conf.py index f11bad4cc1..45f49e78bd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,10 +10,8 @@ # # All configuration values have a default; values that are commented out # serve to show the default. - import os import sys - from typing import Any, Dict, List, Optional # If extensions (or modules to document with autodoc) are in another directory, @@ -310,6 +308,7 @@ source_suffix = { # See https://github.com/zulip/zulip/issues/13263 for details. from recommonmark.parser import CommonMarkParser + class CustomCommonMarkParser(CommonMarkParser): def visit_document(self, node): pass diff --git a/frontend_tests/run-casper b/frontend_tests/run-casper index e59ad4f3b6..0261bdf8c4 100755 --- a/frontend_tests/run-casper +++ b/frontend_tests/run-casper @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import argparse +import glob +import os +import shlex import subprocess import sys -import os -import glob -import shlex # # In order to use remote casperjs debugging, pass the --remote-debug flag @@ -64,13 +64,14 @@ sys.path.insert(0, ZULIP_PATH) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) +from typing import Iterable, List + from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files from tools.lib.test_server import test_server_running -from typing import Iterable, List - assert_provisioning_status_ok(options.force) os.chdir(ZULIP_PATH) diff --git a/manage.py b/manage.py index ce8204c393..30f8fb0e7b 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +import configparser import os import sys -import configparser + if sys.version_info <= (3, 0): print("Error: Zulip is a Python 3 project, and cannot be run with Python 2.") print("Use e.g. `/path/to/manage.py` not `python /path/to/manage.py`.") @@ -37,6 +38,7 @@ if __name__ == "__main__": from django.conf import settings from django.core.management import execute_from_command_line from django.core.management.base import CommandError + from scripts.lib.zulip_tools import log_management_command log_management_command(" ".join(sys.argv), settings.MANAGEMENT_LOG_PATH) diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file index 64f53c11e7..86df41348a 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file @@ -6,6 +6,7 @@ import sys import time from typing import Tuple + def nagios_from_file(results_file: str, max_time_diff: int=60 * 2) -> 'Tuple[int, str]': """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time index 739a78c7a4..67347bfa45 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time @@ -8,12 +8,12 @@ It supports both munin and nagios outputs It must be run on a machine that is using the live database for the Django ORM. """ -import sys import argparse +import os import random +import sys import time import traceback -import os sys.path.append('.') sys.path.append('/home/zulip/deployments/current') @@ -21,10 +21,10 @@ from scripts.lib.setup_path import setup_path setup_path() -import django - from typing import Any, Dict, List, Optional +import django + usage = """Usage: send-receive.py [options] [config] 'config' is optional, if present will return config info. @@ -75,9 +75,10 @@ os.environ['DJANGO_SETTINGS_MODULE'] = "zproject.settings" django.setup() -from zerver.models import get_system_bot from django.conf import settings +from zerver.models import get_system_bot + states = { "OK": 0, "WARNING": 1, diff --git a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag index 32dd089e05..f209cc32e1 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag +++ b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag @@ -4,10 +4,9 @@ Nagios plugin to check the difference between the primary and secondary Postgres servers' xlog location. """ - import configparser -import subprocess import re +import subprocess from typing import NoReturn states = { diff --git a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log index 9b19f5cbc6..d5665b98e1 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log +++ b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log @@ -4,6 +4,7 @@ Nagios plugin to check the length of the FTS update log. """ import sys + sys.path.append('/home/zulip/deployments/current') try: from scripts.lib.setup_path import setup_path diff --git a/puppet/zulip/files/postgresql/pg_backup_and_purge b/puppet/zulip/files/postgresql/pg_backup_and_purge index e4cd6d3d39..96c1fc5979 100755 --- a/puppet/zulip/files/postgresql/pg_backup_and_purge +++ b/puppet/zulip/files/postgresql/pg_backup_and_purge @@ -1,16 +1,16 @@ #!/usr/bin/env python3 - -import os -import sys import glob +import logging +import os import shlex import subprocess -import logging -import dateutil.parser +import sys import time from datetime import datetime, timedelta, timezone from typing import Dict, List +import dateutil.parser + logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s") logger = logging.getLogger(__name__) diff --git a/puppet/zulip/files/postgresql/process_fts_updates b/puppet/zulip/files/postgresql/process_fts_updates index 8aedfe20ff..d79a6bcc1e 100755 --- a/puppet/zulip/files/postgresql/process_fts_updates +++ b/puppet/zulip/files/postgresql/process_fts_updates @@ -23,14 +23,15 @@ except ImportError: pass import argparse +import configparser +import logging +import os +import select +import sys +import time + import psycopg2 import psycopg2.extensions -import select -import time -import logging -import configparser -import sys -import os BATCH_SIZE = 1000 diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors index 7e66611cbf..fe498f3e04 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors @@ -8,9 +8,9 @@ This script works by just monitoring the files under mirrors when they receive the messages sent every minute by /etc/cron.d/test_zephyr_personal_mirrors """ -from typing import Dict import os import time +from typing import Dict RESULTS_DIR: str = "/home/zulip/mirror_status" diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness index 13b1fdff8a..59c4cd4303 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness @@ -24,9 +24,9 @@ sys.path.append('/home/zulip/deployments/current/zerver') django.setup() -from zerver.models import UserActivity +from typing import Any, Dict, Optional, Set -from typing import Any, Dict, Set, Optional +from zerver.models import UserActivity states: Dict[str, int] = { "OK": 0, diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror index ed7d8a9ed5..d095b257ad 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror @@ -9,9 +9,9 @@ run out of cron. See puppet/zulip_ops/files/cron.d/zephyr-mirror for the crontab details. """ -from typing import Dict import os import time +from typing import Dict RESULTS_FILE = "/var/lib/nagios_state/check-mirroring-results" diff --git a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces index d5ecbcf852..9c94333e94 100755 --- a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces +++ b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces @@ -40,16 +40,15 @@ Note that it currently does not handle the deconfiguration of interfaces. ''' - -import sys import logging import logging.handlers import subprocess +import sys +from typing import Optional import boto.utils import netifaces -from typing import Optional def address_of(device_id: int) -> Optional[str]: try: diff --git a/scripts/lib/check_rabbitmq_queue.py b/scripts/lib/check_rabbitmq_queue.py index cb4095539d..614728aa8e 100644 --- a/scripts/lib/check_rabbitmq_queue.py +++ b/scripts/lib/check_rabbitmq_queue.py @@ -1,9 +1,8 @@ +import json import os import re -import time import subprocess -import json - +import time from collections import defaultdict from typing import Any, DefaultDict, Dict, List diff --git a/scripts/lib/clean-unused-caches b/scripts/lib/clean-unused-caches index 480c6a2e03..5c548918bc 100755 --- a/scripts/lib/clean-unused-caches +++ b/scripts/lib/clean-unused-caches @@ -4,8 +4,9 @@ import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) +from scripts.lib import clean_emoji_cache, clean_node_cache, clean_venv_cache from scripts.lib.zulip_tools import parse_cache_script_args -from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache + def main() -> None: args = parse_cache_script_args("This script cleans unused zulip caches.") diff --git a/scripts/lib/clean_emoji_cache.py b/scripts/lib/clean_emoji_cache.py index aaa6936437..6457949c54 100755 --- a/scripts/lib/clean_emoji_cache.py +++ b/scripts/lib/clean_emoji_cache.py @@ -2,14 +2,16 @@ import argparse import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, \ - parse_cache_script_args, purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache" diff --git a/scripts/lib/clean_node_cache.py b/scripts/lib/clean_node_cache.py index f4b2868738..b1fd1db0c3 100755 --- a/scripts/lib/clean_node_cache.py +++ b/scripts/lib/clean_node_cache.py @@ -2,14 +2,16 @@ import argparse import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, parse_cache_script_args, \ - purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() NODE_MODULES_CACHE_PATH = "/srv/zulip-npm-cache" diff --git a/scripts/lib/clean_venv_cache.py b/scripts/lib/clean_venv_cache.py index dd03b8aa99..513b4d532f 100755 --- a/scripts/lib/clean_venv_cache.py +++ b/scripts/lib/clean_venv_cache.py @@ -3,15 +3,17 @@ import argparse import glob import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) from scripts.lib.hash_reqs import expand_reqs, hash_deps -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, parse_cache_script_args, \ - purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() VENV_CACHE_DIR = '/srv/zulip-venv-cache' diff --git a/scripts/lib/create-production-venv b/scripts/lib/create-production-venv index d7b50cd76b..16c27a4a13 100755 --- a/scripts/lib/create-production-venv +++ b/scripts/lib/create-production-venv @@ -1,17 +1,14 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import os_families, overwrite_symlink, run, parse_os_release -from scripts.lib.setup_venv import ( - setup_virtualenv, get_venv_dependencies, -) +from scripts.lib.setup_venv import get_venv_dependencies, setup_virtualenv +from scripts.lib.zulip_tools import os_families, overwrite_symlink, parse_os_release, run parser = argparse.ArgumentParser(description="Create a production virtualenv with caching") parser.add_argument("deploy_path") diff --git a/scripts/lib/create-thumbor-venv b/scripts/lib/create-thumbor-venv index 3ecf1bf533..f22a8c7219 100755 --- a/scripts/lib/create-thumbor-venv +++ b/scripts/lib/create-thumbor-venv @@ -1,17 +1,18 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import os_families, run, parse_os_release from scripts.lib.setup_venv import ( - setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES, + THUMBOR_VENV_DEPENDENCIES, + YUM_THUMBOR_VENV_DEPENDENCIES, + setup_virtualenv, ) +from scripts.lib.zulip_tools import os_families, parse_os_release, run parser = argparse.ArgumentParser(description="Create a thumbor virtualenv with caching") parser.add_argument("deploy_path") diff --git a/scripts/lib/email-mirror-postfix b/scripts/lib/email-mirror-postfix index e2cd3d58e0..7b793cb7ae 100755 --- a/scripts/lib/email-mirror-postfix +++ b/scripts/lib/email-mirror-postfix @@ -40,21 +40,16 @@ Also you can use optional keys to configure the script and change default values -t Disable sending request to the Zulip server. Default value: False. """ - +import argparse +import json import os +import posix import ssl import sys - -import argparse - -import posix -import json - -from urllib.parse import urljoin, urlencode -from urllib.request import Request, urlopen -from urllib.error import HTTPError from configparser import RawConfigParser - +from urllib.error import HTTPError +from urllib.parse import urlencode, urljoin +from urllib.request import Request, urlopen parser = argparse.ArgumentParser() diff --git a/scripts/lib/hash_reqs.py b/scripts/lib/hash_reqs.py index 6115b5ef12..219ee20445 100755 --- a/scripts/lib/hash_reqs.py +++ b/scripts/lib/hash_reqs.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -import os -import sys import argparse import hashlib +import os +import sys from typing import Iterable, List, MutableSet + def expand_reqs_helper(fpath: str, visited: MutableSet[str]) -> List[str]: if fpath in visited: return [] diff --git a/scripts/lib/node_cache.py b/scripts/lib/node_cache.py index 42181daab2..b292e8c3af 100644 --- a/scripts/lib/node_cache.py +++ b/scripts/lib/node_cache.py @@ -1,10 +1,10 @@ -import os import hashlib import json +import os import shutil +from typing import List, Optional -from typing import Optional, List -from scripts.lib.zulip_tools import subprocess_text_output, run +from scripts.lib.zulip_tools import run, subprocess_text_output ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_SRV_PATH = "/srv" diff --git a/scripts/lib/pythonrc.py b/scripts/lib/pythonrc.py index 2b5e4a9a16..016d180943 100644 --- a/scripts/lib/pythonrc.py +++ b/scripts/lib/pythonrc.py @@ -1,8 +1,9 @@ try: from django.conf import settings # noqa: F401 - from zerver.models import * # noqa: F401, F403 - from zerver.lib.actions import * # noqa: F401, F403 + from analytics.models import * # noqa: F401, F403 + from zerver.lib.actions import * # noqa: F401, F403 + from zerver.models import * # noqa: F401, F403 except Exception: import traceback print("\nException importing Zulip core modules on startup!") diff --git a/scripts/lib/queue_workers.py b/scripts/lib/queue_workers.py index 12de371992..68b337c38a 100755 --- a/scripts/lib/queue_workers.py +++ b/scripts/lib/queue_workers.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import sys @@ -13,6 +12,7 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import django + django.setup() from zerver.worker.queue_processors import get_active_worker_queues diff --git a/scripts/lib/setup_path.py b/scripts/lib/setup_path.py index c8748bd9da..20a190242a 100644 --- a/scripts/lib/setup_path.py +++ b/scripts/lib/setup_path.py @@ -1,10 +1,10 @@ """ Use libraries from a virtualenv (by modifying sys.path) in production. """ - import os import sys + def setup_path() -> None: if os.path.basename(sys.prefix) != "zulip-py3-venv": BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index b3c696806f..155df9818e 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -2,10 +2,10 @@ import logging import os import shutil import subprocess -from scripts.lib.zulip_tools import run, run_as_root, ENDC, WARNING, os_families -from scripts.lib.hash_reqs import expand_reqs +from typing import List, Optional, Set, Tuple -from typing import List, Optional, Tuple, Set +from scripts.lib.hash_reqs import expand_reqs +from scripts.lib.zulip_tools import ENDC, WARNING, os_families, run, run_as_root ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) VENV_CACHE_PATH = "/srv/zulip-venv-cache" diff --git a/scripts/lib/sharding.py b/scripts/lib/sharding.py index df68fbfe09..fad230cca0 100755 --- a/scripts/lib/sharding.py +++ b/scripts/lib/sharding.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import json import os import subprocess @@ -14,6 +13,7 @@ setup_path() from scripts.lib.zulip_tools import get_config_file + def write_realm_nginx_config_line(f: Any, host: str, port: str) -> None: f.write("""if ($host = '{}') {{ set $tornado_server http://tornado{}; diff --git a/scripts/lib/unpack-zulip b/scripts/lib/unpack-zulip index 4d73e1b257..bf22d02053 100755 --- a/scripts/lib/unpack-zulip +++ b/scripts/lib/unpack-zulip @@ -1,17 +1,24 @@ #!/usr/bin/env python3 +import glob import os import shutil -import sys import subprocess +import sys import tempfile -import glob os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', "..")) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, make_deploy_path, \ - get_deployment_version, is_invalid_upgrade, overwrite_symlink import version +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + ENDC, + FAIL, + get_deployment_version, + is_invalid_upgrade, + make_deploy_path, + overwrite_symlink, +) if len(sys.argv) != 2: print(FAIL + f"Usage: {sys.argv[0]} " + ENDC) diff --git a/scripts/lib/upgrade-zulip b/scripts/lib/upgrade-zulip index 1454b4a0b0..5a1105ea19 100755 --- a/scripts/lib/upgrade-zulip +++ b/scripts/lib/upgrade-zulip @@ -1,19 +1,27 @@ #!/usr/bin/env python3 +import configparser +import logging import os import shutil -import sys import subprocess -import logging +import sys import time -import configparser TARBALL_ARCHIVE_PATH = "/home/zulip/archives" os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, \ - su_to_zulip, get_deployment_lock, release_deployment_lock, assert_running_as_root, \ - get_config_file, get_deploy_options +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + ENDC, + FAIL, + assert_running_as_root, + get_config_file, + get_deploy_options, + get_deployment_lock, + release_deployment_lock, + su_to_zulip, +) config_file: configparser.RawConfigParser = get_config_file() deploy_options = get_deploy_options(config_file) diff --git a/scripts/lib/upgrade-zulip-from-git b/scripts/lib/upgrade-zulip-from-git index 406354b14b..2359bf9f2c 100755 --- a/scripts/lib/upgrade-zulip-from-git +++ b/scripts/lib/upgrade-zulip-from-git @@ -1,18 +1,27 @@ #!/usr/bin/env python3 -import os -import sys -import subprocess -import logging -import time import argparse +import logging +import os +import subprocess +import sys +import time LOCAL_GIT_CACHE_DIR = '/srv/zulip.git' os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, make_deploy_path, \ - get_deployment_lock, overwrite_symlink, release_deployment_lock, su_to_zulip, assert_running_as_root, \ - get_config_file, get_deploy_options, get_config +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + assert_running_as_root, + get_config, + get_config_file, + get_deploy_options, + get_deployment_lock, + make_deploy_path, + overwrite_symlink, + release_deployment_lock, + su_to_zulip, +) config_file = get_config_file() deploy_options = get_deploy_options(config_file) diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index 8cbc48d730..3677f3a61c 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -8,10 +8,10 @@ import argparse import configparser import glob import hashlib -import subprocess -import os -import sys import logging +import os +import subprocess +import sys import time os.environ["PYTHONUNBUFFERED"] = "y" @@ -22,8 +22,12 @@ os.environ["LANG"] = "en_US.UTF-8" os.environ["LANGUAGE"] = "en_US.UTF-8" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, su_to_zulip, \ - assert_running_as_root, parse_os_release +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + assert_running_as_root, + parse_os_release, + su_to_zulip, +) assert_running_as_root() diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py index dce756d64c..39e96956f7 100755 --- a/scripts/lib/zulip_tools.py +++ b/scripts/lib/zulip_tools.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse +import configparser import datetime import functools import hashlib +import json import logging import os import pwd @@ -13,11 +15,8 @@ import subprocess import sys import tempfile import time -import json import uuid -import configparser - -from typing import Sequence, Set, Any, Dict, List +from typing import Any, Dict, List, Sequence, Set DEPLOYMENTS_DIR = "/home/zulip/deployments" LOCK_DIR = os.path.join(DEPLOYMENTS_DIR, "lock") diff --git a/scripts/nagios/check-rabbitmq-consumers b/scripts/nagios/check-rabbitmq-consumers index cf4bf7e0ca..8cf11a9f2f 100755 --- a/scripts/nagios/check-rabbitmq-consumers +++ b/scripts/nagios/check-rabbitmq-consumers @@ -1,12 +1,11 @@ #!/usr/bin/env python3 - -import sys -import time import argparse import configparser -from collections import defaultdict import os import subprocess +import sys +import time +from collections import defaultdict from typing import Dict ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/nagios/cron_file_helper.py b/scripts/nagios/cron_file_helper.py index f8a7954e3c..ca92ece168 100644 --- a/scripts/nagios/cron_file_helper.py +++ b/scripts/nagios/cron_file_helper.py @@ -1,7 +1,7 @@ import time - from typing import Tuple + def nagios_from_file(results_file: str) -> Tuple[int, str]: """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/scripts/purge-old-deployments b/scripts/purge-old-deployments index f82759c94b..a1fcc56df7 100755 --- a/scripts/purge-old-deployments +++ b/scripts/purge-old-deployments @@ -3,13 +3,12 @@ import argparse import os import subprocess import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, \ - may_be_perform_purging +from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, may_be_perform_purging + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( diff --git a/scripts/restart-server b/scripts/restart-server index 6a0bb789e5..33b2f132c5 100755 --- a/scripts/restart-server +++ b/scripts/restart-server @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import argparse import configparser -import os -import sys -import pwd -import subprocess import logging -import time +import os +import pwd import shlex +import subprocess +import sys +import time sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import ENDC, OKGREEN, WARNING, DEPLOYMENTS_DIR, overwrite_symlink +from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, ENDC, OKGREEN, WARNING, overwrite_symlink logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s restart-server: %(message)s", diff --git a/scripts/setup/flush-memcached b/scripts/setup/flush-memcached index d9c59ddd4c..dafc37bc6b 100755 --- a/scripts/setup/flush-memcached +++ b/scripts/setup/flush-memcached @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import sys @@ -10,9 +9,10 @@ from scripts.lib.setup_path import setup_path setup_path() -from zproject import settings import pylibmc +from zproject import settings + pylibmc.Client( [settings.MEMCACHED_LOCATION], binary=True, diff --git a/scripts/setup/generate_secrets.py b/scripts/setup/generate_secrets.py index 864c684ec7..1bb880e278 100755 --- a/scripts/setup/generate_secrets.py +++ b/scripts/setup/generate_secrets.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # This tools generates /etc/zulip/zulip-secrets.conf - -import sys import os - +import sys from typing import Dict, List BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -15,8 +13,8 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import argparse -import uuid import configparser +import uuid os.chdir(os.path.join(os.path.dirname(__file__), '..', '..')) @@ -140,6 +138,7 @@ def generate_secrets(development: bool = False) -> None: # file directly. import redis + from zerver.lib.redis_utils import get_redis_client redis_password = random_token() diff --git a/scripts/setup/inline_email_css.py b/scripts/setup/inline_email_css.py index 58bf6bdc3e..418ff150f4 100755 --- a/scripts/setup/inline_email_css.py +++ b/scripts/setup/inline_email_css.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import os - -from premailer import Premailer -from cssutils import profile -from cssutils.profiles import Profiles, properties, macros from typing import Set +from cssutils import profile +from cssutils.profiles import Profiles, macros, properties +from premailer import Premailer + ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') EMAIL_TEMPLATES_PATH = os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails') COMPILED_EMAIL_TEMPLATES_PATH = os.path.join(EMAIL_TEMPLATES_PATH, 'compiled') diff --git a/scripts/setup/restore-backup b/scripts/setup/restore-backup index 00f3b3e713..432cf4a61c 100755 --- a/scripts/setup/restore-backup +++ b/scripts/setup/restore-backup @@ -1,17 +1,15 @@ #!/usr/bin/env python3 - import argparse import os import re import subprocess import sys import tempfile - from typing import IO BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(BASE_DIR) -from scripts.lib.zulip_tools import su_to_zulip, run +from scripts.lib.zulip_tools import run, su_to_zulip POSTGRES_USER = "postgres" diff --git a/scripts/zulip-puppet-apply b/scripts/zulip-puppet-apply index 672591c44a..c0e1194c70 100755 --- a/scripts/zulip-puppet-apply +++ b/scripts/zulip-puppet-apply @@ -1,11 +1,11 @@ #!/usr/bin/env python3 - -import os -import sys -import subprocess import configparser +import os import re -from lib.zulip_tools import parse_os_release, assert_running_as_root +import subprocess +import sys + +from lib.zulip_tools import assert_running_as_root, parse_os_release assert_running_as_root() BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/static/assets/favicon/generate b/static/assets/favicon/generate index 9ba478fd64..ae94f76001 100755 --- a/static/assets/favicon/generate +++ b/static/assets/favicon/generate @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import xml.etree.ElementTree as ET import subprocess +from xml.etree import ElementTree as ET # Generates the favicon images containing unread message counts. diff --git a/tools/check-capitalization b/tools/check-capitalization index c99f08ceab..1645ed9816 100755 --- a/tools/check-capitalization +++ b/tools/check-capitalization @@ -2,6 +2,7 @@ # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) import argparse @@ -12,8 +13,7 @@ import subprocess import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import WARNING, FAIL, ENDC - +from scripts.lib.zulip_tools import ENDC, FAIL, WARNING from tools.lib.capitalization import check_capitalization DJANGO_PO_REGEX = re.compile('msgid "(.*?)"') diff --git a/tools/check-frontend-i18n b/tools/check-frontend-i18n index 24c40a70da..6c119cd201 100755 --- a/tools/check-frontend-i18n +++ b/tools/check-frontend-i18n @@ -1,18 +1,20 @@ #!/usr/bin/env python3 - from typing import List + # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) import argparse import json import os -import sys import subprocess +import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import WARNING, FAIL, ENDC +from scripts.lib.zulip_tools import ENDC, FAIL, WARNING + def find_handlebars(translatable_strings: List[str]) -> List[str]: errored = [] diff --git a/tools/check-issue-labels b/tools/check-issue-labels index fea01f1fc7..5115df2fe6 100755 --- a/tools/check-issue-labels +++ b/tools/check-issue-labels @@ -1,13 +1,13 @@ #!/usr/bin/env python3 +import argparse +import os +import re +import sys +from typing import Any, Dict, Optional import requests -import re -import argparse -import sys -import os import ConfigParser -from typing import Any, Dict, Optional # Scans zulip repositary for issues that don't have any `area` labels. # GitHub API token is required as GitHub limits unauthenticated diff --git a/tools/check-provision b/tools/check-provision index 41d5a5bcbb..fe64e18daa 100755 --- a/tools/check-provision +++ b/tools/check-provision @@ -1,16 +1,14 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys tools_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) -from tools.lib.test_script import ( - assert_provisioning_status_ok, -) +from tools.lib.test_script import assert_provisioning_status_ok + def run() -> None: parser = argparse.ArgumentParser() diff --git a/tools/check-templates b/tools/check-templates index 7d1859cbb4..deb796482a 100755 --- a/tools/check-templates +++ b/tools/check-templates @@ -1,18 +1,20 @@ #!/usr/bin/env python3 -from lib.template_parser import validate -from lib.html_branches import build_id_dict -from lib.pretty_print import validate_indent_html import argparse -import sys import logging +import sys # check for the venv from lib import sanity_check +from lib.html_branches import build_id_dict +from lib.pretty_print import validate_indent_html +from lib.template_parser import validate + sanity_check.check_venv(__file__) -from zulint import lister from typing import Dict, Iterable, List +from zulint import lister + EXCLUDED_FILES = [ ## Test data Files for testing modules in tests "tools/tests/test_template_data", diff --git a/tools/check-thirdparty b/tools/check-thirdparty index 501b8f5c98..fc0f0b0175 100755 --- a/tools/check-thirdparty +++ b/tools/check-thirdparty @@ -7,12 +7,12 @@ Disclaimer: This script is not a lawyer. It cannot validate that the claimed licenses are correct. It can only check for basic syntactic issues. """ - import difflib import io import os import subprocess import sys + from debian import copyright COPYRIGHT_FILENAME = "docs/THIRDPARTY" diff --git a/tools/create-test-api-docs b/tools/create-test-api-docs index 2c313659d2..7ba2beb421 100755 --- a/tools/create-test-api-docs +++ b/tools/create-test-api-docs @@ -2,14 +2,15 @@ # check for the venv from lib import sanity_check -sanity_check.check_venv(__file__) -from collections import defaultdict -from typing import Any, Dict, List, Set +sanity_check.check_venv(__file__) import html import os import pprint +from collections import defaultdict +from typing import Any, Dict, List, Set + import ujson Call = Dict[str, Any] diff --git a/tools/diagnose b/tools/diagnose index 657ab87520..80f1ce75e7 100755 --- a/tools/diagnose +++ b/tools/diagnose @@ -1,17 +1,16 @@ #!/usr/bin/env python3 - import os import platform import shlex -import sys import subprocess - +import sys from typing import Callable, List TOOLS_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.dirname(TOOLS_DIR) sys.path.insert(0, ROOT_DIR) from scripts.lib.zulip_tools import get_dev_uuid_var_path + UUID_VAR_PATH = get_dev_uuid_var_path() def run(check_func: Callable[[], bool]) -> None: @@ -88,7 +87,7 @@ def test_models() -> bool: os.environ['DJANGO_SETTINGS_MODULE'] = settings_module import django django.setup() - from zerver.models import UserProfile, Realm + from zerver.models import Realm, UserProfile print('Num realms: ', Realm.objects.count()) print('Num users: ', UserProfile.objects.count()) return True diff --git a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py index 7f51832fc1..010af602b1 100644 --- a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py +++ b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py @@ -1,8 +1,9 @@ import optparse -from scrapy.crawler import Crawler -from scrapy.commands import crawl from typing import List, Union +from scrapy.commands import crawl +from scrapy.crawler import Crawler + class Command(crawl.Command): def run(self, args: List[str], opts: optparse.Values) -> None: diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py index 861553dad4..86dce36313 100755 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py @@ -1,6 +1,5 @@ import os import pathlib - from typing import List from .common.spiders import BaseDocumentationSpider diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py index 4b6bb43ac8..082001eaee 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py @@ -1,12 +1,10 @@ import os - from posixpath import basename +from typing import Any, List, Set from urllib.parse import urlparse from .common.spiders import BaseDocumentationSpider -from typing import Any, List, Set - def get_images_dir(images_path: str) -> str: # Get index html file as start url and convert it to file uri diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py index b75cdf8570..6d17c789a5 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py @@ -1,7 +1,8 @@ import json import re -import scrapy +from typing import Callable, Iterable, List, Optional, Union +import scrapy from scrapy.http import Request, Response from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor @@ -9,8 +10,6 @@ from scrapy.spidermiddlewares.httperror import HttpError from scrapy.utils.url import url_has_any_extension from twisted.python.failure import Failure -from typing import Callable, Iterable, List, Optional, Union - EXCLUDED_URLS = [ # Google calendar returns 404s on HEAD requests unconditionally 'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com', diff --git a/tools/droplets/add_mentor.py b/tools/droplets/add_mentor.py index f505828120..75a977f973 100644 --- a/tools/droplets/add_mentor.py +++ b/tools/droplets/add_mentor.py @@ -9,13 +9,12 @@ # machine: # # $ python3 add_mentor.py --remove - import os +import re +import socket import sys from argparse import ArgumentParser from typing import List -import socket -import re import requests diff --git a/tools/droplets/create.py b/tools/droplets/create.py index 0c2b8e6e2a..320f01fbb4 100644 --- a/tools/droplets/create.py +++ b/tools/droplets/create.py @@ -14,19 +14,18 @@ # Copy conf.ini-template to conf.ini and populate with your api token. # # usage: python3 create.py - -import sys +import argparse import configparser +import json +import os +import sys +import time import urllib.error import urllib.request -import json -import digitalocean -import time -import argparse -import os - from typing import Any, Dict, List +import digitalocean + # initiation argument parser parser = argparse.ArgumentParser(description='Create a Zulip devopment VM Digital Ocean droplet.') parser.add_argument("username", help="Github username for whom you want to create a Zulip dev droplet") diff --git a/tools/fetch-contributor-data b/tools/fetch-contributor-data index 40a82d06fd..327b921e44 100755 --- a/tools/fetch-contributor-data +++ b/tools/fetch-contributor-data @@ -3,31 +3,32 @@ Fetch contributors data from Github using their API, convert it to structured JSON data for the /team page contributors section. """ - import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from scripts.lib.setup_path import setup_path + setup_path() -from typing import Any, Dict, List, Optional, Union -from typing_extensions import TypedDict - import argparse -from time import sleep +import logging from datetime import date from random import randrange -import logging +from time import sleep +from typing import Any, Dict, List, Optional, Union + +from typing_extensions import TypedDict os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import django + django.setup() -from django.conf import settings +import json import requests -import json +from django.conf import settings duplicate_commits_file = os.path.join(os.path.dirname(__file__), 'duplicate_commits.json') diff --git a/tools/generate-integration-docs-screenshot b/tools/generate-integration-docs-screenshot index c7b419e511..44aefa6fd8 100755 --- a/tools/generate-integration-docs-screenshot +++ b/tools/generate-integration-docs-screenshot @@ -5,7 +5,6 @@ from lib import sanity_check sanity_check.check_venv(__file__) - import os import sys @@ -26,23 +25,33 @@ import argparse import base64 import subprocess from typing import Any, Dict, Optional -from urllib.parse import urlencode, parse_qsl +from urllib.parse import parse_qsl, urlencode import requests import ujson from scripts.lib.zulip_tools import BOLDRED, ENDC from tools.lib.test_script import prepare_puppeteer_run -from zerver.models import UserProfile, Message, get_user_by_delivery_email, get_realm from zerver.lib.actions import ( - do_create_user, notify_created_bot, bulk_add_subscriptions, do_change_avatar_fields) + bulk_add_subscriptions, + do_change_avatar_fields, + do_create_user, + notify_created_bot, +) +from zerver.lib.integrations import ( + DOC_SCREENSHOT_CONFIG, + INTEGRATIONS, + ScreenshotConfig, + WebhookIntegration, + get_fixture_and_image_paths, + split_fixture_path, +) +from zerver.lib.storage import static_path from zerver.lib.streams import create_stream_if_needed from zerver.lib.upload import upload_avatar_image -from zerver.lib.integrations import ( - WebhookIntegration, INTEGRATIONS, split_fixture_path, ScreenshotConfig, get_fixture_and_image_paths, - DOC_SCREENSHOT_CONFIG) from zerver.lib.webhooks.common import get_fixture_http_headers -from zerver.lib.storage import static_path +from zerver.models import Message, UserProfile, get_realm, get_user_by_delivery_email + def create_integration_bot(integration: WebhookIntegration, bot_name: Optional[str]=None) -> UserProfile: realm = get_realm('zulip') diff --git a/tools/i18n/process-mobile-i18n b/tools/i18n/process-mobile-i18n index e8be13254d..1ce108879e 100755 --- a/tools/i18n/process-mobile-i18n +++ b/tools/i18n/process-mobile-i18n @@ -5,6 +5,7 @@ import re from subprocess import check_output from typing import Dict, List + def get_json_filename(locale: str) -> str: return f"locale/{locale}/mobile.json" diff --git a/tools/i18n/tagmessages b/tools/i18n/tagmessages index 853a4d2044..9ad637d8b1 100755 --- a/tools/i18n/tagmessages +++ b/tools/i18n/tagmessages @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import configparser -from hashlib import md5 import os +from hashlib import md5 import polib diff --git a/tools/lib/capitalization.py b/tools/lib/capitalization.py index 49aaba7db8..3386d47cfc 100644 --- a/tools/lib/capitalization.py +++ b/tools/lib/capitalization.py @@ -1,5 +1,5 @@ -from typing import List, Tuple, Match import re +from typing import List, Match, Tuple from bs4 import BeautifulSoup diff --git a/tools/lib/gitlint-rules.py b/tools/lib/gitlint-rules.py index 810267dd44..de77c52cc5 100644 --- a/tools/lib/gitlint-rules.py +++ b/tools/lib/gitlint-rules.py @@ -1,9 +1,9 @@ -from typing import Text, List +import re +from typing import List, Text from gitlint.git import GitCommit -from gitlint.rules import LineRule, RuleViolation, CommitMessageTitle from gitlint.options import StrOption -import re +from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation # Word list from https://github.com/m1foley/fit-commit # Copyright (c) 2015 Mike Foley diff --git a/tools/lib/html_branches.py b/tools/lib/html_branches.py index fbfce1a2ec..ceb588e46d 100644 --- a/tools/lib/html_branches.py +++ b/tools/lib/html_branches.py @@ -1,13 +1,8 @@ -from typing import Dict, List, Optional, Set - import re from collections import defaultdict +from typing import Dict, List, Optional, Set -from .template_parser import ( - tokenize, - FormattedException, - Token, -) +from .template_parser import FormattedException, Token, tokenize class HtmlBranchesException(Exception): diff --git a/tools/lib/pretty_print.py b/tools/lib/pretty_print.py index 6f72984f4b..2abc94ac2b 100644 --- a/tools/lib/pretty_print.py +++ b/tools/lib/pretty_print.py @@ -1,13 +1,10 @@ +import subprocess from typing import Any, Dict, List -from .template_parser import ( - tokenize, - is_django_block_tag, -) +from zulint.printer import ENDC, GREEN -from zulint.printer import GREEN, ENDC +from .template_parser import is_django_block_tag, tokenize -import subprocess def pretty_print_html(html: str, num_spaces: int = 4) -> str: # We use 1-based indexing for both rows and columns. diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 7d949ba5ae..b52b94d524 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -1,28 +1,37 @@ #!/usr/bin/env python3 -import os -import sys -import logging import argparse +import hashlib +import logging +import os import platform import subprocess -import hashlib +import sys os.environ["PYTHONUNBUFFERED"] = "y" ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \ - get_dev_uuid_var_path, FAIL, os_families, parse_os_release, \ - overwrite_symlink +from typing import TYPE_CHECKING, List + +from scripts.lib.node_cache import NODE_MODULES_CACHE_PATH, setup_node_modules from scripts.lib.setup_venv import ( - get_venv_dependencies, THUMBOR_VENV_DEPENDENCIES, + THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES, + get_venv_dependencies, +) +from scripts.lib.zulip_tools import ( + ENDC, + FAIL, + WARNING, + get_dev_uuid_var_path, + os_families, + overwrite_symlink, + parse_os_release, + run_as_root, ) -from scripts.lib.node_cache import setup_node_modules, NODE_MODULES_CACHE_PATH from tools.setup import setup_venvs -from typing import List, TYPE_CHECKING if TYPE_CHECKING: from typing import NoReturn diff --git a/tools/lib/provision_inner.py b/tools/lib/provision_inner.py index 3f09e9ed8e..6582e2d2d8 100755 --- a/tools/lib/provision_inner.py +++ b/tools/lib/provision_inner.py @@ -1,22 +1,26 @@ #!/usr/bin/env python3 -import os -import sys import argparse import glob +import os import shutil - +import sys from typing import List ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run, OKBLUE, ENDC, \ - get_dev_uuid_var_path, is_digest_obsolete, write_new_digest - -from version import PROVISION_VERSION from pygments import __version__ as pygments_version +from scripts.lib.zulip_tools import ( + ENDC, + OKBLUE, + get_dev_uuid_var_path, + is_digest_obsolete, + run, + write_new_digest, +) from tools.setup.generate_zulip_bots_static_files import generate_zulip_bots_static_files +from version import PROVISION_VERSION VENV_PATH = "/srv/zulip-py3-venv" UUID_VAR_PATH = get_dev_uuid_var_path() @@ -177,7 +181,7 @@ def clean_unused_caches() -> None: verbose=False, no_headings=True, ) - from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache + from scripts.lib import clean_emoji_cache, clean_node_cache, clean_venv_cache clean_venv_cache.main(args) clean_node_cache.main(args) clean_emoji_cache.main(args) @@ -227,12 +231,13 @@ def main(options: argparse.Namespace) -> int: import django django.setup() + from django.conf import settings + from zerver.lib.test_fixtures import ( DEV_DATABASE, TEST_DATABASE, destroy_leaked_test_databases, ) - from django.conf import settings if options.is_force or need_to_run_configure_rabbitmq( [settings.RABBITMQ_PASSWORD]): diff --git a/tools/lib/sanity_check.py b/tools/lib/sanity_check.py index dac8209fdb..6ebe777b90 100644 --- a/tools/lib/sanity_check.py +++ b/tools/lib/sanity_check.py @@ -2,6 +2,7 @@ import os import pwd import sys + def check_venv(filename: str) -> None: try: import django diff --git a/tools/lib/template_parser.py b/tools/lib/template_parser.py index 124f500dfd..b53bde07ba 100644 --- a/tools/lib/template_parser.py +++ b/tools/lib/template_parser.py @@ -1,5 +1,6 @@ from typing import Callable, List, Optional, Text + class FormattedException(Exception): pass diff --git a/tools/lib/test_script.py b/tools/lib/test_script.py index ca03ffae09..5c7780262b 100644 --- a/tools/lib/test_script.py +++ b/tools/lib/test_script.py @@ -1,12 +1,12 @@ -from typing import Optional, Tuple, Iterable, List - +import glob import os import subprocess import sys from distutils.version import LooseVersion -from version import PROVISION_VERSION +from typing import Iterable, List, Optional, Tuple + from scripts.lib.zulip_tools import get_dev_uuid_var_path -import glob +from version import PROVISION_VERSION ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tools/lib/test_server.py b/tools/lib/test_server.py index 5a1a66429e..5e47a8f343 100644 --- a/tools/lib/test_server.py +++ b/tools/lib/test_server.py @@ -2,13 +2,12 @@ import os import subprocess import sys import time - from contextlib import contextmanager - from typing import Iterator, Optional # Verify the Zulip venv is available. from tools.lib import sanity_check + sanity_check.check_venv(__file__) import django @@ -20,8 +19,9 @@ TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if TOOLS_DIR not in sys.path: sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -from zerver.lib.test_fixtures import update_test_databases_if_required from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.lib.test_fixtures import update_test_databases_if_required + def set_up_django(external_host: str) -> None: os.environ['EXTERNAL_HOST'] = external_host diff --git a/tools/lint b/tools/lint index e222cb6c08..2c379c83e5 100755 --- a/tools/lint +++ b/tools/lint @@ -1,16 +1,20 @@ #!/usr/bin/env python3 +import argparse import os import sys -import argparse # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from linter_lib.custom_check import python_rules, non_py_rules -from zulint.command import add_default_linter_arguments, LinterConfig import random +from zulint.command import LinterConfig, add_default_linter_arguments + +from linter_lib.custom_check import non_py_rules, python_rules + + def run() -> None: parser = argparse.ArgumentParser() parser.add_argument('--force', default=False, @@ -26,13 +30,10 @@ def run() -> None: root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) + from tools.lib.test_script import assert_provisioning_status_ok from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE - from tools.linter_lib.pyflakes import check_pyflakes from tools.linter_lib.pep8 import check_pep8 - - from tools.lib.test_script import ( - assert_provisioning_status_ok, - ) + from tools.linter_lib.pyflakes import check_pyflakes os.chdir(root_dir) diff --git a/tools/linter_lib/pep8.py b/tools/linter_lib/pep8.py index 58e330462e..93ad58de03 100644 --- a/tools/linter_lib/pep8.py +++ b/tools/linter_lib/pep8.py @@ -1,6 +1,7 @@ +from typing import List + from zulint.linters import run_pycodestyle -from typing import List def check_pep8(files: List[str]) -> bool: ignored_rules = [ diff --git a/tools/linter_lib/pyflakes.py b/tools/linter_lib/pyflakes.py index 33ecdd504f..ff630b3a38 100644 --- a/tools/linter_lib/pyflakes.py +++ b/tools/linter_lib/pyflakes.py @@ -1,5 +1,4 @@ import argparse - from typing import List from zulint.linters import run_pyflakes diff --git a/tools/pretty-print-html b/tools/pretty-print-html index d3fb88a4b3..c275c0440f 100755 --- a/tools/pretty-print-html +++ b/tools/pretty-print-html @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -from typing import List -from lib.pretty_print import pretty_print_html import sys +from typing import List + +from lib.pretty_print import pretty_print_html + def clean_html(filenames: List[str]) -> None: for fn in filenames: diff --git a/tools/renumber-migrations b/tools/renumber-migrations index 10998a8da7..600604ccbc 100755 --- a/tools/renumber-migrations +++ b/tools/renumber-migrations @@ -1,13 +1,12 @@ #!/usr/bin/env python3 - +import fileinput import glob import os -import sys -import fileinput import re - +import sys from typing import List + def validate_order(order: List[int], length: int) -> None: if len(order) != length: print("Please enter the sequence of all the conflicting files at once") diff --git a/tools/review b/tools/review index 24541b4f0b..cf2f5b7208 100755 --- a/tools/review +++ b/tools/review @@ -1,10 +1,10 @@ #!/usr/bin/env python3 - import shlex import subprocess import sys from typing import List + def exit(message: str) -> None: print('PROBLEM!') print(message) diff --git a/tools/run-dev.py b/tools/run-dev.py index 10615e874f..8ca71c13c5 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import pwd @@ -7,27 +6,22 @@ import signal import subprocess import sys import traceback - from urllib.parse import urlunparse # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from tornado import httpclient -from tornado import httputil -from tornado import gen -from tornado import web +from tornado import gen, httpclient, httputil, web from tornado.ioloop import IOLoop TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -from tools.lib.test_script import ( - assert_provisioning_status_ok, -) - from typing import Any, Callable, Generator, List, Optional +from tools.lib.test_script import assert_provisioning_status_ok + if 'posix' in os.name and os.geteuid() == 0: raise RuntimeError("run-dev.py should not be run as root.") @@ -103,7 +97,7 @@ os.environ['DJANGO_SETTINGS_MODULE'] = settings_module sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import CYAN, WARNING, FAIL, ENDC +from scripts.lib.zulip_tools import CYAN, ENDC, FAIL, WARNING proxy_port = base_port django_port = base_port + 1 diff --git a/tools/run-mypy b/tools/run-mypy index 939b667616..e72148591b 100755 --- a/tools/run-mypy +++ b/tools/run-mypy @@ -1,10 +1,8 @@ #!/usr/bin/env python3 - - -import os -import sys import argparse +import os import subprocess +import sys from typing import List from zulint import lister diff --git a/tools/setup/build_pygments_data b/tools/setup/build_pygments_data index 7c15888c35..8b55ea399c 100755 --- a/tools/setup/build_pygments_data +++ b/tools/setup/build_pygments_data @@ -1,9 +1,9 @@ #!/usr/bin/env python3 - -from pygments.lexers import get_all_lexers import json import os +from pygments.lexers import get_all_lexers + ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') DATA_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'lang.json') OUT_PATH = os.path.join(ZULIP_PATH, 'static', 'generated', 'pygments_data.json') diff --git a/tools/setup/emoji/build_emoji b/tools/setup/emoji/build_emoji index 23923323bf..1ff34775df 100755 --- a/tools/setup/emoji/build_emoji +++ b/tools/setup/emoji/build_emoji @@ -5,14 +5,20 @@ import os import shutil import sys -import ujson - from typing import Any, Dict, Iterator, List, Optional -from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \ - get_emoji_code, generate_name_to_codepoint_map, emoji_names_for_picker, \ - EMOTICON_CONVERSIONS, REMAPPED_EMOJIS +import ujson + from emoji_names import EMOJI_NAME_MAPS +from emoji_setup_utils import ( + EMOTICON_CONVERSIONS, + REMAPPED_EMOJIS, + emoji_names_for_picker, + generate_codepoint_to_name_map, + generate_emoji_catalog, + generate_name_to_codepoint_map, + get_emoji_code, +) ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') sys.path.append(ZULIP_PATH) diff --git a/tools/setup/emoji/emoji_setup_utils.py b/tools/setup/emoji/emoji_setup_utils.py index 64bdfbdb9e..3daff11a0b 100644 --- a/tools/setup/emoji/emoji_setup_utils.py +++ b/tools/setup/emoji/emoji_setup_utils.py @@ -1,8 +1,6 @@ # This file contains various helper functions used by `build_emoji` tool. # See docs/subsystems/emoji.md for details on how this system works. - from collections import defaultdict - from typing import Any, Dict, List # Emojisets that we currently support. diff --git a/tools/setup/emoji/export_emoji_names_to_csv b/tools/setup/emoji/export_emoji_names_to_csv index da1ab25932..c1e44a7e96 100755 --- a/tools/setup/emoji/export_emoji_names_to_csv +++ b/tools/setup/emoji/export_emoji_names_to_csv @@ -10,12 +10,12 @@ import argparse import csv import os import re +from typing import Any, Dict, List + import ujson from emoji_setup_utils import get_emoji_code -from typing import Any, Dict, List - TOOLS_DIR_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_PATH = os.path.dirname(TOOLS_DIR_PATH) # `emoji.json` file is same in all four emoji-datasource packages. diff --git a/tools/setup/emoji/generate_emoji_names_table b/tools/setup/emoji/generate_emoji_names_table index 71ecd8341e..90a5ba1e11 100755 --- a/tools/setup/emoji/generate_emoji_names_table +++ b/tools/setup/emoji/generate_emoji_names_table @@ -6,12 +6,12 @@ # sources' decisions about what names to provide to each unicode # codepoint. import os -import ujson - from typing import Any, Dict, List -from emoji_setup_utils import emoji_is_universal, get_emoji_code, EMOJISETS +import ujson + from emoji_names import EMOJI_NAME_MAPS +from emoji_setup_utils import EMOJISETS, emoji_is_universal, get_emoji_code TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_PATH = os.path.dirname(TOOLS_DIR) diff --git a/tools/setup/emoji/import_emoji_names_from_csv b/tools/setup/emoji/import_emoji_names_from_csv index f59977f359..cd188c47b2 100755 --- a/tools/setup/emoji/import_emoji_names_from_csv +++ b/tools/setup/emoji/import_emoji_names_from_csv @@ -13,7 +13,6 @@ import csv import os import re import textwrap - from typing import Any, Dict, List, Set EMOJI_DIR_PATH = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/setup/emoji/test-emoji-name-scripts b/tools/setup/emoji/test-emoji-name-scripts index 0dce85763f..1cd7b9b1d6 100755 --- a/tools/setup/emoji/test-emoji-name-scripts +++ b/tools/setup/emoji/test-emoji-name-scripts @@ -3,8 +3,8 @@ import argparse import difflib import filecmp import os -import subprocess import shutil +import subprocess import tempfile TOOLS_DIR = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) diff --git a/tools/setup/generate_integration_bots_avatars.py b/tools/setup/generate_integration_bots_avatars.py index b5b38a1e6d..9cd2f6b8eb 100755 --- a/tools/setup/generate_integration_bots_avatars.py +++ b/tools/setup/generate_integration_bots_avatars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import sys @@ -7,6 +6,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) from scripts.lib.setup_path import setup_path + setup_path() os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" @@ -20,9 +20,10 @@ import io import cairosvg from PIL import Image -from zerver.lib.upload import resize_avatar, DEFAULT_AVATAR_SIZE from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.storage import static_path +from zerver.lib.upload import DEFAULT_AVATAR_SIZE, resize_avatar + def create_square_image(png: bytes) -> bytes: img = Image.open(io.BytesIO(png)) diff --git a/tools/setup/generate_zulip_bots_static_files.py b/tools/setup/generate_zulip_bots_static_files.py index 83c22fbd7b..17ba1f3d1d 100755 --- a/tools/setup/generate_zulip_bots_static_files.py +++ b/tools/setup/generate_zulip_bots_static_files.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 - import glob import os -import sys import shutil +import sys from typing import List ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) from scripts.lib.setup_path import setup_path + setup_path() from zulip_bots.lib import get_bots_directory_path + def generate_zulip_bots_static_files() -> None: bots_dir = 'static/generated/bots' if os.path.isdir(bots_dir): diff --git a/tools/show-profile-results b/tools/show-profile-results index 0ac294cc4f..fb6aa1ce0f 100755 --- a/tools/show-profile-results +++ b/tools/show-profile-results @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import sys import pstats +import sys ''' This is a helper script to make it easy to show profile diff --git a/tools/test-api b/tools/test-api index 7685dd012e..ef61dbbc56 100755 --- a/tools/test-api +++ b/tools/test-api @@ -7,6 +7,7 @@ os.environ["RUNNING_OPENAPI_CURL_TEST"] = "1" # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -30,13 +31,13 @@ assert_provisioning_status_ok(options.force) with test_server_running(force=options.force, external_host='zulipdev.com:9981'): # Zerver imports should happen after `django.setup()` is run # by the test_server_running decorator. - from zerver.openapi.python_examples import test_the_api, test_invalid_api_key - from zerver.openapi.javascript_examples import test_js_bindings - from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success from zerver.lib.actions import do_create_user from zerver.lib.test_helpers import reset_emails_in_zulip_realm from zerver.lib.users import get_api_key - from zerver.models import get_user, get_realm + from zerver.models import get_realm, get_user + from zerver.openapi.javascript_examples import test_js_bindings + from zerver.openapi.python_examples import test_invalid_api_key, test_the_api + from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success print("Running API tests...") diff --git a/tools/test-backend b/tools/test-backend index 6f7830849d..8c8fc3b157 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -2,22 +2,23 @@ # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from typing import List, Iterator -import glob import argparse import contextlib -from unittest import mock +import glob import os import shlex -import sys import subprocess +import sys import tempfile -import ujson -import responses +from typing import Iterator, List +from unittest import mock import django +import responses +import ujson from django.conf import settings from django.test.utils import get_runner @@ -183,11 +184,10 @@ def main() -> None: os.environ.pop("http_proxy", "") os.environ.pop("https_proxy", "") - from zerver.lib.test_fixtures import update_test_databases_if_required, \ - remove_test_run_directories - - from tools.lib.test_script import ( - assert_provisioning_status_ok, + from tools.lib.test_script import assert_provisioning_status_ok + from zerver.lib.test_fixtures import ( + remove_test_run_directories, + update_test_databases_if_required, ) os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings' @@ -460,6 +460,7 @@ def main() -> None: if options.report_slow_tests: from zerver.lib.test_runner import report_slow_tests + # We do this even with failures, since slowness can be # an important clue as to why tests fail. report_slow_tests() diff --git a/tools/test-help-documentation b/tools/test-help-documentation index 7977c2b1c4..e0bb8aa4dc 100755 --- a/tools/test-help-documentation +++ b/tools/test-help-documentation @@ -2,14 +2,15 @@ import argparse import contextlib import os -import sys import subprocess +import sys from typing import Iterator ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) parser = argparse.ArgumentParser() diff --git a/tools/test-js-with-node b/tools/test-js-with-node index 0f9a16d55c..ca9b70c456 100755 --- a/tools/test-js-with-node +++ b/tools/test-js-with-node @@ -5,7 +5,7 @@ import os import pwd import subprocess import sys -from typing import Dict, Any +from typing import Any, Dict TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) @@ -13,12 +13,12 @@ ROOT_DIR = os.path.dirname(TOOLS_DIR) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) # Import this after we do the sanity_check so it doesn't crash. import ujson - -from zulint.printer import CYAN, GREEN, BOLDRED, ENDC +from zulint.printer import BOLDRED, CYAN, ENDC, GREEN INDEX_JS = 'frontend_tests/zjsunit/index.js' NODE_COVERAGE_PATH = 'var/node-coverage/coverage-final.json' diff --git a/tools/test-js-with-puppeteer b/tools/test-js-with-puppeteer index fc2219b67c..9938079c46 100755 --- a/tools/test-js-with-puppeteer +++ b/tools/test-js-with-puppeteer @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import argparse -import subprocess -import sys import os import shlex +import subprocess +import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -42,12 +42,18 @@ sys.path.insert(0, ZULIP_PATH) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) -from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files, prepare_puppeteer_run +from typing import Iterable + +from tools.lib.test_script import ( + assert_provisioning_status_ok, + find_js_test_files, + prepare_puppeteer_run, +) from tools.lib.test_server import test_server_running -from typing import Iterable def run_tests(files: Iterable[str], external_host: str) -> None: test_dir = os.path.join(ZULIP_PATH, 'frontend_tests/puppeteer_tests') diff --git a/tools/test-locked-requirements b/tools/test-locked-requirements index 98cc73ee41..2390264660 100755 --- a/tools/test-locked-requirements +++ b/tools/test-locked-requirements @@ -8,9 +8,9 @@ import shutil import subprocess import sys import tempfile -import ujson +from typing import List, Optional -from typing import Optional, List +import ujson TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) ZULIP_PATH = os.path.dirname(TOOLS_DIR) diff --git a/tools/test-queue-worker-reload b/tools/test-queue-worker-reload index 152edf202e..6f64dd7d8e 100755 --- a/tools/test-queue-worker-reload +++ b/tools/test-queue-worker-reload @@ -1,15 +1,14 @@ #!/usr/bin/env python3 - - import os -import sys -import time import signal import subprocess +import sys +import time import types # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) # TODO: Convert this to use scripts/lib/queue_workers.py diff --git a/tools/test-run-dev b/tools/test-run-dev index 9835c27021..f7d882373f 100755 --- a/tools/test-run-dev +++ b/tools/test-run-dev @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import signal import subprocess @@ -8,6 +7,7 @@ import time from typing import Tuple from lib import sanity_check + sanity_check.check_venv(__file__) TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/test-tools b/tools/test-tools index 14775fd74c..326f726ecf 100755 --- a/tools/test-tools +++ b/tools/test-tools @@ -1,6 +1,4 @@ #!/usr/bin/env python3 - - import argparse import os import sys @@ -8,6 +6,7 @@ import unittest # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) if __name__ == '__main__': diff --git a/tools/tests/test_capitalization_checker.py b/tools/tests/test_capitalization_checker.py index 37096d7391..2789f653c7 100644 --- a/tools/tests/test_capitalization_checker.py +++ b/tools/tests/test_capitalization_checker.py @@ -1,8 +1,9 @@ -from bs4 import BeautifulSoup from unittest import TestCase -from tools.lib.capitalization import check_capitalization, is_capitalized, \ - get_safe_text +from bs4 import BeautifulSoup + +from tools.lib.capitalization import check_capitalization, get_safe_text, is_capitalized + class GetSafeTextTestCase(TestCase): def test_get_safe_text(self) -> None: diff --git a/tools/tests/test_check_rabbitmq_queue.py b/tools/tests/test_check_rabbitmq_queue.py index d135097194..2eb531e1d9 100644 --- a/tools/tests/test_check_rabbitmq_queue.py +++ b/tools/tests/test_check_rabbitmq_queue.py @@ -1,14 +1,8 @@ -from unittest import mock -from unittest import TestCase - -from scripts.lib.check_rabbitmq_queue import ( - analyze_queue_stats, - OK, - WARNING, - CRITICAL, - UNKNOWN, -) import time +from unittest import TestCase, mock + +from scripts.lib.check_rabbitmq_queue import CRITICAL, OK, UNKNOWN, WARNING, analyze_queue_stats + class AnalyzeQueueStatsTests(TestCase): def test_no_stats_available(self) -> None: diff --git a/tools/tests/test_html_branches.py b/tools/tests/test_html_branches.py index 7db3609124..36d268a255 100644 --- a/tools/tests/test_html_branches.py +++ b/tools/tests/test_html_branches.py @@ -1,13 +1,12 @@ -import unittest import os +import unittest import tools.lib.template_parser - from tools.lib.html_branches import ( + build_id_dict, get_tag_info, html_branches, html_tag_tree, - build_id_dict, split_for_id_and_class, ) diff --git a/tools/tests/test_template_parser.py b/tools/tests/test_template_parser.py index 5928101e47..421a10a1a3 100644 --- a/tools/tests/test_template_parser.py +++ b/tools/tests/test_template_parser.py @@ -1,7 +1,6 @@ -from typing import Optional - import sys import unittest +from typing import Optional try: from tools.lib.template_parser import ( diff --git a/tools/tests/test_zulint_custom_rules.py b/tools/tests/test_zulint_custom_rules.py index ce11d9ea7d..f858e8de60 100644 --- a/tools/tests/test_zulint_custom_rules.py +++ b/tools/tests/test_zulint_custom_rules.py @@ -1,11 +1,11 @@ -from io import StringIO import os - -from unittest.mock import patch +from io import StringIO from unittest import TestCase +from unittest.mock import patch from zulint.custom_rules import RuleList -from linter_lib.custom_check import python_rules, non_py_rules + +from linter_lib.custom_check import non_py_rules, python_rules ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..')) CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`." diff --git a/tools/update-prod-static b/tools/update-prod-static index 1acc64781c..c42f45a3a0 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -1,10 +1,8 @@ #!/usr/bin/env python3 # Updates static files for production. - - -import os import argparse +import os import sys # We need settings so we can figure out where the prod-static directory is. @@ -15,11 +13,12 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' from django.conf import settings -from scripts.lib.node_cache import setup_node_modules -from scripts.lib.zulip_tools import run # check for the venv from lib import sanity_check +from scripts.lib.node_cache import setup_node_modules +from scripts.lib.zulip_tools import run + sanity_check.check_venv(__file__) parser = argparse.ArgumentParser() diff --git a/tools/webpack b/tools/webpack index 1187e0bb42..7ff053b14f 100755 --- a/tools/webpack +++ b/tools/webpack @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import argparse -import os import json +import os import subprocess from typing import NoReturn diff --git a/tools/zulip-export/zulip-export b/tools/zulip-export/zulip-export index 3072588c39..b65dec3eed 100755 --- a/tools/zulip-export/zulip-export +++ b/tools/zulip-export/zulip-export @@ -19,11 +19,11 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. - -import sys -import os import argparse import json +import os +import sys + usage = """Export all messages on a given stream to a JSON dump. zulip-export --user= --api-key= --stream= diff --git a/zerver/apps.py b/zerver/apps.py index d1b95a3827..4d6d355daf 100644 --- a/zerver/apps.py +++ b/zerver/apps.py @@ -6,6 +6,7 @@ from django.conf import settings from django.core.cache import cache from django.db.models.signals import post_migrate + def flush_cache(sender: AppConfig, **kwargs: Any) -> None: logging.info("Clearing memcached cache after migrations") cache.clear() diff --git a/zerver/context_processors.py b/zerver/context_processors.py index a0bda24c60..34bbc538dc 100644 --- a/zerver/context_processors.py +++ b/zerver/context_processors.py @@ -1,27 +1,31 @@ +from typing import Any, Dict, Optional from urllib.parse import urljoin -from typing import Any, Dict, Optional -from django.http import HttpRequest from django.conf import settings +from django.http import HttpRequest -from zerver.models import UserProfile, get_realm, Realm +from version import ( + LATEST_MAJOR_VERSION, + LATEST_RELEASE_ANNOUNCEMENT, + LATEST_RELEASE_VERSION, + ZULIP_VERSION, +) +from zerver.decorator import get_client_name +from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description +from zerver.lib.realm_icon import get_realm_icon_url +from zerver.lib.send_email import FromAddress +from zerver.lib.subdomains import get_subdomain +from zerver.models import Realm, UserProfile, get_realm from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + AppleAuthBackend, any_social_backend_enabled, + auth_enabled_helper, get_external_method_dicts, password_auth_enabled, require_email_format_usernames, - auth_enabled_helper, - AUTH_BACKEND_NAME_MAP, - AppleAuthBackend, ) -from zerver.decorator import get_client_name -from zerver.lib.send_email import FromAddress -from zerver.lib.subdomains import get_subdomain -from zerver.lib.realm_icon import get_realm_icon_url -from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description -from version import ZULIP_VERSION, LATEST_RELEASE_VERSION, LATEST_MAJOR_VERSION, \ - LATEST_RELEASE_ANNOUNCEMENT def common_context(user: UserProfile) -> Dict[str, Any]: """Common context used for things like outgoing emails that don't diff --git a/zerver/data_import/gitter.py b/zerver/data_import/gitter.py index 1dd10fa8c2..b168f52854 100644 --- a/zerver/data_import/gitter.py +++ b/zerver/data_import/gitter.py @@ -1,20 +1,31 @@ -import os -import dateutil.parser import logging +import os import subprocess -import ujson +from typing import Any, Dict, List, Set, Tuple +import dateutil.parser +import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Set, Tuple -from zerver.models import UserProfile, Recipient +from zerver.data_import.import_util import ( + ZerverFieldsT, + build_avatar, + build_defaultstream, + build_message, + build_realm, + build_recipient, + build_stream, + build_subscription, + build_usermessages, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + process_avatars, +) from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_avatar, build_subscription, build_recipient, build_usermessages, \ - build_defaultstream, process_avatars, build_realm, build_stream, \ - build_message, create_converted_data_files, make_subscriber_map +from zerver.models import Recipient, UserProfile # stubs GitterDataT = List[Dict[str, Any]] diff --git a/zerver/data_import/hipchat.py b/zerver/data_import/hipchat.py index 92cadb4f3b..386672bf10 100755 --- a/zerver/data_import/hipchat.py +++ b/zerver/data_import/hipchat.py @@ -1,37 +1,29 @@ import base64 -import dateutil import glob -import hypchat import logging import os import re import shutil import subprocess -import ujson - from typing import Any, Callable, Dict, List, Optional, Set +import dateutil +import hypchat +import ujson from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.utils import ( - process_list_in_batches, -) - -from zerver.models import ( - RealmEmoji, - Recipient, - UserProfile, -) - +from zerver.data_import.hipchat_attachment import AttachmentHandler +from zerver.data_import.hipchat_user import UserHandler from zerver.data_import.import_util import ( + SubscriberHandler, build_message, + build_personal_subscriptions, + build_public_stream_subscriptions, build_realm, build_realm_emoji, build_recipients, build_stream, - build_personal_subscriptions, - build_public_stream_subscriptions, build_stream_subscriptions, build_user_profile, build_zerver_realm, @@ -39,12 +31,10 @@ from zerver.data_import.import_util import ( make_subscriber_map, make_user_messages, write_avatar_png, - SubscriberHandler, ) - -from zerver.data_import.hipchat_attachment import AttachmentHandler -from zerver.data_import.hipchat_user import UserHandler from zerver.data_import.sequencer import NEXT_ID, IdMapper +from zerver.lib.utils import process_list_in_batches +from zerver.models import RealmEmoji, Recipient, UserProfile # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/data_import/hipchat_attachment.py b/zerver/data_import/hipchat_attachment.py index de676751dd..596d3e588f 100644 --- a/zerver/data_import/hipchat_attachment.py +++ b/zerver/data_import/hipchat_attachment.py @@ -1,14 +1,11 @@ import logging -import shutil import os - -from zerver.data_import.import_util import ( - build_attachment, - create_converted_data_files, -) - +import shutil from typing import Any, Dict, List, Optional +from zerver.data_import.import_util import build_attachment, create_converted_data_files + + class AttachmentHandler: def __init__(self) -> None: self.info_dict: Dict[str, Dict[str, Any]] = dict() diff --git a/zerver/data_import/hipchat_user.py b/zerver/data_import/hipchat_user.py index 10b3950dee..ef03ece81b 100644 --- a/zerver/data_import/hipchat_user.py +++ b/zerver/data_import/hipchat_user.py @@ -2,12 +2,9 @@ from typing import Any, Dict, List from django.utils.timezone import now as timezone_now -from zerver.data_import.import_util import ( - build_user_profile, -) -from zerver.models import ( - UserProfile, -) +from zerver.data_import.import_util import build_user_profile +from zerver.models import UserProfile + class UserHandler: ''' diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py index f7580e454e..31f235df09 100644 --- a/zerver/data_import/import_util.py +++ b/zerver/data_import/import_util.py @@ -1,20 +1,29 @@ -import random -import requests -import shutil import logging import os +import random +import shutil import traceback -import ujson +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar -from typing import List, Dict, Any, Optional, Set, Callable, Iterable, Tuple, TypeVar +import requests +import ujson from django.forms.models import model_to_dict -from zerver.models import Realm, RealmEmoji, Subscription, Recipient, \ - Attachment, Stream, Message, UserProfile, Huddle from zerver.data_import.sequencer import NEXT_ID from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS as stream_colors from zerver.lib.avatar_hash import user_avatar_path_from_ids from zerver.lib.parallel import run_parallel +from zerver.models import ( + Attachment, + Huddle, + Message, + Realm, + RealmEmoji, + Recipient, + Stream, + Subscription, + UserProfile, +) # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/data_import/mattermost.py b/zerver/data_import/mattermost.py index 1bb4dfa3be..ba1823a4fc 100644 --- a/zerver/data_import/mattermost.py +++ b/zerver/data_import/mattermost.py @@ -2,33 +2,42 @@ spec: https://docs.mattermost.com/administration/bulk-export.html """ -import os import logging -import subprocess -import ujson +import os import re import shutil - +import subprocess from typing import Any, Callable, Dict, List, Set +import ujson from django.conf import settings -from django.utils.timezone import now as timezone_now from django.forms.models import model_to_dict +from django.utils.timezone import now as timezone_now -from zerver.models import Recipient, RealmEmoji, Reaction, UserProfile -from zerver.lib.utils import ( - process_list_in_batches, +from zerver.data_import.import_util import ( + SubscriberHandler, + ZerverFieldsT, + build_huddle, + build_huddle_subscriptions, + build_message, + build_personal_subscriptions, + build_realm, + build_realm_emoji, + build_recipients, + build_stream, + build_stream_subscriptions, + build_user_profile, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + make_user_messages, ) -from zerver.lib.emoji import name_to_codepoint -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_stream, build_realm, build_message, create_converted_data_files, \ - make_subscriber_map, build_recipients, build_user_profile, \ - build_stream_subscriptions, build_huddle_subscriptions, \ - build_personal_subscriptions, SubscriberHandler, \ - build_realm_emoji, make_user_messages, build_huddle - from zerver.data_import.mattermost_user import UserHandler from zerver.data_import.sequencer import NEXT_ID, IdMapper +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.utils import process_list_in_batches +from zerver.models import Reaction, RealmEmoji, Recipient, UserProfile + def make_realm(realm_id: int, team: Dict[str, Any]) -> ZerverFieldsT: # set correct realm details diff --git a/zerver/data_import/mattermost_user.py b/zerver/data_import/mattermost_user.py index c8585584a1..e6d629314e 100644 --- a/zerver/data_import/mattermost_user.py +++ b/zerver/data_import/mattermost_user.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List + class UserHandler: ''' Our UserHandler class is a glorified wrapper diff --git a/zerver/data_import/slack.py b/zerver/data_import/slack.py index 71b487dde7..798345cd8b 100755 --- a/zerver/data_import/slack.py +++ b/zerver/data_import/slack.py @@ -1,33 +1,55 @@ +import logging import os -import ujson +import random import shutil import subprocess -import logging -import random -import requests - from collections import defaultdict - -from django.conf import settings -from django.utils.timezone import now as timezone_now -from django.forms.models import model_to_dict -from typing import Any, Dict, List, Optional, Tuple, Set, Iterator -from zerver.models import Reaction, RealmEmoji, UserProfile, Recipient, \ - CustomProfileField, CustomProfileFieldValue, Realm -from zerver.data_import.slack_message_conversion import convert_to_zulip_markdown, \ - get_user_full_name -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_avatar, build_subscription, build_recipient, build_usermessages, \ - build_defaultstream, build_attachment, process_avatars, process_uploads, \ - process_emojis, build_realm, build_stream, build_huddle, build_message, \ - create_converted_data_files, make_subscriber_map -from zerver.data_import.sequencer import NEXT_ID -from zerver.lib.upload import random_name, sanitize_name -from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE -from zerver.lib.emoji import name_to_codepoint -from zerver.lib.upload import resize_logo +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple from urllib.parse import urlencode +import requests +import ujson +from django.conf import settings +from django.forms.models import model_to_dict +from django.utils.timezone import now as timezone_now + +from zerver.data_import.import_util import ( + ZerverFieldsT, + build_attachment, + build_avatar, + build_defaultstream, + build_huddle, + build_message, + build_realm, + build_recipient, + build_stream, + build_subscription, + build_usermessages, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + process_avatars, + process_emojis, + process_uploads, +) +from zerver.data_import.sequencer import NEXT_ID +from zerver.data_import.slack_message_conversion import ( + convert_to_zulip_markdown, + get_user_full_name, +) +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE +from zerver.lib.upload import random_name, resize_logo, sanitize_name +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + Reaction, + Realm, + RealmEmoji, + Recipient, + UserProfile, +) + SlackToZulipUserIDT = Dict[str, int] AddedChannelsT = Dict[str, Tuple[str, int]] AddedMPIMsT = Dict[str, Tuple[str, int]] diff --git a/zerver/data_import/slack_message_conversion.py b/zerver/data_import/slack_message_conversion.py index 0905541568..bfd8a54eb2 100644 --- a/zerver/data_import/slack_message_conversion.py +++ b/zerver/data_import/slack_message_conversion.py @@ -1,5 +1,5 @@ import re -from typing import Any, Dict, Tuple, List, Optional +from typing import Any, Dict, List, Optional, Tuple # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/decorator.py b/zerver/decorator.py index f87eacdccc..ae4a62d231 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -1,51 +1,60 @@ -import django_otp -from two_factor.utils import default_device -from django_otp import user_has_device - -from django.contrib.auth.decorators import user_passes_test as django_user_passes_test -from django.contrib.auth.models import AnonymousUser -from django.utils.translation import ugettext as _ -from django.http import HttpResponseRedirect, HttpResponse -from django.contrib.auth import REDIRECT_FIELD_NAME, login as django_login -from django.views.decorators.csrf import csrf_exempt -from django.http import QueryDict, HttpResponseNotAllowed, HttpRequest -from django.http.multipartparser import MultiPartParser -from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key -from zerver.lib.response import json_error, json_unauthorized, json_success -from django.shortcuts import resolve_url -from django.utils.decorators import available_attrs -from django.utils.timezone import now as timezone_now -from django.conf import settings -from django.template.response import SimpleTemplateResponse - -from zerver.lib.exceptions import UnexpectedWebhookEventType -from zerver.lib.queue import queue_json_publish -from zerver.lib.subdomains import get_subdomain, user_matches_subdomain -from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime -from zerver.lib.utils import statsd, has_api_key_format -from zerver.lib.exceptions import JsonableError, ErrorCode, \ - InvalidJSONError, InvalidAPIKeyError, InvalidAPIKeyFormatError, \ - OrganizationAdministratorRequired, OrganizationOwnerRequired -from zerver.lib.types import ViewFuncT - -from zerver.lib.rate_limiter import RateLimitedUser -from zerver.lib.request import REQ, has_request_variables - -from functools import wraps import base64 import datetime -import ujson import logging -from io import BytesIO import urllib +from functools import wraps +from io import BytesIO +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, Union -from typing import Union, Any, Callable, Dict, Optional, TypeVar, Tuple +import django_otp +import ujson +from django.conf import settings +from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.auth import login as django_login +from django.contrib.auth.decorators import user_passes_test as django_user_passes_test +from django.contrib.auth.models import AnonymousUser +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseNotAllowed, + HttpResponseRedirect, + QueryDict, +) +from django.http.multipartparser import MultiPartParser +from django.shortcuts import resolve_url +from django.template.response import SimpleTemplateResponse +from django.utils.decorators import available_attrs +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import csrf_exempt +from django_otp import user_has_device +from two_factor.utils import default_device + +from zerver.lib.exceptions import ( + ErrorCode, + InvalidAPIKeyError, + InvalidAPIKeyFormatError, + InvalidJSONError, + JsonableError, + OrganizationAdministratorRequired, + OrganizationOwnerRequired, + UnexpectedWebhookEventType, +) from zerver.lib.logging_util import log_to_file +from zerver.lib.queue import queue_json_publish +from zerver.lib.rate_limiter import RateLimitedUser +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success, json_unauthorized +from zerver.lib.subdomains import get_subdomain, user_matches_subdomain +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.lib.types import ViewFuncT +from zerver.lib.utils import has_api_key_format, statsd +from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key # This is a hack to ensure that RemoteZulipServer always exists even # if Zilencer isn't enabled. if settings.ZILENCER_ENABLED: - from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer + from zilencer.models import RemoteZulipServer, get_remote_server_by_uuid else: # nocoverage # Hack here basically to make impossible code paths compile from unittest.mock import Mock get_remote_server_by_uuid = Mock() @@ -130,6 +139,7 @@ def require_billing_access(func: ViewFuncT) -> ViewFuncT: from zerver.lib.user_agent import parse_user_agent + def get_client_name(request: HttpRequest) -> str: # If the API request specified a client in the request content, # that has priority. Otherwise, extract the client from the diff --git a/zerver/filters.py b/zerver/filters.py index f74eae3aae..8d3e2fbe47 100644 --- a/zerver/filters.py +++ b/zerver/filters.py @@ -4,6 +4,7 @@ from typing import Any, Dict from django.http import HttpRequest from django.views.debug import SafeExceptionReporterFilter + class ZulipExceptionReporterFilter(SafeExceptionReporterFilter): def get_post_parameters(self, request: HttpRequest) -> Dict[str, Any]: filtered_post = SafeExceptionReporterFilter.get_post_parameters(self, request).copy() diff --git a/zerver/forms.py b/zerver/forms.py index 6b70e7f67c..2f1cc9ca1d 100644 --- a/zerver/forms.py +++ b/zerver/forms.py @@ -1,42 +1,44 @@ +import logging +import re +from typing import Any, Dict, List, Optional, Tuple + +import DNS from django import forms from django.conf import settings from django.contrib.auth import authenticate, password_validation -from django.contrib.auth.forms import SetPasswordForm, AuthenticationForm, \ - PasswordResetForm +from django.contrib.auth.forms import AuthenticationForm, PasswordResetForm, SetPasswordForm +from django.contrib.auth.tokens import PasswordResetTokenGenerator, default_token_generator from django.core.exceptions import ValidationError -from django.urls import reverse from django.core.validators import validate_email -from django.utils.translation import ugettext as _ -from django.contrib.auth.tokens import default_token_generator -from django.utils.http import urlsafe_base64_encode -from django.utils.encoding import force_bytes -from django.contrib.auth.tokens import PasswordResetTokenGenerator from django.http import HttpRequest +from django.urls import reverse +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode +from django.utils.translation import ugettext as _ from jinja2 import Markup as mark_safe - -from zerver.lib.actions import do_change_password, email_not_system_bot -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm -from zerver.lib.name_restrictions import is_reserved_subdomain, is_disposable_domain -from zerver.lib.rate_limiter import RateLimited, RateLimitedObject -from zerver.lib.request import JsonableError -from zerver.lib.send_email import send_email, FromAddress -from zerver.lib.subdomains import get_subdomain, is_root_domain_available -from zerver.lib.users import check_full_name -from zerver.models import Realm, get_user_by_delivery_email, UserProfile, get_realm, \ - email_to_domain, \ - DisposableEmailError, DomainNotAllowedForRealmError, \ - EmailContainsPlusError -from zproject.backends import email_auth_enabled, email_belongs_to_ldap, check_password_strength - -import logging -import re -import DNS - -from typing import Any, List, Optional, Dict, Tuple from two_factor.forms import AuthenticationTokenForm as TwoFactorAuthenticationTokenForm from two_factor.utils import totp_digits +from zerver.lib.actions import do_change_password, email_not_system_bot +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm +from zerver.lib.name_restrictions import is_disposable_domain, is_reserved_subdomain +from zerver.lib.rate_limiter import RateLimited, RateLimitedObject +from zerver.lib.request import JsonableError +from zerver.lib.send_email import FromAddress, send_email +from zerver.lib.subdomains import get_subdomain, is_root_domain_available +from zerver.lib.users import check_full_name +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + Realm, + UserProfile, + email_to_domain, + get_realm, + get_user_by_delivery_email, +) +from zproject.backends import check_password_strength, email_auth_enabled, email_belongs_to_ldap + MIT_VALIDATION_ERROR = 'That user does not exist at MIT or is a ' + \ 'mailing list. ' + \ 'If you want to sign up an alias for Zulip, ' + \ diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 42902cf44b..6bce33df3a 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -1,63 +1,116 @@ from typing import ( - AbstractSet, Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, - Optional, Sequence, Set, Tuple, Union, cast, + AbstractSet, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, ) -from typing_extensions import TypedDict import django.db.utils -from django.db.models import Count, Exists, OuterRef -from django.contrib.contenttypes.models import ContentType -from django.utils.html import escape -from django.utils.translation import ugettext as _ from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError from django.core.files import File +from django.db import IntegrityError, connection, transaction +from django.db.models import Count, Exists, F, Max, OuterRef, Q, Sum +from django.db.models.query import QuerySet +from django.utils.html import escape +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ from psycopg2.extras import execute_values from psycopg2.sql import SQL -from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat, \ - RealmCount +from typing_extensions import TypedDict -from zerver.lib.bugdown import ( - version as bugdown_version, +from analytics.lib.counts import COUNT_STATS, RealmCount, do_increment_logging_stat +from analytics.models import StreamCount +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + confirmation_url, + create_confirmation_link, + generate_key, ) +from zerver.decorator import statsd_increment +from zerver.lib import bugdown from zerver.lib.addressee import Addressee -from zerver.lib.bot_config import ( - ConfigError, - get_bot_config, - get_bot_configs, - set_bot_config, +from zerver.lib.alert_words import ( + add_user_alert_words, + get_alert_word_automaton, + remove_user_alert_words, ) +from zerver.lib.avatar import avatar_url, avatar_url_from_dict +from zerver.lib.bot_config import ConfigError, get_bot_config, get_bot_configs, set_bot_config +from zerver.lib.bugdown import version as bugdown_version +from zerver.lib.bulk_create import bulk_create_users from zerver.lib.cache import ( bot_dict_fields, - display_recipient_cache_key, + cache_delete, + cache_delete_many, + cache_set, + cache_set_many, + cache_with_key, delete_user_profile_caches, + display_recipient_cache_key, flush_user_profile, to_dict_cache_key_id, user_profile_by_api_key_cache_key, + user_profile_by_email_cache_key, ) from zerver.lib.context_managers import lockfile +from zerver.lib.create_user import create_user, get_display_email_address from zerver.lib.email_mirror_helpers import encode_email_address, encode_email_address_helper +from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.email_validation import ( + email_reserved_for_system_bots_error, + get_existing_user_errors, + get_realm_email_validator, + validate_email_is_valid, +) from zerver.lib.emoji import emoji_name_to_emoji_code, get_emoji_file_name -from zerver.lib.exceptions import StreamDoesNotExistError, \ - StreamWithIDDoesNotExistError +from zerver.lib.exceptions import ( + BugdownRenderingException, + ErrorCode, + JsonableError, + StreamDoesNotExistError, + StreamWithIDDoesNotExistError, +) from zerver.lib.export import get_realm_exports_serialized from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots +from zerver.lib.i18n import get_language_name from zerver.lib.message import ( - access_message, MessageDict, + access_message, render_markdown, - update_first_visible_message_id, truncate_body, truncate_topic, + update_first_visible_message_id, ) from zerver.lib.pysa import mark_sanitized +from zerver.lib.queue import queue_json_publish from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_data from zerver.lib.retention import move_messages_to_archive -from zerver.lib.send_email import send_email, FromAddress, send_email_to_admins, \ - clear_scheduled_emails, clear_scheduled_invitation_emails +from zerver.lib.send_email import ( + FromAddress, + clear_scheduled_emails, + clear_scheduled_invitation_emails, + send_email, + send_email_to_admins, +) from zerver.lib.server_initialization import create_internal_realm, server_initialized +from zerver.lib.sessions import delete_user_sessions from zerver.lib.storage import static_path +from zerver.lib.stream_recipient import StreamRecipientMap from zerver.lib.stream_subscription import ( get_active_subscriptions_for_stream_id, get_active_subscriptions_for_stream_ids, @@ -68,21 +121,37 @@ from zerver.lib.stream_subscription import ( num_subscribers_for_stream_id, ) from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.streams import ( + access_stream_for_send_message, + check_stream_name, + create_stream_if_needed, + get_default_value_for_history_public_to_subscribers, + render_stream_description, + send_stream_creation_event, + subscribed_to_stream, +) +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.topic import ( + LEGACY_PREV_TOPIC, + ORIG_TOPIC, + TOPIC_LINKS, + TOPIC_NAME, filter_by_exact_message_topic, filter_by_topic_name_via_message, save_message_for_edit_use_case, update_messages_for_topic_edit, - ORIG_TOPIC, - LEGACY_PREV_TOPIC, - TOPIC_LINKS, - TOPIC_NAME, ) -from zerver.lib.topic_mutes import ( - get_topic_mutes, - add_topic_mute, - remove_topic_mute, +from zerver.lib.topic_mutes import add_topic_mute, get_topic_mutes, remove_topic_mute +from zerver.lib.types import ProfileFieldData +from zerver.lib.upload import ( + claim_attachment, + delete_avatar_image, + delete_export_tarball, + delete_message_image, + upload_emoji_image, ) +from zerver.lib.user_groups import access_user_group_by_id, create_user_group +from zerver.lib.user_status import update_user_status from zerver.lib.users import ( check_bot_name_available, check_full_name, @@ -90,93 +159,84 @@ from zerver.lib.users import ( get_api_key, user_profile_to_user_row, ) -from zerver.lib.user_status import ( - update_user_status, -) -from zerver.lib.user_groups import create_user_group, access_user_group_by_id - -from zerver.models import Realm, RealmEmoji, Stream, UserProfile, UserActivity, \ - RealmDomain, Service, SubMessage, \ - Subscription, Recipient, Message, Attachment, UserMessage, RealmAuditLog, \ - UserHotspot, MultiuseInvite, ScheduledMessage, UserStatus, \ - Client, DefaultStream, DefaultStreamGroup, UserPresence, \ - ScheduledEmail, \ - MAX_MESSAGE_LENGTH, get_client, get_stream, \ - get_user_profile_by_id, PreregistrationUser, \ - email_to_username, \ - get_user_by_delivery_email, get_stream_cache_key, active_non_guest_user_ids, \ - UserActivityInterval, active_user_ids, get_active_streams, \ - realm_filters_for_realm, RealmFilter, stream_name_in_use, \ - get_old_unclaimed_attachments, is_cross_realm_bot_email, \ - Reaction, EmailChangeStatus, CustomProfileField, \ - custom_profile_fields_for_realm, get_huddle_user_ids, \ - CustomProfileFieldValue, validate_attachment_request, get_system_bot, \ - query_for_ids, get_huddle_recipient, \ - UserGroup, UserGroupMembership, get_default_stream_groups, \ - get_bot_services, get_bot_dicts_in_realm, \ - get_user_by_id_in_realm_including_cross_realm, \ - get_stream_by_id_in_realm - -from zerver.lib.alert_words import get_alert_word_automaton -from zerver.lib.avatar import avatar_url, avatar_url_from_dict -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_is_valid, get_existing_user_errors, \ - email_reserved_for_system_bots_error -from zerver.lib.stream_recipient import StreamRecipientMap +from zerver.lib.utils import generate_api_key, log_statsd_event, statsd from zerver.lib.validator import check_widget_content from zerver.lib.widget import do_widget_post_save_actions - -from django.db import transaction, IntegrityError, connection -from django.db.models import F, Q, Max, Sum -from django.db.models.query import QuerySet -from django.core.exceptions import ValidationError -from django.utils.timezone import now as timezone_now - -from confirmation.models import Confirmation, create_confirmation_link, generate_key, \ - confirmation_url -from confirmation import settings as confirmation_settings - -from zerver.lib.bulk_create import bulk_create_users -from zerver.lib.timestamp import timestamp_to_datetime, datetime_to_timestamp -from zerver.lib.queue import queue_json_publish -from zerver.lib.utils import generate_api_key -from zerver.lib.create_user import create_user, get_display_email_address -from zerver.lib import bugdown -from zerver.lib.cache import cache_with_key, cache_set, \ - user_profile_by_email_cache_key, \ - cache_set_many, cache_delete, cache_delete_many -from zerver.decorator import statsd_increment -from zerver.lib.utils import log_statsd_event, statsd -from zerver.lib.i18n import get_language_name -from zerver.lib.alert_words import add_user_alert_words, \ - remove_user_alert_words -from zerver.lib.email_notifications import enqueue_welcome_emails -from zerver.lib.exceptions import JsonableError, ErrorCode, BugdownRenderingException -from zerver.lib.sessions import delete_user_sessions -from zerver.lib.upload import claim_attachment, delete_message_image, \ - upload_emoji_image, delete_avatar_image, \ - delete_export_tarball +from zerver.models import ( + MAX_MESSAGE_LENGTH, + Attachment, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + DefaultStreamGroup, + EmailChangeStatus, + Message, + MultiuseInvite, + PreregistrationUser, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + ScheduledEmail, + ScheduledMessage, + Service, + Stream, + SubMessage, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + UserStatus, + active_non_guest_user_ids, + active_user_ids, + custom_profile_fields_for_realm, + email_to_username, + get_active_streams, + get_bot_dicts_in_realm, + get_bot_services, + get_client, + get_default_stream_groups, + get_huddle_recipient, + get_huddle_user_ids, + get_old_unclaimed_attachments, + get_stream, + get_stream_by_id_in_realm, + get_stream_cache_key, + get_system_bot, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, + get_user_profile_by_id, + is_cross_realm_bot_email, + query_for_ids, + realm_filters_for_realm, + stream_name_in_use, + validate_attachment_request, +) from zerver.tornado.event_queue import send_event -from zerver.lib.types import ProfileFieldData -from zerver.lib.streams import access_stream_for_send_message, subscribed_to_stream, check_stream_name, \ - create_stream_if_needed, get_default_value_for_history_public_to_subscribers, \ - render_stream_description, send_stream_creation_event - -from analytics.models import StreamCount if settings.BILLING_ENABLED: from corporate.lib.stripe import update_license_ledger_if_needed, downgrade_now -import ujson -import time import datetime +import itertools +import logging import os import platform -import logging -import itertools +import time from collections import defaultdict from operator import itemgetter +import ujson + # This will be used to type annotate parameters in a function if the function # works on both str and unicode in python 2 but in python 3 it only works on str. SizedTextIterable = Union[Sequence[str], AbstractSet[str]] diff --git a/zerver/lib/addressee.py b/zerver/lib/addressee.py index cb3dab8789..ec1c6fd7a9 100644 --- a/zerver/lib/addressee.py +++ b/zerver/lib/addressee.py @@ -1,15 +1,17 @@ from typing import Iterable, List, Optional, Sequence, Union, cast from django.utils.translation import ugettext as _ + from zerver.lib.exceptions import JsonableError from zerver.models import ( Realm, - UserProfile, - get_user_including_cross_realm, - get_user_by_id_in_realm_including_cross_realm, Stream, + UserProfile, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, ) + def get_user_profiles(emails: Iterable[str], realm: Realm) -> List[UserProfile]: user_profiles: List[UserProfile] = [] for email in emails: diff --git a/zerver/lib/alert_words.py b/zerver/lib/alert_words.py index 51c4c49794..e9658d9b97 100644 --- a/zerver/lib/alert_words.py +++ b/zerver/lib/alert_words.py @@ -1,10 +1,15 @@ +from typing import Dict, Iterable, List + +import ahocorasick from django.db import transaction -from zerver.models import UserProfile, Realm, AlertWord, flush_realm_alert_words -from zerver.lib.cache import cache_with_key, realm_alert_words_cache_key, \ - realm_alert_words_automaton_cache_key -import ahocorasick -from typing import Dict, Iterable, List +from zerver.lib.cache import ( + cache_with_key, + realm_alert_words_automaton_cache_key, + realm_alert_words_cache_key, +) +from zerver.models import AlertWord, Realm, UserProfile, flush_realm_alert_words + @cache_with_key(realm_alert_words_cache_key, timeout=3600*24) def alert_words_in_realm(realm: Realm) -> Dict[int, List[str]]: diff --git a/zerver/lib/attachments.py b/zerver/lib/attachments.py index ededc38cbb..477770f658 100644 --- a/zerver/lib/attachments.py +++ b/zerver/lib/attachments.py @@ -1,10 +1,12 @@ -from django.utils.translation import ugettext as _ from typing import Any, Dict, List +from django.utils.translation import ugettext as _ + from zerver.lib.request import JsonableError from zerver.lib.upload import delete_message_image from zerver.models import Attachment, UserProfile + def user_attachments(user_profile: UserProfile) -> List[Dict[str, Any]]: attachments = Attachment.objects.filter(owner=user_profile).prefetch_related('messages') return [a.to_dict() for a in attachments] diff --git a/zerver/lib/avatar.py b/zerver/lib/avatar.py index 469b8e27f8..c6bddfeb45 100644 --- a/zerver/lib/avatar.py +++ b/zerver/lib/avatar.py @@ -1,11 +1,16 @@ -from django.conf import settings - +import urllib from typing import Any, Dict, Optional -from zerver.lib.avatar_hash import gravatar_hash, user_avatar_path_from_ids, user_avatar_content_hash -from zerver.lib.upload import upload_backend, MEDIUM_AVATAR_SIZE +from django.conf import settings + +from zerver.lib.avatar_hash import ( + gravatar_hash, + user_avatar_content_hash, + user_avatar_path_from_ids, +) +from zerver.lib.upload import MEDIUM_AVATAR_SIZE, upload_backend from zerver.models import UserProfile -import urllib + def avatar_url(user_profile: UserProfile, medium: bool=False, client_gravatar: bool=False) -> Optional[str]: diff --git a/zerver/lib/avatar_hash.py b/zerver/lib/avatar_hash.py index bbd6c10c17..492a575cc0 100644 --- a/zerver/lib/avatar_hash.py +++ b/zerver/lib/avatar_hash.py @@ -1,10 +1,10 @@ +import hashlib + from django.conf import settings from zerver.lib.utils import make_safe_digest - from zerver.models import UserProfile -import hashlib def gravatar_hash(email: str) -> str: """Compute the Gravatar hash for an email address.""" diff --git a/zerver/lib/bot_config.py b/zerver/lib/bot_config.py index caf6cd2059..40e74559cc 100644 --- a/zerver/lib/bot_config.py +++ b/zerver/lib/bot_config.py @@ -1,17 +1,16 @@ -from django.conf import settings -from django.db.models import Sum -from django.db.models.query import F -from django.db.models.functions import Length -from zerver.models import BotConfigData, UserProfile - -from typing import List, Dict, Optional - -from collections import defaultdict - -import os - import configparser import importlib +import os +from collections import defaultdict +from typing import Dict, List, Optional + +from django.conf import settings +from django.db.models import Sum +from django.db.models.functions import Length +from django.db.models.query import F + +from zerver.models import BotConfigData, UserProfile + class ConfigError(Exception): pass diff --git a/zerver/lib/bot_lib.py b/zerver/lib/bot_lib.py index 492dccd9cf..b25b96f521 100644 --- a/zerver/lib/bot_lib.py +++ b/zerver/lib/bot_lib.py @@ -1,23 +1,31 @@ +import importlib import json import os -import importlib -from zerver.lib.actions import internal_send_private_message, \ - internal_send_stream_message_by_name, internal_send_huddle_message -from zerver.models import UserProfile, get_active_user -from zerver.lib.bot_storage import get_bot_storage, set_bot_storage, \ - is_key_in_bot_storage, remove_bot_storage -from zerver.lib.bot_config import get_bot_config, ConfigError -from zerver.lib.integrations import EMBEDDED_BOTS -from zerver.lib.topic import get_topic_from_message_info +from typing import Any, Dict from django.utils.translation import ugettext as _ -from typing import Any, Dict +from zerver.lib.actions import ( + internal_send_huddle_message, + internal_send_private_message, + internal_send_stream_message_by_name, +) +from zerver.lib.bot_config import ConfigError, get_bot_config +from zerver.lib.bot_storage import ( + get_bot_storage, + is_key_in_bot_storage, + remove_bot_storage, + set_bot_storage, +) +from zerver.lib.integrations import EMBEDDED_BOTS +from zerver.lib.topic import get_topic_from_message_info +from zerver.models import UserProfile, get_active_user our_dir = os.path.dirname(os.path.abspath(__file__)) from zulip_bots.lib import RateLimit + def get_bot_handler(service_name: str) -> Any: # Check that this service is present in EMBEDDED_BOTS, add exception handling. diff --git a/zerver/lib/bot_storage.py b/zerver/lib/bot_storage.py index ba3a9b1af3..1b43644f0b 100644 --- a/zerver/lib/bot_storage.py +++ b/zerver/lib/bot_storage.py @@ -1,10 +1,12 @@ +from typing import List, Optional, Tuple + from django.conf import settings from django.db.models import Sum -from django.db.models.query import F from django.db.models.functions import Length +from django.db.models.query import F + from zerver.models import BotStorageData, UserProfile -from typing import Optional, List, Tuple class StateError(Exception): pass diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index bfab78823a..a0bcdc85cc 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -1,65 +1,76 @@ # Zulip's main markdown implementation. See docs/subsystems/markdown.md for # detailed documentation on our markdown syntax. -from typing import (Any, Callable, Dict, Generic, Iterable, List, - Optional, Set, Tuple, TypeVar, Union) -from typing.re import Match, Pattern -from typing_extensions import TypedDict - -import markdown +import functools +import html import logging +import os +import re +import time import traceback import urllib import urllib.parse -import re -import os -import html -import time -import functools +from collections import defaultdict, deque from dataclasses import dataclass +from datetime import datetime from io import StringIO +from typing import ( + Any, + Callable, + Dict, + Generic, + Iterable, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from typing.re import Match, Pattern +from xml.etree import ElementTree as etree +from xml.etree.ElementTree import Element, SubElement + +import ahocorasick import dateutil.parser import dateutil.tz -from datetime import datetime -import xml.etree.ElementTree as etree -from xml.etree.ElementTree import Element, SubElement -import ahocorasick -from hyperlink import parse - -from collections import deque, defaultdict - +import markdown import requests - from django.conf import settings from django.db.models import Q +from hyperlink import parse +from markdown.extensions import codehilite, nl2br, sane_lists, tables +from typing_extensions import TypedDict -from markdown.extensions import codehilite, nl2br, tables, sane_lists +from zerver.lib import mention as mention from zerver.lib.bugdown import fenced_code from zerver.lib.bugdown.fenced_code import FENCE_RE +from zerver.lib.cache import NotFoundInCache, cache_with_key from zerver.lib.camo import get_camo_url -from zerver.lib.emoji import translate_emoticons, emoticon_regex, \ - name_to_codepoint, codepoint_to_name -from zerver.lib.mention import possible_mentions, \ - possible_user_group_mentions, extract_user_group -from zerver.lib.url_encoding import encode_stream, hash_util_encode +from zerver.lib.emoji import ( + codepoint_to_name, + emoticon_regex, + name_to_codepoint, + translate_emoticons, +) +from zerver.lib.exceptions import BugdownRenderingException +from zerver.lib.mention import extract_user_group, possible_mentions, possible_user_group_mentions +from zerver.lib.tex import render_tex from zerver.lib.thumbnail import user_uploads_or_external -from zerver.lib.timeout import timeout, TimeoutExpired -from zerver.lib.cache import cache_with_key, NotFoundInCache +from zerver.lib.timeout import TimeoutExpired, timeout +from zerver.lib.timezone import get_common_timezones +from zerver.lib.url_encoding import encode_stream, hash_util_encode from zerver.lib.url_preview import preview as link_preview from zerver.models import ( - all_realm_filters, - get_active_streams, MAX_MESSAGE_LENGTH, Message, Realm, - realm_filters_for_realm, - UserProfile, UserGroup, UserGroupMembership, + UserProfile, + all_realm_filters, + get_active_streams, + realm_filters_for_realm, ) -import zerver.lib.mention as mention -from zerver.lib.tex import render_tex -from zerver.lib.exceptions import BugdownRenderingException -from zerver.lib.timezone import get_common_timezones ReturnT = TypeVar('ReturnT') diff --git a/zerver/lib/bugdown/api_arguments_table_generator.py b/zerver/lib/bugdown/api_arguments_table_generator.py index a9209f7b05..5aab676008 100644 --- a/zerver/lib/bugdown/api_arguments_table_generator.py +++ b/zerver/lib/bugdown/api_arguments_table_generator.py @@ -1,13 +1,14 @@ -import re -import os import json +import os +import re +from typing import Any, Dict, List, Optional +import markdown from django.utils.html import escape as escape_html from markdown.extensions import Extension from markdown.preprocessors import Preprocessor + from zerver.openapi.openapi import get_openapi_parameters -from typing import Any, Dict, Optional, List -import markdown REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') diff --git a/zerver/lib/bugdown/api_return_values_table_generator.py b/zerver/lib/bugdown/api_return_values_table_generator.py index 865b74006f..9c56ac3790 100644 --- a/zerver/lib/bugdown/api_return_values_table_generator.py +++ b/zerver/lib/bugdown/api_return_values_table_generator.py @@ -1,10 +1,11 @@ import re +from typing import Any, Dict, List, Optional +import markdown from markdown.extensions import Extension from markdown.preprocessors import Preprocessor + from zerver.openapi.openapi import get_openapi_return_values -from typing import Any, Dict, Optional, List -import markdown REGEXP = re.compile(r'\{generate_return_values_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') diff --git a/zerver/lib/bugdown/fenced_code.py b/zerver/lib/bugdown/fenced_code.py index 6eb45bb1db..fe35b6a5f6 100644 --- a/zerver/lib/bugdown/fenced_code.py +++ b/zerver/lib/bugdown/fenced_code.py @@ -75,14 +75,15 @@ Dependencies: * [Pygments (optional)](http://pygments.org) """ - import re +from typing import Any, Dict, Iterable, List, MutableSequence, Optional + import markdown from django.utils.html import escape from markdown.extensions.codehilite import CodeHilite, CodeHiliteExtension + from zerver.lib.exceptions import BugdownRenderingException from zerver.lib.tex import render_tex -from typing import Any, Dict, Iterable, List, MutableSequence, Optional # Global vars FENCE_RE = re.compile(""" diff --git a/zerver/lib/bugdown/help_emoticon_translations_table.py b/zerver/lib/bugdown/help_emoticon_translations_table.py index 26e84e5157..079933c6b7 100644 --- a/zerver/lib/bugdown/help_emoticon_translations_table.py +++ b/zerver/lib/bugdown/help_emoticon_translations_table.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor from zerver.lib.emoji import EMOTICON_CONVERSIONS, name_to_codepoint diff --git a/zerver/lib/bugdown/help_relative_links.py b/zerver/lib/bugdown/help_relative_links.py index 910d908471..1f086b7cab 100644 --- a/zerver/lib/bugdown/help_relative_links.py +++ b/zerver/lib/bugdown/help_relative_links.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List, Optional from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor # There is a lot of duplicated code between this file and diff --git a/zerver/lib/bugdown/help_settings_links.py b/zerver/lib/bugdown/help_settings_links.py index 4c7b9e34e3..f60cccb116 100644 --- a/zerver/lib/bugdown/help_settings_links.py +++ b/zerver/lib/bugdown/help_settings_links.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List, Optional from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor # There is a lot of duplicated code between this file and diff --git a/zerver/lib/bugdown/include.py b/zerver/lib/bugdown/include.py index 0d3ba9938a..00536f56c9 100644 --- a/zerver/lib/bugdown/include.py +++ b/zerver/lib/bugdown/include.py @@ -1,9 +1,9 @@ -import re import os +import re from typing import Any, Dict, List import markdown -from markdown_include.include import MarkdownInclude, IncludePreprocessor +from markdown_include.include import IncludePreprocessor, MarkdownInclude from zerver.lib.exceptions import InvalidMarkdownIncludeStatement diff --git a/zerver/lib/bugdown/nested_code_blocks.py b/zerver/lib/bugdown/nested_code_blocks.py index 92a457e6c2..02f16dd05d 100644 --- a/zerver/lib/bugdown/nested_code_blocks.py +++ b/zerver/lib/bugdown/nested_code_blocks.py @@ -1,9 +1,11 @@ -from markdown.extensions import Extension -from typing import Any, Dict, Optional, List, Tuple -import markdown +from typing import Any, Dict, List, Optional, Tuple from xml.etree.ElementTree import Element, SubElement -from zerver.lib.bugdown import walk_tree_with_family, ResultWithFamily +import markdown +from markdown.extensions import Extension + +from zerver.lib.bugdown import ResultWithFamily, walk_tree_with_family + class NestedCodeBlocksRenderer(Extension): def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: diff --git a/zerver/lib/bugdown/tabbed_sections.py b/zerver/lib/bugdown/tabbed_sections.py index bd38d4e9ed..3cea656852 100644 --- a/zerver/lib/bugdown/tabbed_sections.py +++ b/zerver/lib/bugdown/tabbed_sections.py @@ -1,9 +1,9 @@ import re +from typing import Any, Dict, List, Optional +import markdown from markdown.extensions import Extension from markdown.preprocessors import Preprocessor -from typing import Any, Dict, Optional, List -import markdown START_TABBED_SECTION_REGEX = re.compile(r'^\{start_tabs\}$') END_TABBED_SECTION_REGEX = re.compile(r'^\{end_tabs\}$') diff --git a/zerver/lib/bugdown/testing_mocks.py b/zerver/lib/bugdown/testing_mocks.py index f47e9dd57a..04677c36d6 100644 --- a/zerver/lib/bugdown/testing_mocks.py +++ b/zerver/lib/bugdown/testing_mocks.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Optional -import ujson +import ujson NORMAL_TWEET = """{ "created_at": "Sat Sep 10 22:23:38 +0000 2011", diff --git a/zerver/lib/bulk_create.py b/zerver/lib/bulk_create.py index 5d05681bcc..c988197827 100644 --- a/zerver/lib/bulk_create.py +++ b/zerver/lib/bulk_create.py @@ -1,13 +1,12 @@ -from django.db.models import Model - from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from django.db.models import Model + +from zerver.lib.create_user import create_user_profile, get_display_email_address from zerver.lib.initial_password import initial_password -from zerver.models import Realm, Stream, UserProfile, \ - Subscription, Recipient, RealmAuditLog -from zerver.lib.create_user import create_user_profile, \ - get_display_email_address from zerver.lib.streams import render_stream_description +from zerver.models import Realm, RealmAuditLog, Recipient, Stream, Subscription, UserProfile + def bulk_create_users(realm: Realm, users_raw: Set[Tuple[str, str, str, bool]], diff --git a/zerver/lib/cache.py b/zerver/lib/cache.py index facb9056fc..ad0a7bb320 100644 --- a/zerver/lib/cache.py +++ b/zerver/lib/cache.py @@ -1,32 +1,41 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs -from functools import wraps - -from django.utils.lru_cache import lru_cache -from django.core.cache import cache as djcache -from django.core.cache import caches -from django.conf import settings -from django.db.models import Q -from django.core.cache.backends.base import BaseCache -from django.http import HttpRequest - -from typing import Any, Callable, Dict, Iterable, List, \ - Optional, Sequence, TypeVar, Tuple, TYPE_CHECKING - -from zerver.lib.utils import statsd, statsd_key, make_safe_digest -import time import base64 +import hashlib import logging +import os import random import re import sys +import time import traceback -import os -import hashlib +from functools import wraps +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypeVar, +) + +from django.conf import settings +from django.core.cache import cache as djcache +from django.core.cache import caches +from django.core.cache.backends.base import BaseCache +from django.db.models import Q +from django.http import HttpRequest +from django.utils.lru_cache import lru_cache + +from zerver.lib.utils import make_safe_digest, statsd, statsd_key if TYPE_CHECKING: # These modules have to be imported for type annotations but # they cannot be imported at runtime due to cyclic dependency. - from zerver.models import UserProfile, Realm, Message + from zerver.models import Message, Realm, UserProfile MEMCACHED_MAX_KEY_LENGTH = 250 diff --git a/zerver/lib/cache_helpers.py b/zerver/lib/cache_helpers.py index 944b5c4c27..fe3526c25c 100644 --- a/zerver/lib/cache_helpers.py +++ b/zerver/lib/cache_helpers.py @@ -1,28 +1,38 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs - -from typing import Any, Callable, Dict, List, Tuple - import datetime import logging +from importlib import import_module +from typing import Any, Callable, Dict, List, Tuple + +from django.conf import settings +from django.contrib.sessions.models import Session +from django.db.models import Q +from django.utils.timezone import now as timezone_now # This file needs to be different from cache.py because cache.py # cannot import anything from zerver.models or we'd have an import # loop from analytics.models import RealmCount -from django.conf import settings -from zerver.models import Message, UserProfile, Stream, get_stream_cache_key, \ - Client, get_client_cache_key, \ - Huddle, huddle_hash_cache_key -from zerver.lib.cache import \ - user_profile_by_api_key_cache_key, \ - user_profile_cache_key, get_remote_cache_time, get_remote_cache_requests, \ - cache_set_many, to_dict_cache_key_id +from zerver.lib.cache import ( + cache_set_many, + get_remote_cache_requests, + get_remote_cache_time, + to_dict_cache_key_id, + user_profile_by_api_key_cache_key, + user_profile_cache_key, +) from zerver.lib.message import MessageDict from zerver.lib.users import get_all_api_keys -from importlib import import_module -from django.contrib.sessions.models import Session -from django.db.models import Q -from django.utils.timezone import now as timezone_now +from zerver.models import ( + Client, + Huddle, + Message, + Stream, + UserProfile, + get_client_cache_key, + get_stream_cache_key, + huddle_hash_cache_key, +) MESSAGE_CACHE_SIZE = 75000 diff --git a/zerver/lib/camo.py b/zerver/lib/camo.py index d1c7a66f1c..5b57daa120 100644 --- a/zerver/lib/camo.py +++ b/zerver/lib/camo.py @@ -1,8 +1,10 @@ -from django.conf import settings import binascii import hashlib import hmac +from django.conf import settings + + def generate_camo_url(url: str) -> str: encoded_url = url.encode("utf-8") encoded_camo_key = settings.CAMO_KEY.encode("utf-8") diff --git a/zerver/lib/ccache.py b/zerver/lib/ccache.py index c3c746dec0..a2f159c3cf 100644 --- a/zerver/lib/ccache.py +++ b/zerver/lib/ccache.py @@ -1,4 +1,6 @@ -from typing import Any, Dict, List, Optional +import base64 +import struct +from typing import Any, Dict, List, Optional, Union # This file is adapted from samples/shellinabox/ssh-krb-wrapper in # https://github.com/davidben/webathena, which has the following @@ -26,9 +28,6 @@ from typing import Any, Dict, List, Optional # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import base64 -import struct -from typing import Union def force_bytes(s: Union[str, bytes], encoding: str='utf-8') -> bytes: """converts a string to binary string""" diff --git a/zerver/lib/context_managers.py b/zerver/lib/context_managers.py index 1a6700726c..0ca6c81af6 100644 --- a/zerver/lib/context_managers.py +++ b/zerver/lib/context_managers.py @@ -1,11 +1,10 @@ """ Context managers, i.e. things you can use with the 'with' statement. """ - - import fcntl from contextlib import contextmanager -from typing import Iterator, IO, Any, Union +from typing import IO, Any, Iterator, Union + @contextmanager def flock(lockfile: Union[int, IO[Any]], shared: bool=False) -> Iterator[None]: diff --git a/zerver/lib/create_user.py b/zerver/lib/create_user.py index 7924db0bbb..dc8e16a771 100644 --- a/zerver/lib/create_user.py +++ b/zerver/lib/create_user.py @@ -1,14 +1,22 @@ -from django.contrib.auth.models import UserManager -from django.utils.timezone import now as timezone_now -from zerver.models import UserProfile, Recipient, Subscription, Realm, Stream, \ - PreregistrationUser, get_fake_email_domain -from zerver.lib.upload import copy_avatar -from zerver.lib.hotspots import copy_hotpots -from zerver.lib.utils import generate_api_key +from typing import Optional import ujson +from django.contrib.auth.models import UserManager +from django.utils.timezone import now as timezone_now + +from zerver.lib.hotspots import copy_hotpots +from zerver.lib.upload import copy_avatar +from zerver.lib.utils import generate_api_key +from zerver.models import ( + PreregistrationUser, + Realm, + Recipient, + Stream, + Subscription, + UserProfile, + get_fake_email_domain, +) -from typing import Optional def copy_user_settings(source_profile: UserProfile, target_profile: UserProfile) -> None: """Warning: Does not save, to avoid extra database queries""" diff --git a/zerver/lib/db.py b/zerver/lib/db.py index 2971ac2edc..32d3d89d74 100644 --- a/zerver/lib/db.py +++ b/zerver/lib/db.py @@ -1,9 +1,8 @@ import time -from psycopg2.extensions import cursor, connection -from psycopg2.sql import Composable +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, TypeVar, Union -from typing import Callable, Optional, Iterable, Any, Dict, List, Union, TypeVar, \ - Mapping, Sequence +from psycopg2.extensions import connection, cursor +from psycopg2.sql import Composable CursorObj = TypeVar('CursorObj', bound=cursor) Query = Union[str, Composable] diff --git a/zerver/lib/debug.py b/zerver/lib/debug.py index d248816db1..3b44bd5261 100644 --- a/zerver/lib/debug.py +++ b/zerver/lib/debug.py @@ -8,10 +8,10 @@ import threading import traceback import tracemalloc from types import FrameType +from typing import Optional from django.conf import settings from django.utils.timezone import now as timezone_now -from typing import Optional logger = logging.getLogger('zulip.debug') diff --git a/zerver/lib/dev_ldap_directory.py b/zerver/lib/dev_ldap_directory.py index 2b29cab5e8..f703cad52d 100644 --- a/zerver/lib/dev_ldap_directory.py +++ b/zerver/lib/dev_ldap_directory.py @@ -4,6 +4,7 @@ import os from typing import Any, Dict, List, Optional from django.conf import settings + from zerver.lib.storage import static_path # See https://jackstromberg.com/2013/01/useraccountcontrol-attributeflag-values/ @@ -59,6 +60,7 @@ def init_fakeldap(directory: Optional[Dict[str, Dict[str, List[str]]]]=None) -> # because its dependency pbr uses pkgresources, which is # really slow to import.) from unittest import mock + from fakeldap import MockLDAP # Silent `django_auth_ldap` logger in dev mode to avoid diff --git a/zerver/lib/digest.py b/zerver/lib/digest.py index 4d7efe4915..feadb985de 100644 --- a/zerver/lib/digest.py +++ b/zerver/lib/digest.py @@ -1,21 +1,29 @@ -from typing import Any, Dict, List, Set, Tuple, Union - -from collections import defaultdict import datetime import logging +from collections import defaultdict +from typing import Any, Dict, List, Set, Tuple, Union from django.conf import settings from django.utils.timezone import now as timezone_now from confirmation.models import one_click_unsubscribe_link -from zerver.lib.email_notifications import build_message_list -from zerver.lib.send_email import send_future_email, FromAddress -from zerver.lib.url_encoding import encode_stream -from zerver.models import UserProfile, Recipient, Subscription, UserActivity, \ - get_active_streams, get_user_profile_by_id, Realm, Message, RealmAuditLog from zerver.context_processors import common_context -from zerver.lib.queue import queue_json_publish +from zerver.lib.email_notifications import build_message_list from zerver.lib.logging_util import log_to_file +from zerver.lib.queue import queue_json_publish +from zerver.lib.send_email import FromAddress, send_future_email +from zerver.lib.url_encoding import encode_stream +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Recipient, + Subscription, + UserActivity, + UserProfile, + get_active_streams, + get_user_profile_by_id, +) logger = logging.getLogger(__name__) log_to_file(logger, settings.DIGEST_LOG_PATH) diff --git a/zerver/lib/display_recipient.py b/zerver/lib/display_recipient.py index cea7d78708..2b933245cd 100644 --- a/zerver/lib/display_recipient.py +++ b/zerver/lib/display_recipient.py @@ -1,9 +1,14 @@ from typing import Dict, List, Optional, Set, Tuple -from typing_extensions import TypedDict -from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.lib.cache import cache_with_key, display_recipient_cache_key, generic_bulk_cached_fetch, \ - display_recipient_bulk_get_users_by_id_cache_key +from typing_extensions import TypedDict + +from zerver.lib.cache import ( + cache_with_key, + display_recipient_bulk_get_users_by_id_cache_key, + display_recipient_cache_key, + generic_bulk_cached_fetch, +) +from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient from zerver.models import Recipient, Stream, UserProfile, bulk_get_huddle_user_ids display_recipient_fields = [ diff --git a/zerver/lib/domains.py b/zerver/lib/domains.py index 307dc7d4eb..570d886efd 100644 --- a/zerver/lib/domains.py +++ b/zerver/lib/domains.py @@ -1,9 +1,9 @@ +import re +from typing import Optional + from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from typing import Optional - -import re def validate_domain(domain: Optional[str]) -> None: if domain is None or len(domain) == 0: diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py index 59ef84b214..6e1ddd6d56 100644 --- a/zerver/lib/email_mirror.py +++ b/zerver/lib/email_mirror.py @@ -1,31 +1,44 @@ -from typing import Dict, Optional, Tuple, List - import logging import re - from email.header import decode_header, make_header from email.message import Message as EmailMessage from email.utils import getaddresses +from typing import Dict, List, Optional, Tuple from django.conf import settings -from django.utils.timezone import timedelta, now as timezone_now +from django.utils.timezone import now as timezone_now +from django.utils.timezone import timedelta -from zerver.lib.actions import internal_send_private_message, \ - internal_send_stream_message, internal_send_huddle_message -from zerver.lib.email_mirror_helpers import decode_email_address, \ - get_email_gateway_message_string_from_address, ZulipEmailForwardError +from zerver.lib.actions import ( + internal_send_huddle_message, + internal_send_private_message, + internal_send_stream_message, +) +from zerver.lib.email_mirror_helpers import ( + ZulipEmailForwardError, + decode_email_address, + get_email_gateway_message_string_from_address, +) from zerver.lib.email_notifications import convert_html_to_markdown -from zerver.lib.queue import queue_json_publish -from zerver.lib.utils import generate_random_token -from zerver.lib.upload import upload_message_file -from zerver.lib.send_email import FromAddress -from zerver.lib.rate_limiter import RateLimitedObject from zerver.lib.exceptions import RateLimited from zerver.lib.message import truncate_body, truncate_topic -from zerver.models import Stream, Recipient, MissedMessageEmailAddress, \ - get_display_recipient, \ - Message, Realm, UserProfile, get_system_bot, get_user, get_stream_by_id_in_realm - +from zerver.lib.queue import queue_json_publish +from zerver.lib.rate_limiter import RateLimitedObject +from zerver.lib.send_email import FromAddress +from zerver.lib.upload import upload_message_file +from zerver.lib.utils import generate_random_token +from zerver.models import ( + Message, + MissedMessageEmailAddress, + Realm, + Recipient, + Stream, + UserProfile, + get_display_recipient, + get_stream_by_id_in_realm, + get_system_bot, + get_user, +) from zproject.backends import is_user_active logger = logging.getLogger(__name__) diff --git a/zerver/lib/email_mirror_helpers.py b/zerver/lib/email_mirror_helpers.py index f79ae02ac8..59ed695385 100644 --- a/zerver/lib/email_mirror_helpers.py +++ b/zerver/lib/email_mirror_helpers.py @@ -1,11 +1,11 @@ import re +from typing import Any, Callable, Dict, Tuple from django.conf import settings from django.utils.text import slugify from zerver.models import Stream -from typing import Any, Callable, Dict, Tuple def default_option_handler_factory(address_option: str) -> Callable[[Dict[str, Any]], None]: def option_setter(options_dict: Dict[str, Any]) -> None: diff --git a/zerver/lib/email_notifications.py b/zerver/lib/email_notifications.py index d7d6840787..9ba1b0d9ff 100644 --- a/zerver/lib/email_notifications.py +++ b/zerver/lib/email_notifications.py @@ -1,40 +1,44 @@ +import re +from collections import defaultdict +from datetime import timedelta +from email.utils import formataddr from typing import Any, Dict, Iterable, List, Optional, Tuple -from zerver.lib.types import DisplayRecipientT -from confirmation.models import one_click_unsubscribe_link +import html2text +import lxml.html +import pytz +from bs4 import BeautifulSoup from django.conf import settings +from django.contrib.auth import get_backends from django.utils.timezone import now as timezone_now from django.utils.translation import override as override_language from django.utils.translation import ugettext as _ -from django.contrib.auth import get_backends +from lxml.cssselect import CSSSelector +from confirmation.models import one_click_unsubscribe_link from zerver.decorator import statsd_increment from zerver.lib.message import bulk_access_messages from zerver.lib.queue import queue_json_publish -from zerver.lib.send_email import send_future_email, FromAddress -from zerver.lib.url_encoding import personal_narrow_url, huddle_narrow_url, \ - stream_narrow_url, topic_narrow_url +from zerver.lib.send_email import FromAddress, send_future_email +from zerver.lib.types import DisplayRecipientT +from zerver.lib.url_encoding import ( + huddle_narrow_url, + personal_narrow_url, + stream_narrow_url, + topic_narrow_url, +) from zerver.models import ( + Message, Recipient, - UserMessage, Stream, - get_display_recipient, + UserMessage, UserProfile, + get_context_for_message, + get_display_recipient, get_user_profile_by_id, receives_offline_email_notifications, - get_context_for_message, - Message, ) -from datetime import timedelta -from email.utils import formataddr -import html2text -from lxml.cssselect import CSSSelector -import lxml.html -import re -from collections import defaultdict -import pytz -from bs4 import BeautifulSoup def relative_to_full_url(base_url: str, content: str) -> str: # Convert relative URLs to absolute URLs. @@ -297,6 +301,7 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile, for a group of messages that share a recipient (and topic) """ from zerver.context_processors import common_context + # Disabled missedmessage emails internally if not user_profile.enable_offline_email_notifications: return @@ -558,7 +563,7 @@ def enqueue_welcome_emails(user: UserProfile, realm_creation: bool=False) -> Non context['getting_started_link'] = "https://zulip.com" # Imported here to avoid import cycles. - from zproject.backends import email_belongs_to_ldap, ZulipLDAPAuthBackend + from zproject.backends import ZulipLDAPAuthBackend, email_belongs_to_ldap if email_belongs_to_ldap(user.realm, user.delivery_email): context["ldap"] = True diff --git a/zerver/lib/email_validation.py b/zerver/lib/email_validation.py index d67398fd98..dc37305518 100644 --- a/zerver/lib/email_validation.py +++ b/zerver/lib/email_validation.py @@ -3,21 +3,22 @@ from typing import Callable, Dict, Optional, Set, Tuple from django.core import validators from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from zerver.lib.name_restrictions import is_disposable_domain +from zerver.lib.name_restrictions import is_disposable_domain # TODO: Move DisposableEmailError, etc. into here. from zerver.models import ( - email_to_username, - email_to_domain, - get_users_by_delivery_email, - is_cross_realm_bot_email, DisposableEmailError, DomainNotAllowedForRealmError, EmailContainsPlusError, Realm, RealmDomain, + email_to_domain, + email_to_username, + get_users_by_delivery_email, + is_cross_realm_bot_email, ) + def validate_disposable(email: str) -> None: if is_disposable_domain(email_to_domain(email)): raise DisposableEmailError diff --git a/zerver/lib/emoji.py b/zerver/lib/emoji.py index 902732d747..df339df2f2 100644 --- a/zerver/lib/emoji.py +++ b/zerver/lib/emoji.py @@ -1,14 +1,14 @@ import os import re -import ujson - -from django.utils.translation import ugettext as _ from typing import Optional, Tuple +import ujson +from django.utils.translation import ugettext as _ + +from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.lib.request import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend -from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.models import Reaction, Realm, RealmEmoji, UserProfile emoji_codes_path = static_path("generated/emoji/emoji_codes.json") diff --git a/zerver/lib/error_notify.py b/zerver/lib/error_notify.py index fb3d5ddba5..21b8f09960 100644 --- a/zerver/lib/error_notify.py +++ b/zerver/lib/error_notify.py @@ -1,16 +1,17 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - from collections import defaultdict +from typing import Any, Dict, Optional + from django.conf import settings from django.core.mail import mail_admins from django.http import HttpResponse from django.utils.translation import ugettext as _ -from typing import Any, Dict, Optional from zerver.filters import clean_data_from_query_parameters -from zerver.models import get_stream, get_system_bot from zerver.lib.actions import internal_send_stream_message -from zerver.lib.response import json_success, json_error +from zerver.lib.response import json_error, json_success +from zerver.models import get_stream, get_system_bot + def format_email_subject(email_subject: str) -> str: """ diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 1522dd9fcd..018b18f55d 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -1,20 +1,28 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. - import copy - -from django.utils.translation import ugettext as _ -from django.conf import settings from importlib import import_module -from typing import ( - Any, Callable, Dict, Iterable, Optional, Sequence, Set, -) +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Set + +from django.conf import settings +from django.utils.translation import ugettext as _ session_engine = import_module(settings.SESSION_ENGINE) +from version import API_FEATURE_LEVEL, ZULIP_VERSION +from zerver.lib.actions import ( + default_stream_groups_to_dicts_sorted, + do_get_streams, + gather_subscriptions_helper, + get_available_notification_sounds, + get_default_streams_for_realm, + get_owned_bot_dicts, + streams_to_dicts_sorted, +) from zerver.lib.alert_words import user_alert_words from zerver.lib.avatar import avatar_url from zerver.lib.bot_config import load_bot_config_template +from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots from zerver.lib.integrations import EMBEDDED_BOTS, WEBHOOK_INTEGRATIONS from zerver.lib.message import ( @@ -27,40 +35,34 @@ from zerver.lib.message import ( remove_message_id_from_unread_mgs, ) from zerver.lib.narrow import check_supported_events_narrow_filter, read_stop_words -from zerver.lib.presence import ( - get_presences_for_realm, - get_presence_for_user, -) +from zerver.lib.presence import get_presence_for_user, get_presences_for_realm from zerver.lib.push_notifications import push_notifications_enabled -from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_url from zerver.lib.request import JsonableError +from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated from zerver.lib.stream_subscription import handle_stream_notifications_compatibility from zerver.lib.topic import TOPIC_NAME from zerver.lib.topic_mutes import get_topic_mutes -from zerver.lib.actions import ( - do_get_streams, - get_default_streams_for_realm, - gather_subscriptions_helper, - streams_to_dicts_sorted, - default_stream_groups_to_dicts_sorted, - get_owned_bot_dicts, - get_available_notification_sounds, -) -from zerver.lib.users import get_cross_realm_dicts, get_raw_user_data, is_administrator_role from zerver.lib.user_groups import user_groups_in_realm_serialized from zerver.lib.user_status import get_user_info_dict -from zerver.tornado.event_queue import request_event_queue, get_user_events +from zerver.lib.users import get_cross_realm_dicts, get_raw_user_data, is_administrator_role from zerver.models import ( - Client, Message, Realm, UserProfile, UserMessage, + Client, + CustomProfileField, + Message, + Realm, + Stream, + UserMessage, + UserProfile, + custom_profile_fields_for_realm, + get_default_stream_groups, + get_realm_domains, realm_filters_for_realm, - custom_profile_fields_for_realm, get_realm_domains, - get_default_stream_groups, CustomProfileField, Stream, ) +from zerver.tornado.event_queue import get_user_events, request_event_queue from zproject.backends import email_auth_enabled, password_auth_enabled -from version import ZULIP_VERSION, API_FEATURE_LEVEL -from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS + def add_realm_logo_fields(state: Dict[str, Any], realm: Realm) -> None: state['realm_logo_url'] = get_realm_logo_url(realm, night = False) diff --git a/zerver/lib/exceptions.py b/zerver/lib/exceptions.py index 897c2dca6e..57db95d943 100644 --- a/zerver/lib/exceptions.py +++ b/zerver/lib/exceptions.py @@ -1,9 +1,8 @@ from enum import Enum -from typing import Any, Dict, List, NoReturn, Type, TypeVar, Optional +from typing import Any, Dict, List, NoReturn, Optional, Type, TypeVar from django.utils.translation import ugettext as _ - T = TypeVar("T", bound="AbstractEnum") class AbstractEnum(Enum): diff --git a/zerver/lib/export.py b/zerver/lib/export.py index 7bbe837770..090a40187f 100644 --- a/zerver/lib/export.py +++ b/zerver/lib/export.py @@ -7,35 +7,61 @@ # (2) if it doesn't belong in EXCLUDED_TABLES, add a Config object for # it to get_realm_config. import datetime +import glob +import logging +import os +import shutil +import subprocess +import tempfile +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union + import boto3 +import ujson from boto3.resources.base import ServiceResource from django.apps import apps from django.conf import settings from django.forms.models import model_to_dict -from django.utils.timezone import make_aware as timezone_make_aware from django.utils.timezone import is_naive as timezone_is_naive -import glob -import logging -import os -import ujson -import subprocess -import tempfile -import shutil -from scripts.lib.zulip_tools import overwrite_symlink -from zerver.lib.avatar_hash import user_avatar_path_from_ids -from analytics.models import RealmCount, UserCount, StreamCount -from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \ - UserMessage, Subscription, Message, RealmEmoji, RealmFilter, Reaction, \ - RealmDomain, Recipient, DefaultStream, get_user_profile_by_id, \ - UserPresence, UserActivity, UserActivityInterval, CustomProfileField, \ - CustomProfileFieldValue, get_display_recipient, Attachment, get_system_bot, \ - RealmAuditLog, UserHotspot, MutedTopic, Service, UserGroup, \ - UserGroupMembership, BotStorageData, BotConfigData -from zerver.lib.pysa import mark_sanitized +from django.utils.timezone import make_aware as timezone_make_aware import zerver.lib.upload -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, \ - Union +from analytics.models import RealmCount, StreamCount, UserCount +from scripts.lib.zulip_tools import overwrite_symlink +from zerver.lib.avatar_hash import user_avatar_path_from_ids +from zerver.lib.pysa import mark_sanitized +from zerver.models import ( + Attachment, + BotConfigData, + BotStorageData, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Huddle, + Message, + MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + Service, + Stream, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + get_display_recipient, + get_system_bot, + get_user_profile_by_id, +) # Custom mypy types follow: Record = Dict[str, Any] diff --git a/zerver/lib/external_accounts.py b/zerver/lib/external_accounts.py index bcb08c17c6..81a0772895 100644 --- a/zerver/lib/external_accounts.py +++ b/zerver/lib/external_accounts.py @@ -2,11 +2,15 @@ This module stores data for "External Account" custom profile field. """ from typing import Optional + from django.utils.translation import ugettext as _ -from zerver.lib.validator import check_required_string, \ - check_external_account_url_pattern, check_dict_only from zerver.lib.types import ProfileFieldData +from zerver.lib.validator import ( + check_dict_only, + check_external_account_url_pattern, + check_required_string, +) # Default external account fields are by default available # to realm admins, where realm admin only need to select diff --git a/zerver/lib/fix_unreads.py b/zerver/lib/fix_unreads.py index 9bc46a8f4d..f13b2811f7 100644 --- a/zerver/lib/fix_unreads.py +++ b/zerver/lib/fix_unreads.py @@ -1,7 +1,7 @@ -import time import logging - +import time from typing import Callable, List, TypeVar + from psycopg2.extensions import cursor from psycopg2.sql import SQL diff --git a/zerver/lib/generate_test_data.py b/zerver/lib/generate_test_data.py index 979bd5ad82..a1432535ab 100644 --- a/zerver/lib/generate_test_data.py +++ b/zerver/lib/generate_test_data.py @@ -1,11 +1,13 @@ import itertools -import ujson -import random -from typing import List, Dict, Any import os +import random +from typing import Any, Dict, List + +import ujson from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path + def load_config() -> Dict[str, Any]: with open("zerver/tests/fixtures/config.generate_data.json") as infile: config = ujson.load(infile) diff --git a/zerver/lib/hotspots.py b/zerver/lib/hotspots.py index c37491f3d3..322654d522 100644 --- a/zerver/lib/hotspots.py +++ b/zerver/lib/hotspots.py @@ -1,11 +1,11 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/hotspots.html # for documentation on this subsystem. +from typing import Dict, List + from django.conf import settings from django.utils.translation import ugettext as _ -from zerver.models import UserProfile, UserHotspot - -from typing import List, Dict +from zerver.models import UserHotspot, UserProfile ALL_HOTSPOTS: Dict[str, Dict[str, str]] = { 'intro_reply': { diff --git a/zerver/lib/html_diff.py b/zerver/lib/html_diff.py index 16686a69e4..cc8d4fad6c 100644 --- a/zerver/lib/html_diff.py +++ b/zerver/lib/html_diff.py @@ -1,8 +1,9 @@ -import lxml - -from lxml.html.diff import htmldiff from typing import Optional +import lxml +from lxml.html.diff import htmldiff + + def highlight_with_class(text: str, klass: str) -> str: return f'{text}' diff --git a/zerver/lib/html_to_text.py b/zerver/lib/html_to_text.py index c7bac8e275..9a450f5dc8 100644 --- a/zerver/lib/html_to_text.py +++ b/zerver/lib/html_to_text.py @@ -6,6 +6,7 @@ from django.utils.html import escape from zerver.lib.cache import cache_with_key, open_graph_description_cache_key + def html_to_text(content: str, tags: Optional[Dict[str, str]]=None) -> str: bs = BeautifulSoup(content, features='lxml') # Skip any admonition (warning) blocks, since they're diff --git a/zerver/lib/i18n.py b/zerver/lib/i18n.py index b63d4c5934..8caf89ad9c 100644 --- a/zerver/lib/i18n.py +++ b/zerver/lib/i18n.py @@ -1,16 +1,15 @@ +import logging import operator +import os +from itertools import zip_longest +from typing import Any, Dict, List +import ujson from django.conf import settings from django.utils import translation -from django.utils.translation import ugettext as _ from django.utils.lru_cache import lru_cache +from django.utils.translation import ugettext as _ -from itertools import zip_longest -from typing import Any, List, Dict - -import os -import ujson -import logging def with_language(string: str, language: str) -> str: """ diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index c58673002a..ce0f769afb 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -1,44 +1,71 @@ import datetime import logging import os -import ujson import shutil +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast import boto3 +import ujson from bs4 import BeautifulSoup from django.conf import settings from django.db import connection from django.db.models import Max from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Optional, Set, Tuple, \ - Iterable, cast from psycopg2.extras import execute_values -from psycopg2.sql import Identifier, SQL +from psycopg2.sql import SQL, Identifier from analytics.models import RealmCount, StreamCount, UserCount -from zerver.lib.actions import UserMessageLite, bulk_insert_ums, \ - do_change_plan_type, do_change_avatar_fields +from zerver.lib.actions import ( + UserMessageLite, + bulk_insert_ums, + do_change_avatar_fields, + do_change_plan_type, +) from zerver.lib.avatar_hash import user_avatar_path_from_ids -from zerver.lib.bulk_create import bulk_create_users, bulk_set_users_or_streams_recipient_fields -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.export import DATE_FIELDS, \ - Record, TableData, TableName, Field, Path -from zerver.lib.message import do_render_markdown from zerver.lib.bugdown import version as bugdown_version -from zerver.lib.streams import render_stream_description -from zerver.lib.upload import random_name, sanitize_name, \ - guess_type, BadImageError -from zerver.lib.utils import generate_api_key, process_list_in_batches +from zerver.lib.bulk_create import bulk_create_users, bulk_set_users_or_streams_recipient_fields +from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName +from zerver.lib.message import do_render_markdown from zerver.lib.parallel import run_parallel -from zerver.lib.server_initialization import server_initialized, create_internal_realm -from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \ - UserMessage, Subscription, Message, RealmEmoji, \ - RealmDomain, Recipient, get_user_profile_by_id, \ - UserPresence, UserActivity, UserActivityInterval, Reaction, \ - CustomProfileField, CustomProfileFieldValue, RealmAuditLog, \ - Attachment, get_system_bot, email_to_username, get_huddle_hash, \ - UserHotspot, MutedTopic, Service, UserGroup, UserGroupMembership, \ - BotStorageData, BotConfigData, DefaultStream, RealmFilter +from zerver.lib.server_initialization import create_internal_realm, server_initialized +from zerver.lib.streams import render_stream_description +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.upload import BadImageError, guess_type, random_name, sanitize_name +from zerver.lib.utils import generate_api_key, process_list_in_batches +from zerver.models import ( + Attachment, + BotConfigData, + BotStorageData, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Huddle, + Message, + MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + Service, + Stream, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + email_to_username, + get_huddle_hash, + get_system_bot, + get_user_profile_by_id, +) realm_tables = [("zerver_defaultstream", DefaultStream, "defaultstream"), ("zerver_realmemoji", RealmEmoji, "realmemoji"), @@ -707,6 +734,7 @@ def import_uploads(realm: Realm, import_dir: Path, processes: int, processing_av if processing_avatars: from zerver.lib.upload import upload_backend + # Ensure that we have medium-size avatar images for every # avatar. TODO: This implementation is hacky, both in that it # does get_user_profile_by_id for each user, and in that it diff --git a/zerver/lib/initial_password.py b/zerver/lib/initial_password.py index 4e2d5360f9..e53a088ef1 100644 --- a/zerver/lib/initial_password.py +++ b/zerver/lib/initial_password.py @@ -1,10 +1,9 @@ -from django.conf import settings - -import hashlib import base64 - +import hashlib from typing import Optional +from django.conf import settings + def initial_password(email: str) -> Optional[str]: """Given an email address, returns the initial password for that account, as diff --git a/zerver/lib/integrations.py b/zerver/lib/integrations.py index abadb2db15..8a1c81e1fd 100644 --- a/zerver/lib/integrations.py +++ b/zerver/lib/integrations.py @@ -1,16 +1,16 @@ import os - from dataclasses import dataclass -from typing import Dict, List, Optional, Any, Tuple +from typing import Any, Dict, List, Optional, Tuple + from django.conf.urls import url from django.contrib.staticfiles.storage import staticfiles_storage from django.urls.resolvers import RegexPattern from django.utils.module_loading import import_string from django.utils.translation import ugettext as _ + from zerver.lib.storage import static_path from zerver.lib.types import Validator - """This module declares all of the (documented) integrations available in the Zulip server. The Integration class is used as part of generating the documentation on the /integrations page, while the diff --git a/zerver/lib/logging_util.py b/zerver/lib/logging_util.py index 23ddb43e21..e250494fd9 100644 --- a/zerver/lib/logging_util.py +++ b/zerver/lib/logging_util.py @@ -1,16 +1,16 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - -from django.utils.timezone import now as timezone_now - import hashlib import logging import threading import traceback -from typing import Optional, Tuple from datetime import datetime, timedelta, timezone +from logging import Logger +from typing import Optional, Tuple + from django.conf import settings from django.core.cache import cache -from logging import Logger +from django.utils.timezone import now as timezone_now + class _RateLimitFilter: """This class is designed to rate-limit Django error reporting diff --git a/zerver/lib/management.py b/zerver/lib/management.py index f7c90fa9f5..3da9b47a6e 100644 --- a/zerver/lib/management.py +++ b/zerver/lib/management.py @@ -1,15 +1,14 @@ # Library code for use in management commands - import time - from argparse import ArgumentParser, RawTextHelpFormatter +from typing import Any, Dict, List, Optional from django.conf import settings from django.core.exceptions import MultipleObjectsReturned from django.core.management.base import BaseCommand, CommandError -from typing import Any, Dict, Optional, List -from zerver.models import Realm, UserProfile, Client, get_client +from zerver.models import Client, Realm, UserProfile, get_client + def is_integer_string(val: str) -> bool: try: diff --git a/zerver/lib/mdiff.py b/zerver/lib/mdiff.py index 09a2a93098..3bde46030f 100755 --- a/zerver/lib/mdiff.py +++ b/zerver/lib/mdiff.py @@ -1,6 +1,7 @@ +import logging import os import subprocess -import logging + class DiffException(Exception): pass diff --git a/zerver/lib/mention.py b/zerver/lib/mention.py index b1fc0e8e9c..2691190dc1 100644 --- a/zerver/lib/mention.py +++ b/zerver/lib/mention.py @@ -1,6 +1,5 @@ -from typing import Optional, Set, Tuple - import re +from typing import Optional, Set, Tuple # Match multi-word string between @** ** or match any one-word # sequences after @ diff --git a/zerver/lib/message.py b/zerver/lib/message.py index 1d783e39e7..5be36afc37 100644 --- a/zerver/lib/message.py +++ b/zerver/lib/message.py @@ -1,64 +1,54 @@ -import datetime -import ujson -import zlib -import ahocorasick import copy +import datetime +import zlib +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now +import ahocorasick +import ujson from django.db import connection from django.db.models import Sum +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ from psycopg2.sql import SQL +from typing_extensions import TypedDict from analytics.lib.counts import COUNT_STATS, RealmCount - +from zerver.lib import bugdown as bugdown from zerver.lib.avatar import get_avatar_field -import zerver.lib.bugdown as bugdown from zerver.lib.cache import ( cache_with_key, generic_bulk_cached_fetch, to_dict_cache_key, to_dict_cache_key_id, ) -from zerver.lib.display_recipient import UserDisplayRecipient, DisplayRecipientT, \ - bulk_fetch_display_recipients +from zerver.lib.display_recipient import ( + DisplayRecipientT, + UserDisplayRecipient, + bulk_fetch_display_recipients, +) from zerver.lib.request import JsonableError -from zerver.lib.stream_subscription import ( - get_stream_subscriptions_for_user, -) +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - DB_TOPIC_NAME, - MESSAGE__TOPIC, - TOPIC_LINKS, - TOPIC_NAME, -) -from zerver.lib.topic_mutes import ( - build_topic_mute_checker, - topic_is_muted, -) - +from zerver.lib.topic import DB_TOPIC_NAME, MESSAGE__TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.topic_mutes import build_topic_mute_checker, topic_is_muted from zerver.models import ( - get_display_recipient_by_id, - get_user_profile_by_id, - query_for_ids, + MAX_MESSAGE_LENGTH, + MAX_TOPIC_NAME_LENGTH, Message, + Reaction, Realm, Recipient, Stream, SubMessage, Subscription, - UserProfile, UserMessage, - Reaction, + UserProfile, + get_display_recipient_by_id, + get_user_profile_by_id, get_usermessage_by_message_id, - MAX_MESSAGE_LENGTH, - MAX_TOPIC_NAME_LENGTH, + query_for_ids, ) -from typing import Any, Dict, List, Optional, Set, Tuple, Sequence -from typing_extensions import TypedDict - RealmAlertWord = Dict[int, List[str]] class RawUnreadMessagesResult(TypedDict): diff --git a/zerver/lib/migrate.py b/zerver/lib/migrate.py index 3fa2bd3167..8fb5399261 100644 --- a/zerver/lib/migrate.py +++ b/zerver/lib/migrate.py @@ -1,8 +1,8 @@ -from psycopg2.extensions import cursor -from psycopg2.sql import Composable, Identifier, SQL +import time from typing import List, TypeVar -import time +from psycopg2.extensions import cursor +from psycopg2.sql import SQL, Composable, Identifier CursorObj = TypeVar('CursorObj', bound=cursor) diff --git a/zerver/lib/mobile_auth_otp.py b/zerver/lib/mobile_auth_otp.py index 83df1fef71..bd009fc92e 100644 --- a/zerver/lib/mobile_auth_otp.py +++ b/zerver/lib/mobile_auth_otp.py @@ -7,10 +7,11 @@ # # The decryption logic here isn't actually used by the flow; we just # have it here as part of testing the overall library. - import binascii + from zerver.models import UserProfile + def xor_hex_strings(bytes_a: str, bytes_b: str) -> str: """Given two hex strings of equal length, return a hex string with the bitwise xor of the two hex strings.""" diff --git a/zerver/lib/name_restrictions.py b/zerver/lib/name_restrictions.py index 2db6d57ceb..02f3f9f401 100644 --- a/zerver/lib/name_restrictions.py +++ b/zerver/lib/name_restrictions.py @@ -1,5 +1,6 @@ from disposable_email_domains import blacklist + def is_reserved_subdomain(subdomain: str) -> bool: if subdomain in ZULIP_RESERVED_SUBDOMAINS: return True diff --git a/zerver/lib/narrow.py b/zerver/lib/narrow.py index d3c080a8e5..e77f189130 100644 --- a/zerver/lib/narrow.py +++ b/zerver/lib/narrow.py @@ -1,13 +1,11 @@ import os +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence -from zerver.lib.request import JsonableError -from zerver.lib.topic import ( - get_topic_from_message_info, -) from django.conf import settings from django.utils.translation import ugettext as _ -from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence +from zerver.lib.request import JsonableError +from zerver.lib.topic import get_topic_from_message_info stop_words_list: Optional[List[str]] = None def read_stop_words() -> List[str]: diff --git a/zerver/lib/onboarding.py b/zerver/lib/onboarding.py index bcdde3ac5a..9b8b593a2d 100644 --- a/zerver/lib/onboarding.py +++ b/zerver/lib/onboarding.py @@ -1,15 +1,19 @@ +from typing import Dict, List + from django.conf import settings from django.db.models import Count from django.utils.translation import ugettext as _ -from zerver.lib.actions import \ - internal_prep_stream_message_by_name, internal_send_private_message, \ - do_send_messages, \ - do_add_reaction, create_users +from zerver.lib.actions import ( + create_users, + do_add_reaction, + do_send_messages, + internal_prep_stream_message_by_name, + internal_send_private_message, +) from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.models import Message, Realm, UserProfile, get_system_bot -from typing import Dict, List def missing_any_realm_internal_bots() -> bool: bot_emails = [bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,) diff --git a/zerver/lib/outgoing_webhook.py b/zerver/lib/outgoing_webhook.py index 88f2e62255..bfcc7e2ac9 100644 --- a/zerver/lib/outgoing_webhook.py +++ b/zerver/lib/outgoing_webhook.py @@ -1,23 +1,29 @@ -from typing import Any, AnyStr, Dict, Optional - import abc -import requests import json import logging +from typing import Any, AnyStr, Dict, Optional + +import requests +from django.utils.translation import ugettext as _ from requests import Response -from django.utils.translation import ugettext as _ - -from zerver.models import UserProfile, get_user_profile_by_id, get_client, \ - GENERIC_INTERFACE, Service, SLACK_INTERFACE, email_to_domain +from version import ZULIP_VERSION +from zerver.decorator import JsonableError from zerver.lib.actions import check_send_message from zerver.lib.message import MessageDict from zerver.lib.queue import retry_event from zerver.lib.topic import get_topic_from_message_info from zerver.lib.url_encoding import near_message_url -from zerver.decorator import JsonableError +from zerver.models import ( + GENERIC_INTERFACE, + SLACK_INTERFACE, + Service, + UserProfile, + email_to_domain, + get_client, + get_user_profile_by_id, +) -from version import ZULIP_VERSION class OutgoingWebhookServiceInterface(metaclass=abc.ABCMeta): diff --git a/zerver/lib/parallel.py b/zerver/lib/parallel.py index 91765874bf..b182b73aae 100644 --- a/zerver/lib/parallel.py +++ b/zerver/lib/parallel.py @@ -1,9 +1,8 @@ -from typing import Dict, Iterable, Tuple, Callable, TypeVar, Iterator - +import errno import os import pty import sys -import errno +from typing import Callable, Dict, Iterable, Iterator, Tuple, TypeVar JobData = TypeVar('JobData') diff --git a/zerver/lib/presence.py b/zerver/lib/presence.py index 58c99a343d..425153f628 100644 --- a/zerver/lib/presence.py +++ b/zerver/lib/presence.py @@ -1,21 +1,14 @@ -from collections import defaultdict - import datetime import itertools import time +from collections import defaultdict +from typing import Any, Dict, List, Set from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Set - from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import ( - query_for_ids, - PushDeviceToken, - Realm, - UserPresence, - UserProfile, -) +from zerver.models import PushDeviceToken, Realm, UserPresence, UserProfile, query_for_ids + def get_status_dicts_for_rows(all_rows: List[Dict[str, Any]], mobile_user_ids: Set[int], diff --git a/zerver/lib/profile.py b/zerver/lib/profile.py index 200a434e5d..3aac3844be 100644 --- a/zerver/lib/profile.py +++ b/zerver/lib/profile.py @@ -1,7 +1,6 @@ import cProfile - from functools import wraps -from typing import Any, TypeVar, Callable +from typing import Any, Callable, TypeVar ReturnT = TypeVar('ReturnT') diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index 5abbdd370a..1029eab664 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -1,32 +1,37 @@ import base64 import binascii import logging -import lxml.html import re import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING, Union - +import gcm +import lxml.html +import ujson from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import F from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -import gcm -import ujson from zerver.decorator import statsd_increment from zerver.lib.avatar import absolute_avatar_url from zerver.lib.exceptions import JsonableError -from zerver.lib.message import access_message, \ - bulk_access_messages_expect_usermessage, huddle_users -from zerver.lib.remote_server import send_to_push_bouncer, send_json_to_push_bouncer +from zerver.lib.message import access_message, bulk_access_messages_expect_usermessage, huddle_users +from zerver.lib.remote_server import send_json_to_push_bouncer, send_to_push_bouncer from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import PushDeviceToken, Message, Recipient, \ - UserMessage, UserProfile, \ - get_display_recipient, receives_offline_push_notifications, \ - receives_online_notifications, get_user_profile_by_id, \ - ArchivedMessage +from zerver.models import ( + ArchivedMessage, + Message, + PushDeviceToken, + Recipient, + UserMessage, + UserProfile, + get_display_recipient, + get_user_profile_by_id, + receives_offline_push_notifications, + receives_online_notifications, +) if TYPE_CHECKING: from apns2.client import APNsClient diff --git a/zerver/lib/pysa.py b/zerver/lib/pysa.py index 6019b29378..c0fa12484a 100644 --- a/zerver/lib/pysa.py +++ b/zerver/lib/pysa.py @@ -1,6 +1,5 @@ from typing import TypeVar - T = TypeVar("T") diff --git a/zerver/lib/queue.py b/zerver/lib/queue.py index 176a06ac02..f132e2a11a 100644 --- a/zerver/lib/queue.py +++ b/zerver/lib/queue.py @@ -1,17 +1,17 @@ -from collections import defaultdict import logging import random import threading import time +from collections import defaultdict from typing import Any, Callable, Dict, List, Mapping, Optional, Set -from django.conf import settings import pika import pika.adapters.tornado_connection +import ujson +from django.conf import settings from pika.adapters.blocking_connection import BlockingChannel from pika.spec import Basic from tornado import ioloop -import ujson from zerver.lib.utils import statsd diff --git a/zerver/lib/rate_limiter.py b/zerver/lib/rate_limiter.py index 87c7fbf944..78826dc48d 100644 --- a/zerver/lib/rate_limiter.py +++ b/zerver/lib/rate_limiter.py @@ -1,20 +1,18 @@ +import logging import os - +import time from abc import ABC, abstractmethod from typing import Dict, List, Optional, Tuple, Type +import redis from django.conf import settings from django.http import HttpRequest + from zerver.lib.exceptions import RateLimited from zerver.lib.redis_utils import get_redis_client from zerver.lib.utils import statsd - from zerver.models import UserProfile -import logging -import redis -import time - # Implement a rate-limiting scheme inspired by the one described here, but heavily modified # https://www.domaintools.com/resources/blog/rate-limiting-with-redis diff --git a/zerver/lib/realm_description.py b/zerver/lib/realm_description.py index 39b10e9156..409a4d3fbd 100644 --- a/zerver/lib/realm_description.py +++ b/zerver/lib/realm_description.py @@ -1,8 +1,12 @@ -from zerver.models import Realm -from zerver.lib.cache import cache_with_key, realm_rendered_description_cache_key, \ - realm_text_description_cache_key from zerver.lib.bugdown import convert as bugdown_convert +from zerver.lib.cache import ( + cache_with_key, + realm_rendered_description_cache_key, + realm_text_description_cache_key, +) from zerver.lib.html_to_text import html_to_text +from zerver.models import Realm + @cache_with_key(realm_rendered_description_cache_key, timeout=3600*24*7) def get_realm_rendered_description(realm: Realm) -> str: diff --git a/zerver/lib/realm_icon.py b/zerver/lib/realm_icon.py index 81a9eeb1d1..3813d08048 100644 --- a/zerver/lib/realm_icon.py +++ b/zerver/lib/realm_icon.py @@ -4,6 +4,7 @@ from zerver.lib.avatar_hash import gravatar_hash from zerver.lib.upload import upload_backend from zerver.models import Realm + def realm_icon_url(realm: Realm) -> str: return get_realm_icon_url(realm) diff --git a/zerver/lib/realm_logo.py b/zerver/lib/realm_logo.py index 6c3eac83ea..a4273ac805 100644 --- a/zerver/lib/realm_logo.py +++ b/zerver/lib/realm_logo.py @@ -5,6 +5,7 @@ from django.conf import settings from zerver.lib.upload import upload_backend from zerver.models import Realm + def get_realm_logo_url(realm: Realm, night: bool) -> str: if night: logo_source = realm.night_logo_source diff --git a/zerver/lib/redis_utils.py b/zerver/lib/redis_utils.py index 11144e1627..af150680bb 100644 --- a/zerver/lib/redis_utils.py +++ b/zerver/lib/redis_utils.py @@ -1,10 +1,11 @@ -from django.conf import settings -from typing import Any, Dict, Optional -from zerver.lib.utils import generate_random_token - import re +from typing import Any, Dict, Optional + import redis import ujson +from django.conf import settings + +from zerver.lib.utils import generate_random_token # Redis accepts keys up to 512MB in size, but there's no reason for us to use such size, # so we want to stay limited to 1024 characters. diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 892c249b79..7ed113fe44 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -1,9 +1,9 @@ import logging -import requests -import ujson import urllib from typing import Any, Dict, List, Optional, Tuple, Union +import requests +import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ @@ -14,6 +14,7 @@ from zerver.lib.exceptions import JsonableError from zerver.lib.export import floatify_datetime_fields from zerver.models import RealmAuditLog + class PushNotificationBouncerException(Exception): pass diff --git a/zerver/lib/request.py b/zerver/lib/request.py index 515c420998..d542445b57 100644 --- a/zerver/lib/request.py +++ b/zerver/lib/request.py @@ -1,18 +1,16 @@ from collections import defaultdict from functools import wraps from types import FunctionType +from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union, cast, overload + import ujson - +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ +from typing_extensions import Literal -from zerver.lib.exceptions import JsonableError, ErrorCode, \ - InvalidJSONError +from zerver.lib.exceptions import ErrorCode, InvalidJSONError, JsonableError from zerver.lib.types import Validator, ViewFuncT -from django.http import HttpRequest, HttpResponse - -from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union, cast, overload -from typing_extensions import Literal class RequestConfusingParmsError(JsonableError): code = ErrorCode.REQUEST_CONFUSING_VAR diff --git a/zerver/lib/response.py b/zerver/lib/response.py index 52e1576401..a9e5516a76 100644 --- a/zerver/lib/response.py +++ b/zerver/lib/response.py @@ -1,10 +1,12 @@ +from typing import Any, Dict, List, Optional + +import ujson from django.http import HttpResponse, HttpResponseNotAllowed from django.utils.translation import ugettext as _ -import ujson -from typing import Optional, Any, Dict, List from zerver.lib.exceptions import JsonableError + class HttpResponseUnauthorized(HttpResponse): status_code = 401 diff --git a/zerver/lib/rest.py b/zerver/lib/rest.py index 3954b4aa05..1430ac21df 100644 --- a/zerver/lib/rest.py +++ b/zerver/lib/rest.py @@ -1,16 +1,20 @@ from functools import wraps from typing import Any, Callable, Dict -from django.utils.module_loading import import_string +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.utils.cache import add_never_cache_headers +from django.utils.module_loading import import_string from django.views.decorators.csrf import csrf_exempt, csrf_protect -from zerver.decorator import authenticated_json_view, authenticated_rest_api_view, \ - process_as_post, authenticated_uploads_api_view, \ - ReturnT +from zerver.decorator import ( + ReturnT, + authenticated_json_view, + authenticated_rest_api_view, + authenticated_uploads_api_view, + process_as_post, +) from zerver.lib.response import json_method_not_allowed, json_unauthorized -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect -from django.conf import settings METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH') FLAGS = ('override_api_url_scheme') diff --git a/zerver/lib/retention.py b/zerver/lib/retention.py index f23a03319c..8320c56ac7 100644 --- a/zerver/lib/retention.py +++ b/zerver/lib/retention.py @@ -1,20 +1,30 @@ +import logging from datetime import timedelta +from typing import Any, Dict, List, Optional, Tuple from django.conf import settings from django.db import connection, transaction from django.db.models import Model from django.utils.timezone import now as timezone_now -from psycopg2.sql import Composable, Identifier, Literal, SQL +from psycopg2.sql import SQL, Composable, Identifier, Literal from zerver.lib.logging_util import log_to_file -from zerver.models import (Message, UserMessage, ArchivedUserMessage, Realm, - Attachment, ArchivedAttachment, Reaction, ArchivedReaction, - SubMessage, ArchivedSubMessage, Recipient, Stream, ArchiveTransaction, - get_user_including_cross_realm) - -from typing import Any, Dict, List, Optional, Tuple - -import logging +from zerver.models import ( + ArchivedAttachment, + ArchivedReaction, + ArchivedSubMessage, + ArchivedUserMessage, + ArchiveTransaction, + Attachment, + Message, + Reaction, + Realm, + Recipient, + Stream, + SubMessage, + UserMessage, + get_user_including_cross_realm, +) logger = logging.getLogger('zulip.retention') log_to_file(logger, settings.RETENTION_LOG_PATH) diff --git a/zerver/lib/send_email.py b/zerver/lib/send_email.py index 955786c9f2..ee9a523ed9 100644 --- a/zerver/lib/send_email.py +++ b/zerver/lib/send_email.py @@ -1,30 +1,26 @@ +import datetime +import hashlib +import logging +import os +from email.parser import Parser +from email.policy import default +from email.utils import formataddr, parseaddr +from typing import Any, Dict, List, Mapping, Optional, Tuple + +import ujson from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.core.management import CommandError from django.template import loader +from django.template.exceptions import TemplateDoesNotExist from django.utils.timezone import now as timezone_now from django.utils.translation import override as override_language from django.utils.translation import ugettext as _ -from django.template.exceptions import TemplateDoesNotExist -from scripts.setup.inline_email_css import inline_template -from zerver.models import ScheduledEmail, get_user_profile_by_id, \ - EMAIL_TYPES, Realm, UserProfile - -import datetime -from email.utils import parseaddr, formataddr -from email.parser import Parser -from email.policy import default - -import logging -import ujson -import hashlib - -import os -from typing import Any, Dict, List, Mapping, Optional, Tuple - -from zerver.lib.logging_util import log_to_file from confirmation.models import generate_key +from scripts.setup.inline_email_css import inline_template +from zerver.lib.logging_util import log_to_file +from zerver.models import EMAIL_TYPES, Realm, ScheduledEmail, UserProfile, get_user_profile_by_id ## Logging setup ## diff --git a/zerver/lib/server_initialization.py b/zerver/lib/server_initialization.py index cf74b46cb2..cdf0185760 100644 --- a/zerver/lib/server_initialization.py +++ b/zerver/lib/server_initialization.py @@ -1,10 +1,10 @@ +from typing import Iterable, Optional, Tuple + from django.conf import settings from zerver.lib.bulk_create import bulk_create_users -from zerver.models import Realm, UserProfile, email_to_username, get_client, \ - get_system_bot +from zerver.models import Realm, UserProfile, email_to_username, get_client, get_system_bot -from typing import Iterable, Optional, Tuple def server_initialized() -> bool: return Realm.objects.exists() diff --git a/zerver/lib/sessions.py b/zerver/lib/sessions.py index d61a4cbecd..63422aadcb 100644 --- a/zerver/lib/sessions.py +++ b/zerver/lib/sessions.py @@ -1,15 +1,15 @@ import logging - from datetime import timedelta +from importlib import import_module +from typing import Any, List, Mapping, Optional + from django.conf import settings from django.contrib.auth import SESSION_KEY, get_user_model from django.contrib.sessions.models import Session from django.utils.timezone import now as timezone_now -from importlib import import_module -from typing import Any, List, Mapping, Optional -from zerver.models import Realm, UserProfile, get_user_profile_by_id from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.models import Realm, UserProfile, get_user_profile_by_id session_engine = import_module(settings.SESSION_ENGINE) diff --git a/zerver/lib/soft_deactivation.py b/zerver/lib/soft_deactivation.py index bd0e2d1fbe..f6f2aa45e3 100644 --- a/zerver/lib/soft_deactivation.py +++ b/zerver/lib/soft_deactivation.py @@ -1,16 +1,24 @@ # Documented in https://zulip.readthedocs.io/en/latest/subsystems/sending-messages.html#soft-deactivation - -from zerver.lib.logging_util import log_to_file -from collections import defaultdict import logging +from collections import defaultdict +from typing import Any, DefaultDict, Dict, List, Optional, Union + +from django.conf import settings from django.db import transaction from django.db.models import Max -from django.conf import settings from django.utils.timezone import now as timezone_now -from typing import DefaultDict, Dict, List, Optional, Union, Any -from zerver.models import UserProfile, UserMessage, RealmAuditLog, \ - Subscription, Message, Recipient, UserActivity, Realm +from zerver.lib.logging_util import log_to_file +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Recipient, + Subscription, + UserActivity, + UserMessage, + UserProfile, +) logger = logging.getLogger("zulip.soft_deactivation") log_to_file(logger, settings.SOFT_DEACTIVATION_LOG_PATH) diff --git a/zerver/lib/sqlalchemy_utils.py b/zerver/lib/sqlalchemy_utils.py index 1770f70c09..c6314bbf8f 100644 --- a/zerver/lib/sqlalchemy_utils.py +++ b/zerver/lib/sqlalchemy_utils.py @@ -1,9 +1,10 @@ -from typing import Optional, Any - -from django.db import connection -from zerver.lib.db import TimeTrackingConnection +from typing import Any, Optional import sqlalchemy +from django.db import connection + +from zerver.lib.db import TimeTrackingConnection + # This is a Pool that doesn't close connections. Therefore it can be used with # existing Django database connections. diff --git a/zerver/lib/statistics.py b/zerver/lib/statistics.py index 72d4f47356..7b41508e2d 100644 --- a/zerver/lib/statistics.py +++ b/zerver/lib/statistics.py @@ -1,7 +1,8 @@ -from zerver.models import UserProfile, UserActivityInterval - from datetime import datetime, timedelta +from zerver.models import UserActivityInterval, UserProfile + + # Return the amount of Zulip usage for this user between the two # given dates def seconds_usage_between(user_profile: UserProfile, begin: datetime, end: datetime) -> timedelta: diff --git a/zerver/lib/stream_recipient.py b/zerver/lib/stream_recipient.py index d25cab5cb3..7d96d7b90b 100644 --- a/zerver/lib/stream_recipient.py +++ b/zerver/lib/stream_recipient.py @@ -1,9 +1,11 @@ -from typing import (Dict, List) +from typing import Dict, List from django.db import connection from psycopg2.sql import SQL + from zerver.models import Recipient + class StreamRecipientMap: ''' This class maps stream_id -> recipient_id and vice versa. diff --git a/zerver/lib/stream_subscription.py b/zerver/lib/stream_subscription.py index dc9b6d2c0b..2185083fbd 100644 --- a/zerver/lib/stream_subscription.py +++ b/zerver/lib/stream_subscription.py @@ -1,12 +1,9 @@ from typing import Any, Dict, List, Tuple from django.db.models.query import QuerySet -from zerver.models import ( - Recipient, - Stream, - Subscription, - UserProfile, -) + +from zerver.models import Recipient, Stream, Subscription, UserProfile + def get_active_subscriptions_for_stream_id(stream_id: int) -> QuerySet: # TODO: Change return type to QuerySet[Subscription] diff --git a/zerver/lib/stream_topic.py b/zerver/lib/stream_topic.py index d3bacf1a1e..e5fd1d4c64 100644 --- a/zerver/lib/stream_topic.py +++ b/zerver/lib/stream_topic.py @@ -1,13 +1,10 @@ from typing import Set + from django.db.models.query import QuerySet -from zerver.lib.stream_subscription import ( - get_active_subscriptions_for_stream_id, -) +from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id +from zerver.models import MutedTopic -from zerver.models import ( - MutedTopic, -) class StreamTopicTarget: ''' diff --git a/zerver/lib/streams.py b/zerver/lib/streams.py index 3f7158ddf9..cf96a5d9c5 100644 --- a/zerver/lib/streams.py +++ b/zerver/lib/streams.py @@ -1,19 +1,27 @@ -from typing import Any, Iterable, List, Mapping, Set, Tuple, Optional, Union +from typing import Any, Iterable, List, Mapping, Optional, Set, Tuple, Union -from django.utils.translation import ugettext as _ from django.conf import settings +from django.db.models.query import QuerySet +from django.utils.translation import ugettext as _ +from zerver.lib.bugdown import convert as bugdown_convert from zerver.lib.request import JsonableError from zerver.models import ( - UserProfile, Stream, Subscription, Realm, Recipient, get_stream, - bulk_get_streams, get_realm_stream, DefaultStreamGroup, get_stream_by_id_in_realm, - is_cross_realm_bot_email, active_non_guest_user_ids, + DefaultStreamGroup, + Realm, + Recipient, + Stream, + Subscription, + UserProfile, + active_non_guest_user_ids, + bulk_get_streams, + get_realm_stream, + get_stream, + get_stream_by_id_in_realm, + is_cross_realm_bot_email, ) -from zerver.lib.bugdown import convert as bugdown_convert from zerver.tornado.event_queue import send_event -from django.db.models.query import QuerySet - def get_default_value_for_history_public_to_subscribers( realm: Realm, diff --git a/zerver/lib/subdomains.py b/zerver/lib/subdomains.py index 77800dc1aa..824c95937d 100644 --- a/zerver/lib/subdomains.py +++ b/zerver/lib/subdomains.py @@ -1,10 +1,12 @@ -from django.conf import settings -from django.http import HttpRequest import re from typing import Optional +from django.conf import settings +from django.http import HttpRequest + from zerver.models import Realm, UserProfile + def get_subdomain(request: HttpRequest) -> str: # The HTTP spec allows, but doesn't require, a client to omit the diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index d96ad05dcb..de686cd6de 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -1,59 +1,49 @@ +import base64 +import os +import re +import shutil +import tempfile +import urllib from contextlib import contextmanager from email.utils import parseaddr -from fakeldap import MockLDAP -from typing import (cast, Any, Dict, Iterable, - Iterator, List, Optional, - Tuple, Union, Set) +from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union, cast +from unittest import mock +import ujson from django.apps import apps -from django.db.migrations.state import StateApps -from django.urls import resolve from django.conf import settings -from django.test import TestCase -from django.test.client import ( - BOUNDARY, MULTIPART_CONTENT, encode_multipart, -) -from django.test.testcases import SerializeMixin -from django.http import HttpResponse -from django.db.migrations.executor import MigrationExecutor from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.db.migrations.state import StateApps from django.db.utils import IntegrityError -from django.http import HttpRequest +from django.http import HttpRequest, HttpResponse +from django.test import TestCase +from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart +from django.test.testcases import SerializeMixin +from django.urls import resolve from django.utils import translation - +from fakeldap import MockLDAP from two_factor.models import PhoneDevice -from zerver.lib.initial_password import initial_password -from zerver.lib.users import get_api_key -from zerver.lib.sessions import get_session_dict_user -from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers +from zerver.decorator import do_two_factor_login from zerver.lib.actions import ( - check_send_message, bulk_add_subscriptions, + bulk_add_subscriptions, bulk_remove_subscriptions, - check_send_stream_message, gather_subscriptions, + check_send_message, + check_send_stream_message, + gather_subscriptions, ) +from zerver.lib.initial_password import initial_password +from zerver.lib.sessions import get_session_dict_user +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.streams import ( create_stream_if_needed, get_default_value_for_history_public_to_subscribers, ) -from zerver.lib.stream_subscription import ( - get_stream_subscriptions_for_user, -) - -from zerver.lib.test_helpers import ( - instrument_url, find_key_by_email, -) - +from zerver.lib.test_helpers import find_key_by_email, instrument_url +from zerver.lib.users import get_api_key +from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers from zerver.models import ( - clear_supported_auth_backends_cache, - flush_per_request_caches, - get_stream, - get_client, - get_display_recipient, - get_user, - get_user_by_delivery_email, - get_realm, - get_system_bot, Client, Message, Realm, @@ -61,20 +51,20 @@ from zerver.models import ( Stream, Subscription, UserProfile, + clear_supported_auth_backends_cache, + flush_per_request_caches, + get_client, + get_display_recipient, + get_realm, get_realm_stream, + get_stream, + get_system_bot, + get_user, + get_user_by_delivery_email, ) -from zilencer.models import get_remote_server_by_uuid -from zerver.decorator import do_two_factor_login from zerver.tornado.event_queue import clear_client_event_queues_for_testing +from zilencer.models import get_remote_server_by_uuid -import base64 -from unittest import mock -import os -import re -import ujson -import urllib -import shutil -import tempfile class UploadSerializeMixin(SerializeMixin): """ diff --git a/zerver/lib/test_fixtures.py b/zerver/lib/test_fixtures.py index b038dc627f..8d918aa0c5 100644 --- a/zerver/lib/test_fixtures.py +++ b/zerver/lib/test_fixtures.py @@ -1,27 +1,29 @@ +import glob import json import os import re +import shutil import subprocess import sys -from typing import Any, List, Set +import time from importlib import import_module from io import StringIO -import glob -import time -import shutil +from typing import Any, List, Set -from django.db import connections, DEFAULT_DB_ALIAS, ProgrammingError, \ - connection -from django.db.utils import OperationalError from django.apps import apps from django.conf import settings from django.core.management import call_command +from django.db import DEFAULT_DB_ALIAS, ProgrammingError, connection, connections +from django.db.utils import OperationalError from django.utils.module_loading import module_has_submodule sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from scripts.lib.zulip_tools import ( - get_dev_uuid_var_path, run, TEMPLATE_DATABASE_DIR, - is_digest_obsolete, write_new_digest, + TEMPLATE_DATABASE_DIR, + get_dev_uuid_var_path, + is_digest_obsolete, + run, + write_new_digest, ) UUID_VAR_DIR = get_dev_uuid_var_path() diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index 3f0bcb302a..d1d7c9ef12 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -1,57 +1,68 @@ from contextlib import contextmanager from typing import ( - Any, Callable, Dict, Generator, Iterable, Iterator, List, Mapping, - Optional, Tuple, Union, IO, TypeVar, TYPE_CHECKING, + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union, ) -from django.urls import URLResolver -from django.conf import settings -from django.test import override_settings -from django.http import HttpResponse, HttpResponseRedirect -from django.db.migrations.state import StateApps import boto3 from boto3.resources.base import ServiceResource +from django.conf import settings +from django.db.migrations.state import StateApps +from django.http import HttpResponse, HttpResponseRedirect +from django.test import override_settings +from django.urls import URLResolver import zerver.lib.upload +from zerver.lib import cache from zerver.lib.actions import do_set_realm_property -from zerver.lib.upload import S3UploadBackend, LocalUploadBackend from zerver.lib.avatar import avatar_url from zerver.lib.cache import get_cache_backend from zerver.lib.db import Params, ParamsT, Query, TimeTrackingCursor -from zerver.lib import cache -from zerver.tornado import event_queue -from zerver.tornado.handlers import allocate_handler_id -from zerver.worker import queue_processors from zerver.lib.integrations import WEBHOOK_INTEGRATIONS - +from zerver.lib.upload import LocalUploadBackend, S3UploadBackend from zerver.models import ( - get_realm, - get_stream, Client, Message, Realm, Subscription, UserMessage, UserProfile, + get_realm, + get_stream, ) - -from zproject.backends import ExternalAuthResult, ExternalAuthDataDict +from zerver.tornado import event_queue +from zerver.tornado.handlers import allocate_handler_id +from zerver.worker import queue_processors +from zproject.backends import ExternalAuthDataDict, ExternalAuthResult if TYPE_CHECKING: # Avoid an import cycle; we only need these for type annotations. from zerver.lib.test_classes import ZulipTestCase, MigrationsTestCase import collections -from unittest import mock import os import re import sys import time -import ujson -from moto import mock_s3 +from unittest import mock import fakeldap import ldap +import ujson +from moto import mock_s3 + class MockLDAP(fakeldap.MockLDAP): class LDAPError(ldap.LDAPError): diff --git a/zerver/lib/test_runner.py b/zerver/lib/test_runner.py index e01590441a..faa2527876 100644 --- a/zerver/lib/test_runner.py +++ b/zerver/lib/test_runner.py @@ -1,38 +1,33 @@ -from functools import partial +import os import random +import shutil import sys - -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, \ - Type, TypeVar, Union +import time +import unittest +from functools import partial +from multiprocessing.sharedctypes import Synchronized +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union from unittest import loader, runner from unittest.result import TestResult from django.conf import settings -from django.db import connections, ProgrammingError -from django.urls.resolvers import URLPattern +from django.db import ProgrammingError, connections from django.test import TestCase from django.test import runner as django_runner from django.test.runner import DiscoverRunner from django.test.signals import template_rendered +from django.urls.resolvers import URLPattern +from scripts.lib.zulip_tools import ( + TEMPLATE_DATABASE_DIR, + get_dev_uuid_var_path, + get_or_create_dev_uuid_var_path, +) from zerver.lib import test_helpers from zerver.lib.cache import bounce_key_prefix_for_testing from zerver.lib.rate_limiter import bounce_redis_key_prefix_for_testing from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import ( - write_instrumentation_reports, - append_instrumentation_data, -) - -import os -import time -import unittest -import shutil - -from multiprocessing.sharedctypes import Synchronized - -from scripts.lib.zulip_tools import get_dev_uuid_var_path, TEMPLATE_DATABASE_DIR, \ - get_or_create_dev_uuid_var_path +from zerver.lib.test_helpers import append_instrumentation_data, write_instrumentation_reports # We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py index 429f2d6d7a..a53f2d4d16 100644 --- a/zerver/lib/tex.py +++ b/zerver/lib/tex.py @@ -1,10 +1,13 @@ import logging import os import subprocess -from django.conf import settings from typing import Optional + +from django.conf import settings + from zerver.lib.storage import static_path + def render_tex(tex: str, is_inline: bool=True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX diff --git a/zerver/lib/thumbnail.py b/zerver/lib/thumbnail.py index 7897a9d5fc..115589a208 100644 --- a/zerver/lib/thumbnail.py +++ b/zerver/lib/thumbnail.py @@ -4,6 +4,7 @@ import os import sys import urllib from urllib.parse import urljoin + from django.conf import settings from django.utils.http import is_safe_url from libthumbor import CryptoURL @@ -11,10 +12,9 @@ from libthumbor import CryptoURL ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from zthumbor.loaders.helpers import ( - THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE, -) from zerver.lib.camo import get_camo_url +from zthumbor.loaders.helpers import THUMBOR_EXTERNAL_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_S3_TYPE + def is_thumbor_enabled() -> bool: return settings.THUMBOR_URL != '' diff --git a/zerver/lib/timeout.py b/zerver/lib/timeout.py index 84418df695..043afe4c9a 100644 --- a/zerver/lib/timeout.py +++ b/zerver/lib/timeout.py @@ -1,11 +1,10 @@ +import ctypes +import sys +import threading +import time from types import TracebackType from typing import Any, Callable, Optional, Tuple, Type, TypeVar -import sys -import time -import ctypes -import threading - # Based on https://code.activestate.com/recipes/483752/ class TimeoutExpired(Exception): diff --git a/zerver/lib/timestamp.py b/zerver/lib/timestamp.py index d344bef6f5..dcbc3dee49 100644 --- a/zerver/lib/timestamp.py +++ b/zerver/lib/timestamp.py @@ -1,5 +1,6 @@ -import datetime import calendar +import datetime + class TimezoneNotUTCException(Exception): pass diff --git a/zerver/lib/timezone.py b/zerver/lib/timezone.py index 949a985682..ac88347c6f 100644 --- a/zerver/lib/timezone.py +++ b/zerver/lib/timezone.py @@ -1,7 +1,8 @@ -from typing import List, Dict, Union, Any +import datetime +from typing import Any, Dict, List, Union import pytz -import datetime + def get_all_timezones() -> List[str]: return sorted(pytz.all_timezones) diff --git a/zerver/lib/topic.py b/zerver/lib/topic.py index b5098f3739..55fe2b504c 100644 --- a/zerver/lib/topic.py +++ b/zerver/lib/topic.py @@ -1,22 +1,11 @@ -from django.db import connection -from django.db.models.query import QuerySet, Q +from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.sql import ( - column, - literal, - func, -) +from django.db import connection +from django.db.models.query import Q, QuerySet +from sqlalchemy.sql import column, func, literal from zerver.lib.request import REQ -from zerver.models import ( - Message, - Recipient, - Stream, - UserMessage, - UserProfile, -) - -from typing import Any, Dict, List, Optional, Tuple +from zerver.models import Message, Recipient, Stream, UserMessage, UserProfile # Only use these constants for events. ORIG_TOPIC = "orig_subject" diff --git a/zerver/lib/topic_mutes.py b/zerver/lib/topic_mutes.py index 359dbcf0d9..f1a30df323 100644 --- a/zerver/lib/topic_mutes.py +++ b/zerver/lib/topic_mutes.py @@ -1,24 +1,13 @@ -from typing import Any, Callable, Dict, List, Optional, Union import datetime - -from zerver.lib.topic import ( - topic_match_sa, -) -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import ( - get_stream, - MutedTopic, - UserProfile, -) -from sqlalchemy.sql import ( - and_, - column, - not_, - or_, - Selectable, -) +from typing import Any, Callable, Dict, List, Optional, Union from django.utils.timezone import now as timezone_now +from sqlalchemy.sql import Selectable, and_, column, not_, or_ + +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.topic import topic_match_sa +from zerver.models import MutedTopic, UserProfile, get_stream + def get_topic_mutes(user_profile: UserProfile) -> List[List[Union[str, float]]]: rows = MutedTopic.objects.filter( diff --git a/zerver/lib/transfer.py b/zerver/lib/transfer.py index cf07727993..5656240b61 100644 --- a/zerver/lib/transfer.py +++ b/zerver/lib/transfer.py @@ -1,14 +1,14 @@ -import os import logging +import os +from mimetypes import guess_type from django.conf import settings from django.db import connection -from mimetypes import guess_type -from zerver.models import UserProfile, Attachment, RealmEmoji from zerver.lib.avatar_hash import user_avatar_path -from zerver.lib.upload import S3UploadBackend, upload_image_to_s3 from zerver.lib.parallel import run_parallel +from zerver.lib.upload import S3UploadBackend, upload_image_to_s3 +from zerver.models import Attachment, RealmEmoji, UserProfile s3backend = S3UploadBackend() diff --git a/zerver/lib/type_debug.py b/zerver/lib/type_debug.py index 35a07fbc45..fed24fc977 100644 --- a/zerver/lib/type_debug.py +++ b/zerver/lib/type_debug.py @@ -1,7 +1,7 @@ -import sys import functools +import sys +from typing import IO, Any, Callable, Mapping, Sequence, TypeVar -from typing import Any, Callable, IO, Mapping, Sequence, TypeVar def get_mapping_type_str(x: Mapping[Any, Any]) -> str: container_type = type(x).__name__ diff --git a/zerver/lib/types.py b/zerver/lib/types.py index 04eb073207..ab274eeecd 100644 --- a/zerver/lib/types.py +++ b/zerver/lib/types.py @@ -1,6 +1,7 @@ -from typing import TypeVar, Callable, Optional, List, Dict, Union, Tuple, Any -from typing_extensions import TypedDict +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union + from django.http import HttpResponse +from typing_extensions import TypedDict ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse]) diff --git a/zerver/lib/unminify.py b/zerver/lib/unminify.py index d6b41b720f..3c19ea7a8d 100644 --- a/zerver/lib/unminify.py +++ b/zerver/lib/unminify.py @@ -1,9 +1,9 @@ -import re import os -import sourcemap - +import re from typing import Dict, List +import sourcemap + from zerver.lib.pysa import mark_sanitized diff --git a/zerver/lib/upload.py b/zerver/lib/upload.py index 70f9db9ea6..65aa0278db 100644 --- a/zerver/lib/upload.py +++ b/zerver/lib/upload.py @@ -1,45 +1,45 @@ -from typing import Any, Optional, Tuple - -from datetime import timedelta - -from django.utils.translation import ugettext as _ -from django.conf import settings -from django.core.files import File -from django.core.signing import TimestampSigner, BadSignature -from django.http import HttpRequest -from django.urls import reverse -from jinja2 import Markup as mark_safe +import base64 +import binascii +import io +import logging +import os +import random +import re +import shutil +import sys import unicodedata - -from zerver.lib.avatar_hash import user_avatar_path -from zerver.lib.exceptions import JsonableError, ErrorCode -from zerver.lib.utils import generate_random_token +import urllib +from datetime import timedelta +from mimetypes import guess_extension, guess_type +from typing import Any, Optional, Tuple import boto3 import botocore -from botocore.client import Config from boto3.resources.base import ServiceResource from boto3.session import Session - -from mimetypes import guess_type, guess_extension - -from zerver.models import get_user_profile_by_id -from zerver.models import Attachment -from zerver.models import Realm, RealmEmoji, UserProfile, Message - -import urllib -import base64 -import binascii -import os -import re -from PIL import Image, ImageOps, ExifTags -from PIL.Image import DecompressionBombError +from botocore.client import Config +from django.conf import settings +from django.core.files import File +from django.core.signing import BadSignature, TimestampSigner +from django.http import HttpRequest +from django.urls import reverse +from django.utils.translation import ugettext as _ +from jinja2 import Markup as mark_safe +from PIL import ExifTags, Image, ImageOps from PIL.GifImagePlugin import GifImageFile -import io -import random -import logging -import shutil -import sys +from PIL.Image import DecompressionBombError + +from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.exceptions import ErrorCode, JsonableError +from zerver.lib.utils import generate_random_token +from zerver.models import ( + Attachment, + Message, + Realm, + RealmEmoji, + UserProfile, + get_user_profile_by_id, +) DEFAULT_AVATAR_SIZE = 100 MEDIUM_AVATAR_SIZE = 500 diff --git a/zerver/lib/url_encoding.py b/zerver/lib/url_encoding.py index 3bd12dd276..6c8058b90f 100644 --- a/zerver/lib/url_encoding.py +++ b/zerver/lib/url_encoding.py @@ -5,6 +5,7 @@ from zerver.lib.pysa import mark_sanitized from zerver.lib.topic import get_topic_from_message_info from zerver.models import Realm, Stream, UserProfile + def hash_util_encode(string: str) -> str: # Do the same encoding operation as hash_util.encodeHashComponent on the # frontend. diff --git a/zerver/lib/url_preview/oembed.py b/zerver/lib/url_preview/oembed.py index 16831f50a2..b9451806b1 100644 --- a/zerver/lib/url_preview/oembed.py +++ b/zerver/lib/url_preview/oembed.py @@ -1,6 +1,8 @@ -from typing import Optional, Dict, Any -from pyoembed import oEmbed, PyOembedException import json +from typing import Any, Dict, Optional + +from pyoembed import PyOembedException, oEmbed + def get_oembed_data(url: str, maxwidth: Optional[int]=640, diff --git a/zerver/lib/url_preview/parsers/__init__.py b/zerver/lib/url_preview/parsers/__init__.py index 74168a2c3b..b812d115a0 100644 --- a/zerver/lib/url_preview/parsers/__init__.py +++ b/zerver/lib/url_preview/parsers/__init__.py @@ -1,4 +1,4 @@ -from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser from zerver.lib.url_preview.parsers.generic import GenericParser +from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser __all__ = ['OpenGraphParser', 'GenericParser'] diff --git a/zerver/lib/url_preview/parsers/base.py b/zerver/lib/url_preview/parsers/base.py index 25ed9e8ff6..b173221302 100644 --- a/zerver/lib/url_preview/parsers/base.py +++ b/zerver/lib/url_preview/parsers/base.py @@ -1,5 +1,6 @@ from typing import Any + class BaseParser: def __init__(self, html_source: str) -> None: # We import BeautifulSoup here, because it's not used by most diff --git a/zerver/lib/url_preview/parsers/generic.py b/zerver/lib/url_preview/parsers/generic.py index a57db1664e..d938114f36 100644 --- a/zerver/lib/url_preview/parsers/generic.py +++ b/zerver/lib/url_preview/parsers/generic.py @@ -1,4 +1,5 @@ from typing import Dict, Optional + from zerver.lib.url_preview.parsers.base import BaseParser diff --git a/zerver/lib/url_preview/parsers/open_graph.py b/zerver/lib/url_preview/parsers/open_graph.py index 75eee049e2..8c31546e01 100644 --- a/zerver/lib/url_preview/parsers/open_graph.py +++ b/zerver/lib/url_preview/parsers/open_graph.py @@ -1,4 +1,5 @@ from typing import Dict + from .base import BaseParser diff --git a/zerver/lib/url_preview/preview.py b/zerver/lib/url_preview/preview.py index f69d574043..f764744100 100644 --- a/zerver/lib/url_preview/preview.py +++ b/zerver/lib/url_preview/preview.py @@ -1,17 +1,17 @@ import re -import requests +from typing import Any, Callable, Dict, Optional +from typing.re import Match +import magic +import requests from django.conf import settings from django.utils.encoding import smart_text -import magic -from typing import Any, Optional, Dict, Callable -from typing.re import Match from version import ZULIP_VERSION from zerver.lib.cache import cache_with_key, get_cache_with_key, preview_url_cache_key from zerver.lib.pysa import mark_sanitized from zerver.lib.url_preview.oembed import get_oembed_data -from zerver.lib.url_preview.parsers import OpenGraphParser, GenericParser +from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser # FIXME: Should we use a database cache or a memcached in production? What if # opengraph data is changed for a site? diff --git a/zerver/lib/user_groups.py b/zerver/lib/user_groups.py index 9aec110b3c..5ce7209dc4 100644 --- a/zerver/lib/user_groups.py +++ b/zerver/lib/user_groups.py @@ -1,8 +1,11 @@ +from typing import Any, Dict, List + from django.db import transaction from django.utils.translation import ugettext as _ + from zerver.lib.exceptions import JsonableError -from zerver.models import UserProfile, Realm, UserGroupMembership, UserGroup -from typing import Dict, List, Any +from zerver.models import Realm, UserGroup, UserGroupMembership, UserProfile + def access_user_group_by_id(user_group_id: int, user_profile: UserProfile) -> UserGroup: try: diff --git a/zerver/lib/user_status.py b/zerver/lib/user_status.py index fa2cca500b..da4aa90f39 100644 --- a/zerver/lib/user_status.py +++ b/zerver/lib/user_status.py @@ -1,11 +1,10 @@ +from typing import Any, Dict, Optional + from django.db.models import Q from django.utils.timezone import now as timezone_now -from zerver.models import ( - UserStatus, -) +from zerver.models import UserStatus -from typing import Any, Dict, Optional def get_user_info_dict(realm_id: int) -> Dict[int, Dict[str, Any]]: rows = UserStatus.objects.filter( diff --git a/zerver/lib/users.py b/zerver/lib/users.py index 61fe52318e..d92024540c 100644 --- a/zerver/lib/users.py +++ b/zerver/lib/users.py @@ -1,25 +1,34 @@ -from typing import Any, Dict, List, Optional, Tuple, Union, cast - import re import unicodedata from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple, Union, cast from django.conf import settings from django.db.models.query import QuerySet from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ - -from zerver.lib.cache import generic_bulk_cached_fetch, user_profile_cache_key_id, \ - user_profile_by_id_cache_key, realm_user_dict_fields -from zerver.lib.request import JsonableError -from zerver.lib.avatar import avatar_url, get_avatar_field -from zerver.lib.exceptions import OrganizationAdministratorRequired -from zerver.models import UserProfile, Service, Realm, \ - get_user_profile_by_id_in_realm, CustomProfileFieldValue, \ - get_realm_user_dicts, CustomProfileField - from zulip_bots.custom_exceptions import ConfigValidationError +from zerver.lib.avatar import avatar_url, get_avatar_field +from zerver.lib.cache import ( + generic_bulk_cached_fetch, + realm_user_dict_fields, + user_profile_by_id_cache_key, + user_profile_cache_key_id, +) +from zerver.lib.exceptions import OrganizationAdministratorRequired +from zerver.lib.request import JsonableError +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + Realm, + Service, + UserProfile, + get_realm_user_dicts, + get_user_profile_by_id_in_realm, +) + + def check_full_name(full_name_raw: str) -> str: full_name = full_name_raw.strip() if len(full_name) > UserProfile.MAX_NAME_LENGTH: diff --git a/zerver/lib/utils.py b/zerver/lib/utils.py index 4b1d35ef8e..59926fc8a8 100644 --- a/zerver/lib/utils.py +++ b/zerver/lib/utils.py @@ -1,4 +1,3 @@ -from typing import Any, Callable, List, Optional, Sequence, TypeVar, Iterable, Set, Tuple import base64 import hashlib import heapq @@ -6,8 +5,9 @@ import itertools import os import re import string -from time import sleep from itertools import zip_longest +from time import sleep +from typing import Any, Callable, Iterable, List, Optional, Sequence, Set, Tuple, TypeVar from django.conf import settings diff --git a/zerver/lib/validator.py b/zerver/lib/validator.py index 00920b0706..59f4e24b61 100644 --- a/zerver/lib/validator.py +++ b/zerver/lib/validator.py @@ -26,17 +26,17 @@ To extend this concept, it's simply a matter of writing your own validator for any particular type of object. ''' import re +from datetime import datetime +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union, cast + import ujson -from django.utils.translation import ugettext as _ from django.conf import settings from django.core.exceptions import ValidationError -from django.core.validators import validate_email, URLValidator -from typing import Any, Dict, Iterable, Optional, Tuple, cast, List, Callable, TypeVar, \ - Set, Union +from django.core.validators import URLValidator, validate_email +from django.utils.translation import ugettext as _ -from datetime import datetime from zerver.lib.request import JsonableError -from zerver.lib.types import Validator, ProfileFieldData +from zerver.lib.types import ProfileFieldData, Validator FuncT = Callable[..., Any] TypeStructure = TypeVar("TypeStructure") diff --git a/zerver/lib/webhooks/common.py b/zerver/lib/webhooks/common.py index 442d5e406f..3edd2ca0d2 100644 --- a/zerver/lib/webhooks/common.py +++ b/zerver/lib/webhooks/common.py @@ -1,19 +1,25 @@ import importlib +from typing import Any, Callable, Dict, Optional, Union from urllib.parse import unquote from django.http import HttpRequest from django.utils.translation import ugettext as _ -from typing import Optional, Dict, Union, Any, Callable -from zerver.lib.actions import check_send_stream_message, \ - check_send_private_message, send_rate_limited_pm_notification_to_bot_owner -from zerver.lib.exceptions import StreamDoesNotExistError, JsonableError, \ - ErrorCode, UnexpectedWebhookEventType +from zerver.lib.actions import ( + check_send_private_message, + check_send_stream_message, + send_rate_limited_pm_notification_to_bot_owner, +) +from zerver.lib.exceptions import ( + ErrorCode, + JsonableError, + StreamDoesNotExistError, + UnexpectedWebhookEventType, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.send_email import FromAddress from zerver.models import UserProfile - MISSING_EVENT_HEADER_MESSAGE = """ Hi there! Your bot {bot_name} just sent an HTTP request to {request_path} that is missing the HTTP {header_name} header. Because this header is how diff --git a/zerver/lib/webhooks/git.py b/zerver/lib/webhooks/git.py index b57f33b843..84b548e888 100644 --- a/zerver/lib/webhooks/git.py +++ b/zerver/lib/webhooks/git.py @@ -1,6 +1,7 @@ import string -from typing import Optional, Any, Dict, List, Tuple from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple + TOPIC_WITH_BRANCH_TEMPLATE = '{repo} / {branch}' TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE = '{repo} / {type} #{id} {title}' TOPIC_WITH_RELEASE_TEMPLATE = '{repo} / {tag} {title}' diff --git a/zerver/lib/widget.py b/zerver/lib/widget.py index 319242c43b..fa3ad9cc3d 100644 --- a/zerver/lib/widget.py +++ b/zerver/lib/widget.py @@ -1,7 +1,6 @@ -from typing import MutableMapping, Any, Optional, Tuple - -import re import json +import re +from typing import Any, MutableMapping, Optional, Tuple from zerver.models import SubMessage diff --git a/zerver/lib/zcommand.py b/zerver/lib/zcommand.py index d32253368f..f6909b347d 100644 --- a/zerver/lib/zcommand.py +++ b/zerver/lib/zcommand.py @@ -1,9 +1,11 @@ from typing import Any, Dict + from django.utils.translation import ugettext as _ -from zerver.models import UserProfile from zerver.lib.actions import do_set_user_display_setting from zerver.lib.exceptions import JsonableError +from zerver.models import UserProfile + def process_zcommands(content: str, user_profile: UserProfile) -> Dict[str, Any]: def change_mode_setting(command: str, switch_command: str, diff --git a/zerver/lib/zephyr.py b/zerver/lib/zephyr.py index c39b2f52a7..116ae1beae 100644 --- a/zerver/lib/zephyr.py +++ b/zerver/lib/zephyr.py @@ -1,7 +1,9 @@ import re import traceback + import DNS + def compute_mit_user_fullname(email: str) -> str: try: # Input is either e.g. username@mit.edu or user|CROSSREALM.INVALID@mit.edu diff --git a/zerver/logging_handlers.py b/zerver/logging_handlers.py index 5b6539a0f0..19cb2bb18b 100644 --- a/zerver/logging_handlers.py +++ b/zerver/logging_handlers.py @@ -1,8 +1,7 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - import logging -import platform import os +import platform import subprocess import traceback from typing import Any, Dict, Optional @@ -11,9 +10,10 @@ from django.conf import settings from django.http import HttpRequest from django.views.debug import get_exception_reporter_filter +from version import ZULIP_VERSION from zerver.lib.logging_util import find_log_caller_module from zerver.lib.queue import queue_json_publish -from version import ZULIP_VERSION + def try_git_describe() -> Optional[str]: try: # nocoverage diff --git a/zerver/management/commands/check_redis.py b/zerver/management/commands/check_redis.py index d523de3f86..bc4cd91d1a 100644 --- a/zerver/management/commands/check_redis.py +++ b/zerver/management/commands/check_redis.py @@ -3,8 +3,7 @@ import time from typing import Any, Callable, Optional from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.lib.rate_limiter import RateLimitedUser, client from zerver.models import get_user_profile_by_id diff --git a/zerver/management/commands/convert_gitter_data.py b/zerver/management/commands/convert_gitter_data.py index 62e88568c5..e18bd893b9 100644 --- a/zerver/management/commands/convert_gitter_data.py +++ b/zerver/management/commands/convert_gitter_data.py @@ -3,8 +3,7 @@ import os import tempfile from typing import Any -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.gitter import do_convert_data diff --git a/zerver/management/commands/convert_hipchat_data.py b/zerver/management/commands/convert_hipchat_data.py index 9061c44766..e3f4e77cfe 100644 --- a/zerver/management/commands/convert_hipchat_data.py +++ b/zerver/management/commands/convert_hipchat_data.py @@ -21,8 +21,7 @@ spec: exporting-from-hipchat-server-or-data-center-for-data-portability-950821555.html ''' -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.hipchat import do_convert_data diff --git a/zerver/management/commands/convert_mattermost_data.py b/zerver/management/commands/convert_mattermost_data.py index d0e1df3bdd..292a8b9c3d 100644 --- a/zerver/management/commands/convert_mattermost_data.py +++ b/zerver/management/commands/convert_mattermost_data.py @@ -14,8 +14,7 @@ Test out the realm: go to browser and use your dev url ''' -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.mattermost import do_convert_data diff --git a/zerver/management/commands/convert_slack_data.py b/zerver/management/commands/convert_slack_data.py index 7a75f0bbed..5168146d14 100644 --- a/zerver/management/commands/convert_slack_data.py +++ b/zerver/management/commands/convert_slack_data.py @@ -3,8 +3,7 @@ import os import tempfile from typing import Any -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.slack import do_convert_data diff --git a/zerver/management/commands/generate_invite_links.py b/zerver/management/commands/generate_invite_links.py index d6fdbe05fa..cfb832469d 100644 --- a/zerver/management/commands/generate_invite_links.py +++ b/zerver/management/commands/generate_invite_links.py @@ -4,9 +4,9 @@ from typing import Any from django.core.management.base import CommandError from confirmation.models import Confirmation, create_confirmation_link +from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.management import ZulipBaseCommand from zerver.models import DomainNotAllowedForRealmError, PreregistrationUser -from zerver.lib.email_validation import email_allowed_for_realm class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/import.py b/zerver/management/commands/import.py index 1f3406ecd8..92282013c8 100644 --- a/zerver/management/commands/import.py +++ b/zerver/management/commands/import.py @@ -5,8 +5,7 @@ from typing import Any from django.conf import settings from django.core.management import call_command -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.forms import check_subdomain_available from zerver.lib.import_realm import do_import_realm, do_import_system_bots diff --git a/zerver/management/commands/knight.py b/zerver/management/commands/knight.py index 51194ae72b..61532c355a 100644 --- a/zerver/management/commands/knight.py +++ b/zerver/management/commands/knight.py @@ -3,7 +3,7 @@ from typing import Any from django.core.management.base import CommandError -from zerver.lib.actions import do_change_user_role, do_change_is_api_super_user +from zerver.lib.actions import do_change_is_api_super_user, do_change_user_role from zerver.lib.management import ZulipBaseCommand from zerver.models import UserProfile diff --git a/zerver/management/commands/logout_all_users.py b/zerver/management/commands/logout_all_users.py index 6108643a58..b41b1c867c 100644 --- a/zerver/management/commands/logout_all_users.py +++ b/zerver/management/commands/logout_all_users.py @@ -2,8 +2,11 @@ from argparse import ArgumentParser from typing import Any from zerver.lib.management import ZulipBaseCommand -from zerver.lib.sessions import delete_all_deactivated_user_sessions, \ - delete_all_user_sessions, delete_realm_user_sessions +from zerver.lib.sessions import ( + delete_all_deactivated_user_sessions, + delete_all_user_sessions, + delete_realm_user_sessions, +) class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/makemessages.py b/zerver/management/commands/makemessages.py index 80f6220fd7..69ac464471 100644 --- a/zerver/management/commands/makemessages.py +++ b/zerver/management/commands/makemessages.py @@ -29,8 +29,8 @@ https://stackoverflow.com/questions/2090717 """ import glob -import json import itertools +import json import os import re from argparse import ArgumentParser diff --git a/zerver/management/commands/merge_streams.py b/zerver/management/commands/merge_streams.py index 517446da72..5d56f9e5a1 100644 --- a/zerver/management/commands/merge_streams.py +++ b/zerver/management/commands/merge_streams.py @@ -1,8 +1,11 @@ from argparse import ArgumentParser from typing import Any, List -from zerver.lib.actions import bulk_add_subscriptions, \ - bulk_remove_subscriptions, do_deactivate_stream +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_deactivate_stream, +) from zerver.lib.cache import cache_delete_many, to_dict_cache_key_id from zerver.lib.management import ZulipBaseCommand from zerver.models import Message, Subscription, get_stream diff --git a/zerver/management/commands/restore_messages.py b/zerver/management/commands/restore_messages.py index 9570658d01..fa0b2a8b20 100644 --- a/zerver/management/commands/restore_messages.py +++ b/zerver/management/commands/restore_messages.py @@ -3,8 +3,11 @@ from typing import Any from django.core.management.base import CommandParser from zerver.lib.management import ZulipBaseCommand -from zerver.lib.retention import restore_all_data_from_archive, \ - restore_data_from_archive, restore_data_from_archive_by_realm +from zerver.lib.retention import ( + restore_all_data_from_archive, + restore_data_from_archive, + restore_data_from_archive_by_realm, +) from zerver.models import ArchiveTransaction diff --git a/zerver/management/commands/runtornado.py b/zerver/management/commands/runtornado.py index 02e27760c0..de3adfe1ad 100644 --- a/zerver/management/commands/runtornado.py +++ b/zerver/management/commands/runtornado.py @@ -3,8 +3,7 @@ import sys from typing import Any, Callable from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from tornado import ioloop from tornado.log import app_log @@ -19,11 +18,14 @@ settings.RUNNING_INSIDE_TORNADO = True instrument_tornado_ioloop() from zerver.lib.debug import interactive_debug_listen -from zerver.tornado.application import create_tornado_application, \ - setup_tornado_rabbitmq +from zerver.tornado.application import create_tornado_application, setup_tornado_rabbitmq from zerver.tornado.autoreload import start as zulip_autoreload_start -from zerver.tornado.event_queue import add_client_gc_hook, \ - missedmessage_hook, get_wrapped_process_notification, setup_event_queue +from zerver.tornado.event_queue import ( + add_client_gc_hook, + get_wrapped_process_notification, + missedmessage_hook, + setup_event_queue, +) from zerver.tornado.sharding import notify_tornado_queue_name if settings.USING_RABBITMQ: diff --git a/zerver/management/commands/show_admins.py b/zerver/management/commands/show_admins.py index 7962adaee5..f94479ec6a 100644 --- a/zerver/management/commands/show_admins.py +++ b/zerver/management/commands/show_admins.py @@ -2,8 +2,10 @@ from argparse import ArgumentParser from typing import Any from django.core.management.base import CommandError + from zerver.lib.management import ZulipBaseCommand + class Command(ZulipBaseCommand): help = """Show the owners and administrators in an organization.""" diff --git a/zerver/management/commands/soft_deactivate_users.py b/zerver/management/commands/soft_deactivate_users.py index f379cd9edd..26481ab1ac 100644 --- a/zerver/management/commands/soft_deactivate_users.py +++ b/zerver/management/commands/soft_deactivate_users.py @@ -6,8 +6,12 @@ from django.conf import settings from django.core.management.base import CommandError from zerver.lib.management import ZulipBaseCommand -from zerver.lib.soft_deactivation import do_auto_soft_deactivate_users, \ - do_soft_activate_users, do_soft_deactivate_users, logger +from zerver.lib.soft_deactivation import ( + do_auto_soft_deactivate_users, + do_soft_activate_users, + do_soft_deactivate_users, + logger, +) from zerver.models import Realm, UserProfile diff --git a/zerver/management/commands/transfer_uploads_to_s3.py b/zerver/management/commands/transfer_uploads_to_s3.py index d2c70f9f38..77ade9cf48 100644 --- a/zerver/management/commands/transfer_uploads_to_s3.py +++ b/zerver/management/commands/transfer_uploads_to_s3.py @@ -1,8 +1,7 @@ from typing import Any from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.lib.transfer import transfer_uploads_to_s3 diff --git a/zerver/middleware.py b/zerver/middleware.py index 3ceeb11576..95795b64b3 100644 --- a/zerver/middleware.py +++ b/zerver/middleware.py @@ -2,8 +2,7 @@ import cProfile import logging import time import traceback -from typing import Any, AnyStr, Dict, \ - Iterable, List, MutableMapping, Optional +from typing import Any, AnyStr, Dict, Iterable, List, MutableMapping, Optional from django.conf import settings from django.core.exceptions import DisallowedHost @@ -17,15 +16,15 @@ from django.views.csrf import csrf_failure as html_csrf_failure from zerver.lib.bugdown import get_bugdown_requests, get_bugdown_time from zerver.lib.cache import get_remote_cache_requests, get_remote_cache_time -from zerver.lib.debug import maybe_tracemalloc_listen from zerver.lib.db import reset_queries +from zerver.lib.debug import maybe_tracemalloc_listen from zerver.lib.exceptions import ErrorCode, JsonableError, RateLimited from zerver.lib.html_to_text import get_content_description from zerver.lib.rate_limiter import RateLimitResult from zerver.lib.response import json_error, json_response_from_error from zerver.lib.subdomains import get_subdomain -from zerver.lib.utils import statsd from zerver.lib.types import ViewFuncT +from zerver.lib.utils import statsd from zerver.models import Realm, flush_per_request_caches, get_realm logger = logging.getLogger('zulip.requests') diff --git a/zerver/migrations/0030_realm_org_type.py b/zerver/migrations/0030_realm_org_type.py index 6f32a07e1e..b086b48584 100644 --- a/zerver/migrations/0030_realm_org_type.py +++ b/zerver/migrations/0030_realm_org_type.py @@ -1,6 +1,5 @@ from django.db import migrations, models - CORPORATE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0032_verify_all_medium_avatar_images.py b/zerver/migrations/0032_verify_all_medium_avatar_images.py index f477ca52f7..ceb86eeb5a 100644 --- a/zerver/migrations/0032_verify_all_medium_avatar_images.py +++ b/zerver/migrations/0032_verify_all_medium_avatar_images.py @@ -1,11 +1,11 @@ import hashlib from typing import Text +from unittest.mock import patch from django.conf import settings from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -from unittest.mock import patch from zerver.lib.upload import upload_backend from zerver.lib.utils import make_safe_digest diff --git a/zerver/migrations/0038_realm_change_to_community_defaults.py b/zerver/migrations/0038_realm_change_to_community_defaults.py index ad8c885a03..50f65f00e5 100644 --- a/zerver/migrations/0038_realm_change_to_community_defaults.py +++ b/zerver/migrations/0038_realm_change_to_community_defaults.py @@ -1,6 +1,5 @@ from django.db import migrations, models - COMMUNITY = 2 class Migration(migrations.Migration): diff --git a/zerver/migrations/0060_move_avatars_to_be_uid_based.py b/zerver/migrations/0060_move_avatars_to_be_uid_based.py index 6200d286a7..8d834eb38a 100644 --- a/zerver/migrations/0060_move_avatars_to_be_uid_based.py +++ b/zerver/migrations/0060_move_avatars_to_be_uid_based.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-02-27 17:03 from django.db import migrations + class Migration(migrations.Migration): dependencies = [ diff --git a/zerver/migrations/0064_sync_uploads_filesize_with_db.py b/zerver/migrations/0064_sync_uploads_filesize_with_db.py index 674e98f237..337e1c6a81 100644 --- a/zerver/migrations/0064_sync_uploads_filesize_with_db.py +++ b/zerver/migrations/0064_sync_uploads_filesize_with_db.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-03-18 12:38 from django.db import migrations + class Migration(migrations.Migration): dependencies = [ diff --git a/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py b/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py index 62f513ec91..3d475d118c 100644 --- a/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py +++ b/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-03-09 05:23 from django.db import migrations, models + class Migration(migrations.Migration): dependencies = [ ('zerver', '0076_userprofile_emojiset'), diff --git a/zerver/migrations/0086_realm_alter_default_org_type.py b/zerver/migrations/0086_realm_alter_default_org_type.py index 1dca4623d8..6b5096d180 100644 --- a/zerver/migrations/0086_realm_alter_default_org_type.py +++ b/zerver/migrations/0086_realm_alter_default_org_type.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.2 on 2017-06-26 21:56 from django.db import migrations, models - CORPORATE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0143_realm_bot_creation_policy.py b/zerver/migrations/0143_realm_bot_creation_policy.py index 93414f9522..7b8e7f0faa 100644 --- a/zerver/migrations/0143_realm_bot_creation_policy.py +++ b/zerver/migrations/0143_realm_bot_creation_policy.py @@ -1,10 +1,8 @@ # Generated by Django 1.11.6 on 2018-03-09 18:00 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps - BOT_CREATION_EVERYONE = 1 def set_initial_value_for_bot_creation_policy(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: Realm = apps.get_model("zerver", "Realm") diff --git a/zerver/migrations/0185_realm_plan_type.py b/zerver/migrations/0185_realm_plan_type.py index 3e9935ebcb..c7349655df 100644 --- a/zerver/migrations/0185_realm_plan_type.py +++ b/zerver/migrations/0185_realm_plan_type.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.14 on 2018-08-10 21:36 - from django.db import migrations, models - SELF_HOSTED = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0214_realm_invite_to_stream_policy.py b/zerver/migrations/0214_realm_invite_to_stream_policy.py index 75e2fbbf56..f9d3d85f2a 100644 --- a/zerver/migrations/0214_realm_invite_to_stream_policy.py +++ b/zerver/migrations/0214_realm_invite_to_stream_policy.py @@ -1,10 +1,8 @@ # Generated by Django 1.11.20 on 2019-04-29 05:29 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps - INVITE_TO_STREAM_POLICY_MEMBERS = 1 def handle_waiting_period(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: Realm = apps.get_model('zerver', 'Realm') diff --git a/zerver/migrations/0228_userprofile_demote_inactive_streams.py b/zerver/migrations/0228_userprofile_demote_inactive_streams.py index 30ac8d84fa..bdb779d0b0 100644 --- a/zerver/migrations/0228_userprofile_demote_inactive_streams.py +++ b/zerver/migrations/0228_userprofile_demote_inactive_streams.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.20 on 2019-03-08 19:50 - from django.db import migrations, models - DEMOTE_STREAMS_AUTOMATIC = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0235_userprofile_desktop_icon_count_display.py b/zerver/migrations/0235_userprofile_desktop_icon_count_display.py index 8ff03ae5b1..c62476ff87 100644 --- a/zerver/migrations/0235_userprofile_desktop_icon_count_display.py +++ b/zerver/migrations/0235_userprofile_desktop_icon_count_display.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.20 on 2019-06-29 18:22 - from django.db import migrations, models - DESKTOP_ICON_COUNT_DISPLAY_MESSAGES = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0248_userprofile_role_start.py b/zerver/migrations/0248_userprofile_role_start.py index 0306767867..9408ef52dd 100644 --- a/zerver/migrations/0248_userprofile_role_start.py +++ b/zerver/migrations/0248_userprofile_role_start.py @@ -1,9 +1,9 @@ # Generated by Django 1.11.24 on 2019-10-03 22:27 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps + def update_role(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') # The values at the time of this migration diff --git a/zerver/migrations/0252_realm_user_group_edit_policy.py b/zerver/migrations/0252_realm_user_group_edit_policy.py index 25a1e61ee1..193fc7b1dd 100644 --- a/zerver/migrations/0252_realm_user_group_edit_policy.py +++ b/zerver/migrations/0252_realm_user_group_edit_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.24 on 2019-10-16 22:48 - from django.db import migrations, models - USER_GROUP_EDIT_POLICY_MEMBERS = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0261_realm_private_message_policy.py b/zerver/migrations/0261_realm_private_message_policy.py index 33a056a81e..8688872bcc 100644 --- a/zerver/migrations/0261_realm_private_message_policy.py +++ b/zerver/migrations/0261_realm_private_message_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-08 00:32 - from django.db import migrations, models - PRIVATE_MESSAGE_POLICY_UNLIMITED = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0262_mutedtopic_date_muted.py b/zerver/migrations/0262_mutedtopic_date_muted.py index 22ee32a792..f42dd83e2b 100644 --- a/zerver/migrations/0262_mutedtopic_date_muted.py +++ b/zerver/migrations/0262_mutedtopic_date_muted.py @@ -1,6 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-17 15:26 - import datetime + from django.db import migrations, models diff --git a/zerver/migrations/0263_stream_stream_post_policy.py b/zerver/migrations/0263_stream_stream_post_policy.py index 6cf2945ef9..e619f5c859 100644 --- a/zerver/migrations/0263_stream_stream_post_policy.py +++ b/zerver/migrations/0263_stream_stream_post_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-27 22:03 - from django.db import migrations, models - STREAM_POST_POLICY_EVERYONE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0266_userpresence_realm.py b/zerver/migrations/0266_userpresence_realm.py index 1937e5339c..4d0d7cae9e 100644 --- a/zerver/migrations/0266_userpresence_realm.py +++ b/zerver/migrations/0266_userpresence_realm.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.28 on 2020-02-08 20:19 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py b/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py index 2eeb6fe187..9c28d38376 100644 --- a/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py +++ b/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.28 on 2020-02-08 20:34 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0270_huddle_recipient.py b/zerver/migrations/0270_huddle_recipient.py index 9eac7771d7..3273c63d74 100644 --- a/zerver/migrations/0270_huddle_recipient.py +++ b/zerver/migrations/0270_huddle_recipient.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.10 on 2020-03-15 17:25 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0273_migrate_old_bot_messages.py b/zerver/migrations/0273_migrate_old_bot_messages.py index 7cd7a0e864..9494b09318 100644 --- a/zerver/migrations/0273_migrate_old_bot_messages.py +++ b/zerver/migrations/0273_migrate_old_bot_messages.py @@ -1,9 +1,10 @@ +from typing import Any + from django.conf import settings from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -from typing import Any def fix_messages(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """Conceptually, this migration cleans up the old NEW_USER_BOT and FEEDBACK_BOT diff --git a/zerver/migrations/0276_alertword.py b/zerver/migrations/0276_alertword.py index 75d3eafb3c..4de29acc54 100644 --- a/zerver/migrations/0276_alertword.py +++ b/zerver/migrations/0276_alertword.py @@ -1,6 +1,6 @@ +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/zerver/migrations/0277_migrate_alert_word.py b/zerver/migrations/0277_migrate_alert_word.py index c4f755bde7..cca8edc8ef 100644 --- a/zerver/migrations/0277_migrate_alert_word.py +++ b/zerver/migrations/0277_migrate_alert_word.py @@ -1,8 +1,10 @@ +from typing import Dict, List + +import ujson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -import ujson -from typing import Dict, List + def move_to_seperate_table(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') diff --git a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py index 8e7463ffda..bbd56d5e0e 100644 --- a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py +++ b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py @@ -1,12 +1,13 @@ # Generated by Django 2.2.12 on 2020-05-16 18:34 from typing import Any, Dict +import ujson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models import Count from django.utils.timezone import now as timezone_now -import ujson + def set_realm_admins_as_realm_owners(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') diff --git a/zerver/models.py b/zerver/models.py index d10c1d3ac4..6e3fd7969c 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -1,52 +1,92 @@ -from typing import Any, DefaultDict, Dict, List, Set, Tuple, TypeVar, \ - Union, Optional, Sequence, AbstractSet, Callable, Iterable - -from django.db import models -from django.db.models.query import QuerySet -from django.db.models import Manager, Q, Sum, CASCADE -from django.conf import settings -from django.contrib.auth.models import AbstractBaseUser, UserManager, \ - PermissionsMixin -import django.contrib.auth -from django.core.exceptions import ValidationError -from django.core.validators import URLValidator, MinLengthValidator, \ - RegexValidator, validate_email -from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \ - user_profile_by_api_key_cache_key, active_non_guest_user_ids_cache_key, \ - user_profile_by_id_cache_key, user_profile_by_email_cache_key, \ - user_profile_cache_key, generic_bulk_cached_fetch, cache_set, flush_stream, \ - cache_delete, active_user_ids_cache_key, \ - get_stream_cache_key, realm_user_dicts_cache_key, \ - bot_dicts_in_realm_cache_key, realm_user_dict_fields, \ - bot_dict_fields, flush_message, flush_submessage, bot_profile_cache_key, \ - flush_used_upload_space_cache, get_realm_used_upload_space_cache_key, \ - realm_alert_words_cache_key, realm_alert_words_automaton_cache_key -from zerver.lib.utils import make_safe_digest, generate_random_token -from django.db import transaction -from django.utils.timezone import now as timezone_now -from zerver.lib.timestamp import datetime_to_timestamp -from django.db.models.signals import post_save, post_delete -from django.utils.translation import ugettext_lazy as _ -from zerver.lib import cache -from zerver.lib.pysa import mark_sanitized -from zerver.lib.validator import check_int, \ - check_short_string, check_long_string, validate_choice_field, check_date, \ - check_url, check_list -from zerver.lib.types import Validator, ExtendedValidator, \ - ProfileDataElement, ProfileData, RealmUserValidator, \ - ExtendedFieldElement, UserFieldElement, FieldElement, \ - DisplayRecipientT -from zerver.lib.exceptions import JsonableError -from django.contrib.postgres.fields import JSONField - -from bitfield import BitField -from bitfield.types import BitHandler -from collections import defaultdict -from datetime import timedelta +import datetime import re import sre_constants import time -import datetime +from collections import defaultdict +from datetime import timedelta +from typing import ( + AbstractSet, + Any, + Callable, + DefaultDict, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + TypeVar, + Union, +) + +import django.contrib.auth +from bitfield import BitField +from bitfield.types import BitHandler +from django.conf import settings +from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin, UserManager +from django.contrib.postgres.fields import JSONField +from django.core.exceptions import ValidationError +from django.core.validators import MinLengthValidator, RegexValidator, URLValidator, validate_email +from django.db import models, transaction +from django.db.models import CASCADE, Manager, Q, Sum +from django.db.models.query import QuerySet +from django.db.models.signals import post_delete, post_save +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext_lazy as _ + +from zerver.lib import cache +from zerver.lib.cache import ( + active_non_guest_user_ids_cache_key, + active_user_ids_cache_key, + bot_dict_fields, + bot_dicts_in_realm_cache_key, + bot_profile_cache_key, + cache_delete, + cache_set, + cache_with_key, + flush_message, + flush_realm, + flush_stream, + flush_submessage, + flush_used_upload_space_cache, + flush_user_profile, + generic_bulk_cached_fetch, + get_realm_used_upload_space_cache_key, + get_stream_cache_key, + realm_alert_words_automaton_cache_key, + realm_alert_words_cache_key, + realm_user_dict_fields, + realm_user_dicts_cache_key, + user_profile_by_api_key_cache_key, + user_profile_by_email_cache_key, + user_profile_by_id_cache_key, + user_profile_cache_key, +) +from zerver.lib.exceptions import JsonableError +from zerver.lib.pysa import mark_sanitized +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.types import ( + DisplayRecipientT, + ExtendedFieldElement, + ExtendedValidator, + FieldElement, + ProfileData, + ProfileDataElement, + RealmUserValidator, + UserFieldElement, + Validator, +) +from zerver.lib.utils import generate_random_token, make_safe_digest +from zerver.lib.validator import ( + check_date, + check_int, + check_list, + check_long_string, + check_short_string, + check_url, + validate_choice_field, +) MAX_TOPIC_NAME_LENGTH = 60 MAX_MESSAGE_LENGTH = 10000 diff --git a/zerver/openapi/curl_param_value_generators.py b/zerver/openapi/curl_param_value_generators.py index 59cb4d68e0..b6ebadd67a 100644 --- a/zerver/openapi/curl_param_value_generators.py +++ b/zerver/openapi/curl_param_value_generators.py @@ -1,13 +1,12 @@ -from typing import Dict, Any, Callable, Set, List, Optional, Tuple - from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Set, Tuple from django.utils.timezone import now as timezone_now -from zerver.models import Client, Message, UserPresence, UserGroup, get_realm -from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.actions import do_add_reaction, do_add_realm_filter, update_user_presence from zerver.lib.events import do_events_register -from zerver.lib.actions import update_user_presence, do_add_realm_filter, do_add_reaction +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Client, Message, UserGroup, UserPresence, get_realm GENERATOR_FUNCTIONS: Dict[str, Callable[..., Dict[Any, Any]]] = dict() REGISTERED_GENERATOR_FUNCTIONS: Set[str] = set() diff --git a/zerver/openapi/javascript_examples.py b/zerver/openapi/javascript_examples.py index cbb41a1457..47318cda2b 100644 --- a/zerver/openapi/javascript_examples.py +++ b/zerver/openapi/javascript_examples.py @@ -1,10 +1,12 @@ -import os import json +import os import subprocess from zulip import Client + from zerver.openapi.openapi import validate_against_openapi_schema + def test_js_bindings(client: Client) -> None: os.environ['ZULIP_USERNAME'] = client.email os.environ['ZULIP_API_KEY'] = client.api_key diff --git a/zerver/openapi/markdown_extension.py b/zerver/openapi/markdown_extension.py index 1a6c5d946b..4dc85f88b7 100644 --- a/zerver/openapi/markdown_extension.py +++ b/zerver/openapi/markdown_extension.py @@ -1,16 +1,15 @@ -import re -import json import inspect +import json +import re +from typing import Any, Dict, List, Optional, Pattern, Tuple +import markdown from django.conf import settings - from markdown.extensions import Extension from markdown.preprocessors import Preprocessor -from typing import Any, Dict, Optional, List, Tuple, Pattern -import markdown import zerver.openapi.python_examples -from zerver.openapi.openapi import get_openapi_fixture, openapi_spec, get_openapi_description +from zerver.openapi.openapi import get_openapi_description, get_openapi_fixture, openapi_spec MACRO_REGEXP = re.compile( r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}') diff --git a/zerver/openapi/python_examples.py b/zerver/openapi/python_examples.py index abcd4e864b..885ff60594 100644 --- a/zerver/openapi/python_examples.py +++ b/zerver/openapi/python_examples.py @@ -1,17 +1,15 @@ -from typing import Dict, Any, Optional, Iterable, Callable, Set, List - import json import os import sys from functools import wraps - -from zerver.lib import mdiff -from zerver.openapi.openapi import validate_against_openapi_schema - -from zerver.models import get_realm, get_user +from typing import Any, Callable, Dict, Iterable, List, Optional, Set from zulip import Client +from zerver.lib import mdiff +from zerver.models import get_realm, get_user +from zerver.openapi.openapi import validate_against_openapi_schema + ZULIP_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) TEST_FUNCTIONS: Dict[str, Callable[..., None]] = dict() diff --git a/zerver/openapi/test_curl_examples.py b/zerver/openapi/test_curl_examples.py index 9d1ec803ce..f2e6a268c3 100644 --- a/zerver/openapi/test_curl_examples.py +++ b/zerver/openapi/test_curl_examples.py @@ -1,14 +1,19 @@ import glob +import html import json import shlex import subprocess -import markdown -import html +import markdown from zulip import Client -from zerver.openapi import markdown_extension + from zerver.models import get_realm -from zerver.openapi.curl_param_value_generators import REGISTERED_GENERATOR_FUNCTIONS, CALLED_GENERATOR_FUNCTIONS +from zerver.openapi import markdown_extension +from zerver.openapi.curl_param_value_generators import ( + CALLED_GENERATOR_FUNCTIONS, + REGISTERED_GENERATOR_FUNCTIONS, +) + def test_generated_curl_examples_for_success(client: Client) -> None: authentication_line = f"{client.email}:{client.api_key}" diff --git a/zerver/signals.py b/zerver/signals.py index ddcf2c080b..86a40dfe14 100644 --- a/zerver/signals.py +++ b/zerver/signals.py @@ -3,8 +3,7 @@ from typing import Any, Optional from django.conf import settings from django.contrib.auth.signals import user_logged_in, user_logged_out from django.dispatch import receiver -from django.utils.timezone import \ - get_current_timezone_name as timezone_get_current_timezone_name +from django.utils.timezone import get_current_timezone_name as timezone_get_current_timezone_name from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ @@ -12,8 +11,8 @@ from confirmation.models import one_click_unsubscribe_link from zerver.lib.actions import do_set_zoom_token from zerver.lib.queue import queue_json_publish from zerver.lib.send_email import FromAddress -from zerver.models import UserProfile from zerver.lib.timezone import get_timezone +from zerver.models import UserProfile JUST_CREATED_THRESHOLD = 60 diff --git a/zerver/templatetags/app_filters.py b/zerver/templatetags/app_filters.py index ad5437159c..2c23d10f17 100644 --- a/zerver/templatetags/app_filters.py +++ b/zerver/templatetags/app_filters.py @@ -9,17 +9,17 @@ from django.template import Library, engines from django.utils.safestring import mark_safe from jinja2.exceptions import TemplateNotFound -import zerver.lib.bugdown.fenced_code import zerver.lib.bugdown.api_arguments_table_generator import zerver.lib.bugdown.api_return_values_table_generator -import zerver.openapi.markdown_extension +import zerver.lib.bugdown.fenced_code +import zerver.lib.bugdown.help_emoticon_translations_table +import zerver.lib.bugdown.help_relative_links +import zerver.lib.bugdown.help_settings_links +import zerver.lib.bugdown.include import zerver.lib.bugdown.nested_code_blocks import zerver.lib.bugdown.tabbed_sections -import zerver.lib.bugdown.help_settings_links -import zerver.lib.bugdown.help_relative_links -import zerver.lib.bugdown.help_emoticon_translations_table -import zerver.lib.bugdown.include -from zerver.lib.cache import ignore_unhashable_lru_cache, dict_to_items_tuple, items_tuple_to_dict +import zerver.openapi.markdown_extension +from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict register = Library() diff --git a/zerver/tests/test_alert_words.py b/zerver/tests/test_alert_words.py index 1ca4d37534..4dd5583f36 100644 --- a/zerver/tests/test_alert_words.py +++ b/zerver/tests/test_alert_words.py @@ -1,28 +1,12 @@ -from zerver.lib.alert_words import ( - alert_words_in_realm, - user_alert_words, -) - -from zerver.lib.actions import ( - do_add_alert_words, - do_remove_alert_words, -) - -from zerver.lib.test_helpers import ( - most_recent_message, - most_recent_usermessage, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - UserProfile, -) - import ujson +from zerver.lib.actions import do_add_alert_words, do_remove_alert_words +from zerver.lib.alert_words import alert_words_in_realm, user_alert_words +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, most_recent_usermessage +from zerver.models import UserProfile + + class AlertWordTests(ZulipTestCase): interesting_alert_word_list = ['alert', 'multi-word word', '☃'] diff --git a/zerver/tests/test_archive.py b/zerver/tests/test_archive.py index c42ecbe322..24a333ac94 100644 --- a/zerver/tests/test_archive.py +++ b/zerver/tests/test_archive.py @@ -1,10 +1,15 @@ from django.http import HttpResponse + +from zerver.lib.actions import ( + do_change_stream_web_public, + do_deactivate_stream, + get_web_public_streams, + get_web_public_subs, +) from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.actions import do_change_stream_web_public -from zerver.lib.actions import get_web_public_streams, get_web_public_subs, \ - do_deactivate_stream from zerver.models import get_realm + class GlobalPublicStreamTest(ZulipTestCase): def test_non_existant_stream_id(self) -> None: # Here we use a relatively big number as stream id assuming such an id diff --git a/zerver/tests/test_attachments.py b/zerver/tests/test_attachments.py index 5fefdb9471..7b8ec3b832 100644 --- a/zerver/tests/test_attachments.py +++ b/zerver/tests/test_attachments.py @@ -1,6 +1,5 @@ -from unittest import mock - from typing import Any +from unittest import mock from zerver.lib.attachments import user_attachments from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index 34ed744067..8718eda40a 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -1,20 +1,32 @@ +from datetime import timedelta +from typing import Any, Dict + +import ujson +from django.contrib.auth.password_validation import validate_password from django.utils.timezone import now as timezone_now -from zerver.lib.actions import do_create_user, do_deactivate_user, \ - do_activate_user, do_reactivate_user, do_change_password, \ - do_change_user_delivery_email, do_change_avatar_fields, do_change_bot_owner, \ - do_regenerate_api_key, do_change_tos_version, \ - bulk_add_subscriptions, bulk_remove_subscriptions, get_streams_traffic, \ - do_change_user_role, do_deactivate_realm, do_reactivate_realm -from zerver.lib.test_classes import ZulipTestCase -from zerver.models import RealmAuditLog, get_client, get_realm, UserProfile from analytics.models import StreamCount +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_activate_user, + do_change_avatar_fields, + do_change_bot_owner, + do_change_password, + do_change_tos_version, + do_change_user_delivery_email, + do_change_user_role, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, + do_regenerate_api_key, + get_streams_traffic, +) +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import RealmAuditLog, UserProfile, get_client, get_realm -from datetime import timedelta -from django.contrib.auth.password_validation import validate_password - -from typing import Any, Dict -import ujson class TestRealmAuditLog(ZulipTestCase): def check_role_count_schema(self, role_counts: Dict[str, Any]) -> None: diff --git a/zerver/tests/test_auth_backends.py b/zerver/tests/test_auth_backends.py index 992cbc27e7..6621cba44b 100644 --- a/zerver/tests/test_auth_backends.py +++ b/zerver/tests/test_auth_backends.py @@ -1,29 +1,39 @@ +import base64 +import copy +import datetime +import json +import re +import time +import urllib +from typing import Any, Callable, Dict, List, Optional, Tuple +from unittest import mock + +import jwt +import ldap +import requests +import responses +import ujson from bs4 import BeautifulSoup from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.contrib.auth import authenticate from django.core import mail -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.test import override_settings -from django_auth_ldap.backend import LDAPSearch, _LDAPUser from django.test.client import RequestFactory -from django.utils.timezone import now as timezone_now -from typing import Any, Callable, Dict, List, Optional, Tuple from django.urls import reverse +from django.utils.timezone import now as timezone_now +from django_auth_ldap.backend import LDAPSearch, _LDAPUser +from onelogin.saml2.auth import OneLogin_Saml2_Auth +from onelogin.saml2.response import OneLogin_Saml2_Response +from social_core.exceptions import AuthFailed, AuthStateForbidden +from social_django.storage import BaseDjangoStorage +from social_django.strategy import DjangoStrategy -import responses - -import ldap -import jwt -from unittest import mock -import re -import datetime -import time -import requests - +from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.actions import ( - do_create_user, do_create_realm, + do_create_user, do_deactivate_realm, do_deactivate_user, do_invite_users, @@ -35,57 +45,92 @@ from zerver.lib.actions import ( from zerver.lib.avatar import avatar_url from zerver.lib.avatar_hash import user_avatar_path from zerver.lib.dev_ldap_directory import generate_dev_ldap_dir -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_is_valid, get_existing_user_errors +from zerver.lib.email_validation import ( + get_existing_user_errors, + get_realm_email_validator, + validate_email_is_valid, +) from zerver.lib.exceptions import RateLimited +from zerver.lib.initial_password import initial_password from zerver.lib.mobile_auth_otp import otp_decrypt_api_key -from zerver.lib.validator import validate_login_email, \ - check_bool, check_dict_only, check_list, check_string, check_int, Validator from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule from zerver.lib.request import JsonableError from zerver.lib.storage import static_path -from zerver.lib.upload import resize_avatar, MEDIUM_AVATAR_SIZE +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + create_s3_buckets, + get_test_image_file, + load_subdomain_token, + use_s3_backend, +) +from zerver.lib.upload import MEDIUM_AVATAR_SIZE, resize_avatar from zerver.lib.users import get_all_api_keys from zerver.lib.utils import generate_random_token -from zerver.lib.initial_password import initial_password -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.validator import ( + Validator, + check_bool, + check_dict_only, + check_int, + check_list, + check_string, + validate_login_email, +) +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + MultiuseInvite, + PasswordTooWeakError, + PreregistrationUser, + Realm, + RealmDomain, + UserProfile, + clear_supported_auth_backends_cache, + email_to_username, + get_realm, + get_user_by_delivery_email, ) -from zerver.models import \ - get_realm, email_to_username, CustomProfileField, CustomProfileFieldValue, \ - UserProfile, PreregistrationUser, Realm, RealmDomain, MultiuseInvite, \ - clear_supported_auth_backends_cache, PasswordTooWeakError, get_user_by_delivery_email from zerver.signals import JUST_CREATED_THRESHOLD - -from confirmation.models import Confirmation, create_confirmation_link - -from zproject.backends import ZulipDummyBackend, EmailAuthBackend, AppleAuthBackend, \ - GoogleAuthBackend, ZulipRemoteUserBackend, ZulipLDAPAuthBackend, \ - ZulipLDAPUserPopulator, DevAuthBackend, GitHubAuthBackend, GitLabAuthBackend, ZulipAuthMixin, \ - dev_auth_enabled, password_auth_enabled, github_auth_enabled, gitlab_auth_enabled, \ - apple_auth_enabled, google_auth_enabled, require_email_format_usernames, AUTH_BACKEND_NAME_MAP, \ - ZulipLDAPConfigurationError, ZulipLDAPExceptionNoMatchingLDAPUser, ZulipLDAPExceptionOutsideDomain, \ - ZulipLDAPException, query_ldap, sync_user_from_ldap, SocialAuthMixin, \ - PopulateUserLDAPError, SAMLAuthBackend, saml_auth_enabled, email_belongs_to_ldap, \ - get_external_method_dicts, AzureADAuthBackend, check_password_strength, \ - ZulipLDAPUser, RateLimitedAuthenticationByUsername, ExternalAuthResult, \ - ExternalAuthDataDict - from zerver.views.auth import maybe_send_to_registration +from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + AppleAuthBackend, + AzureADAuthBackend, + DevAuthBackend, + EmailAuthBackend, + ExternalAuthDataDict, + ExternalAuthResult, + GitHubAuthBackend, + GitLabAuthBackend, + GoogleAuthBackend, + PopulateUserLDAPError, + RateLimitedAuthenticationByUsername, + SAMLAuthBackend, + SocialAuthMixin, + ZulipAuthMixin, + ZulipDummyBackend, + ZulipLDAPAuthBackend, + ZulipLDAPConfigurationError, + ZulipLDAPException, + ZulipLDAPExceptionNoMatchingLDAPUser, + ZulipLDAPExceptionOutsideDomain, + ZulipLDAPUser, + ZulipLDAPUserPopulator, + ZulipRemoteUserBackend, + apple_auth_enabled, + check_password_strength, + dev_auth_enabled, + email_belongs_to_ldap, + get_external_method_dicts, + github_auth_enabled, + gitlab_auth_enabled, + google_auth_enabled, + password_auth_enabled, + query_ldap, + require_email_format_usernames, + saml_auth_enabled, + sync_user_from_ldap, +) -from onelogin.saml2.auth import OneLogin_Saml2_Auth -from onelogin.saml2.response import OneLogin_Saml2_Response -from social_core.exceptions import AuthFailed, AuthStateForbidden -from social_django.strategy import DjangoStrategy -from social_django.storage import BaseDjangoStorage - -import base64 -import copy -import json -import urllib -import ujson -from zerver.lib.test_helpers import load_subdomain_token, \ - use_s3_backend, create_s3_buckets, get_test_image_file class AuthBackendTest(ZulipTestCase): def get_username(self, email_to_username: Optional[Callable[[str], str]]=None) -> str: diff --git a/zerver/tests/test_bots.py b/zerver/tests/test_bots.py index a1e1069456..40267ecee7 100644 --- a/zerver/tests/test_bots.py +++ b/zerver/tests/test_bots.py @@ -1,28 +1,38 @@ import filecmp import os -import ujson +from typing import Any, Dict, List, Mapping, Optional +from unittest.mock import MagicMock, patch +import ujson from django.core import mail from django.test import override_settings -from unittest.mock import patch, MagicMock -from typing import Any, Dict, List, Mapping, Optional +from zulip_bots.custom_exceptions import ConfigValidationError -from zerver.lib.actions import do_change_stream_invite_only, do_deactivate_user, \ - do_set_realm_property -from zerver.lib.bot_config import get_bot_config, ConfigError -from zerver.models import get_realm, get_stream, \ - Realm, UserProfile, get_user, get_bot_services, Service, \ - is_cross_realm_bot_email -from zerver.lib.test_classes import ZulipTestCase, UploadSerializeMixin +from zerver.lib.actions import ( + do_change_stream_invite_only, + do_deactivate_user, + do_set_realm_property, +) +from zerver.lib.bot_config import ConfigError, get_bot_config +from zerver.lib.bot_lib import get_bot_handler +from zerver.lib.integrations import EMBEDDED_BOTS, WebhookIntegration +from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase from zerver.lib.test_helpers import ( avatar_disk_path, get_test_image_file, queries_captured, tornado_redirected_to_list, ) -from zerver.lib.integrations import EMBEDDED_BOTS, WebhookIntegration -from zerver.lib.bot_lib import get_bot_handler -from zulip_bots.custom_exceptions import ConfigValidationError +from zerver.models import ( + Realm, + Service, + UserProfile, + get_bot_services, + get_realm, + get_stream, + get_user, + is_cross_realm_bot_email, +) # A test validator diff --git a/zerver/tests/test_bugdown.py b/zerver/tests/test_bugdown.py index f3e7efb2b9..b8311d8259 100644 --- a/zerver/tests/test_bugdown.py +++ b/zerver/tests/test_bugdown.py @@ -1,12 +1,19 @@ +import copy +import os +import re +from typing import Any, Dict, List, Optional, Set, Tuple, cast +from unittest import mock + +import ujson from django.conf import settings from django.test import TestCase, override_settings -from zerver.lib import bugdown +from zerver.lib import bugdown, mdiff from zerver.lib.actions import ( - do_set_user_display_setting, - do_remove_realm_emoji, do_add_alert_words, + do_remove_realm_emoji, do_set_realm_property, + do_set_user_display_setting, ) from zerver.lib.alert_words import get_alert_word_automaton from zerver.lib.create_user import create_user @@ -14,42 +21,30 @@ from zerver.lib.emoji import get_emoji_url from zerver.lib.exceptions import BugdownRenderingException from zerver.lib.mention import possible_mentions, possible_user_group_mentions from zerver.lib.message import render_markdown -from zerver.lib.request import ( - JsonableError, -) -from zerver.lib.user_groups import create_user_group -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.request import JsonableError +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_runner import slow -from zerver.lib import mdiff from zerver.lib.tex import render_tex +from zerver.lib.user_groups import create_user_group from zerver.models import ( - realm_in_local_realm_filters_cache, + MAX_MESSAGE_LENGTH, + Message, + Realm, + RealmEmoji, + RealmFilter, + Stream, + UserGroup, + UserMessage, + UserProfile, flush_per_request_caches, flush_realm_filter, get_client, get_realm, get_stream, realm_filters_for_realm, - MAX_MESSAGE_LENGTH, - Message, - Stream, - Realm, - RealmEmoji, - RealmFilter, - UserMessage, - UserProfile, - UserGroup, + realm_in_local_realm_filters_cache, ) -import copy -from unittest import mock -import os -import ujson -import re - -from typing import cast, Any, Dict, List, Optional, Set, Tuple class FakeMessage: pass diff --git a/zerver/tests/test_cache.py b/zerver/tests/test_cache.py index d9c1a32482..a28455a856 100644 --- a/zerver/tests/test_cache.py +++ b/zerver/tests/test_cache.py @@ -1,16 +1,31 @@ +from typing import Any, Dict, List, Optional +from unittest.mock import Mock, patch + from django.conf import settings -from unittest.mock import Mock, patch -from typing import Any, List, Dict, Optional - from zerver.apps import flush_cache -from zerver.lib.cache import generic_bulk_cached_fetch, user_profile_by_email_cache_key, cache_with_key, \ - validate_cache_key, InvalidCacheKeyException, MEMCACHED_MAX_KEY_LENGTH, get_cache_with_key, \ - NotFoundInCache, cache_set, cache_get, cache_delete, cache_delete_many, cache_get_many, cache_set_many, \ - safe_cache_get_many, safe_cache_set_many +from zerver.lib.cache import ( + MEMCACHED_MAX_KEY_LENGTH, + InvalidCacheKeyException, + NotFoundInCache, + cache_delete, + cache_delete_many, + cache_get, + cache_get_many, + cache_set, + cache_set_many, + cache_with_key, + generic_bulk_cached_fetch, + get_cache_with_key, + safe_cache_get_many, + safe_cache_set_many, + user_profile_by_email_cache_key, + validate_cache_key, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured -from zerver.models import get_system_bot, get_user_profile_by_email, UserProfile +from zerver.models import UserProfile, get_system_bot, get_user_profile_by_email + class AppsTest(ZulipTestCase): def test_cache_gets_flushed(self) -> None: diff --git a/zerver/tests/test_camo.py b/zerver/tests/test_camo.py index 8224047a31..3ef7c1e28a 100644 --- a/zerver/tests/test_camo.py +++ b/zerver/tests/test_camo.py @@ -1,5 +1,6 @@ from zerver.lib.test_classes import ZulipTestCase + class CamoURLTest(ZulipTestCase): def test_legacy_camo_url(self) -> None: # Test with valid hex and url pair diff --git a/zerver/tests/test_compatibility.py b/zerver/tests/test_compatibility.py index 6f21f510d2..0b8101d308 100644 --- a/zerver/tests/test_compatibility.py +++ b/zerver/tests/test_compatibility.py @@ -1,7 +1,7 @@ from unittest import mock from zerver.lib.test_classes import ZulipTestCase -from zerver.views.compatibility import find_mobile_os, version_lt, is_outdated_desktop_app +from zerver.views.compatibility import find_mobile_os, is_outdated_desktop_app, version_lt class VersionTest(ZulipTestCase): diff --git a/zerver/tests/test_create_video_call.py b/zerver/tests/test_create_video_call.py index f4c1bbb238..e393b1da2d 100644 --- a/zerver/tests/test_create_video_call.py +++ b/zerver/tests/test_create_video_call.py @@ -1,4 +1,5 @@ import responses + from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_custom_profile_data.py b/zerver/tests/test_custom_profile_data.py index 6f2cf7d6b8..7889531f4d 100644 --- a/zerver/tests/test_custom_profile_data.py +++ b/zerver/tests/test_custom_profile_data.py @@ -1,17 +1,25 @@ -from typing import Union, List, Dict, Any +from typing import Any, Dict, List, Union +from unittest import mock -from zerver.lib.actions import try_add_realm_custom_profile_field, \ - do_update_user_custom_profile_data_if_changed, do_remove_realm_custom_profile_field, \ - try_reorder_realm_custom_profile_fields -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import queries_captured -from zerver.lib.bugdown import convert as bugdown_convert -from zerver.models import CustomProfileField, \ - custom_profile_fields_for_realm, CustomProfileFieldValue, get_realm -from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS import ujson -from unittest import mock +from zerver.lib.actions import ( + do_remove_realm_custom_profile_field, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_reorder_realm_custom_profile_fields, +) +from zerver.lib.bugdown import convert as bugdown_convert +from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + custom_profile_fields_for_realm, + get_realm, +) + class CustomProfileFieldTestCase(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_decorators.py b/zerver/tests/test_decorators.py index f20da1ea75..bbc82ccc3b 100644 --- a/zerver/tests/test_decorators.py +++ b/zerver/tests/test_decorators.py @@ -1,57 +1,81 @@ import base64 -from unittest import mock -import re import os +import re from collections import defaultdict - from typing import Any, Dict, Iterable, List, Optional, Tuple +from unittest import mock -from django.test import TestCase -from django.http import HttpResponse, HttpRequest +import ujson from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.test import TestCase -from zerver.forms import OurAuthenticationForm -from zerver.lib.actions import do_deactivate_realm, do_deactivate_user, \ - do_reactivate_user, do_reactivate_realm, do_set_realm_property -from zerver.lib.exceptions import JsonableError, InvalidAPIKeyError, InvalidAPIKeyFormatError -from zerver.lib.initial_password import initial_password -from zerver.lib.test_helpers import ( - HostRequestMock, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.response import json_response, json_success -from zerver.lib.users import get_api_key -from zerver.lib.user_agent import parse_user_agent -from zerver.lib.utils import generate_api_key, has_api_key_format -from zerver.lib.request import \ - REQ, has_request_variables, RequestVariableMissingError, \ - RequestVariableConversionError, RequestConfusingParmsError -from zerver.lib.webhooks.common import UnexpectedWebhookEventType from zerver.decorator import ( api_key_only_webhook_view, + authenticate_notify, authenticated_json_view, authenticated_rest_api_view, authenticated_uploads_api_view, - authenticate_notify, cachify, - get_client_name, internal_notify_view, is_local_addr, - rate_limit, validate_api_key, + cachify, + get_client_name, + internal_notify_view, + is_local_addr, + rate_limit, return_success_on_head_request, + validate_api_key, zulip_login_required, ) -from zerver.lib.cache import ignore_unhashable_lru_cache, dict_to_items_tuple, items_tuple_to_dict +from zerver.forms import OurAuthenticationForm +from zerver.lib.actions import ( + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, + do_set_realm_property, +) +from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict +from zerver.lib.exceptions import InvalidAPIKeyError, InvalidAPIKeyFormatError, JsonableError +from zerver.lib.initial_password import initial_password +from zerver.lib.request import ( + REQ, + RequestConfusingParmsError, + RequestVariableConversionError, + RequestVariableMissingError, + has_request_variables, +) +from zerver.lib.response import json_response, json_success +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import HostRequestMock +from zerver.lib.user_agent import parse_user_agent +from zerver.lib.users import get_api_key +from zerver.lib.utils import generate_api_key, has_api_key_format from zerver.lib.validator import ( - check_string, check_dict, check_dict_only, check_bool, check_float, check_int, check_list, Validator, - check_variable_type, equals, check_none_or, check_url, check_short_string, - check_string_fixed_length, check_capped_string, check_color, to_non_negative_int, - check_string_or_int_list, check_string_or_int, check_int_in, check_string_in, + Validator, + check_bool, + check_capped_string, + check_color, + check_dict, + check_dict_only, + check_float, + check_int, + check_int_in, + check_list, + check_none_or, + check_short_string, + check_string, + check_string_fixed_length, + check_string_in, + check_string_or_int, + check_string_or_int_list, + check_url, + check_variable_type, + equals, + to_non_negative_int, to_positive_or_allowed_int, ) -from zerver.models import \ - get_realm, get_user, UserProfile, Realm +from zerver.lib.webhooks.common import UnexpectedWebhookEventType +from zerver.models import Realm, UserProfile, get_realm, get_user -import ujson class DecoratorTestCase(TestCase): def test_get_client_name(self) -> None: diff --git a/zerver/tests/test_digest.py b/zerver/tests/test_digest.py index 6f634d6765..cc3d3fb124 100644 --- a/zerver/tests/test_digest.py +++ b/zerver/tests/test_digest.py @@ -1,20 +1,34 @@ import datetime -from unittest import mock import time from typing import List +from unittest import mock from django.test import override_settings from django.utils.timezone import now as timezone_now from confirmation.models import one_click_unsubscribe_link from zerver.lib.actions import do_create_user -from zerver.lib.digest import gather_new_streams, handle_digest_email, enqueue_emails, \ - exclude_subscription_modified_streams +from zerver.lib.digest import ( + enqueue_emails, + exclude_subscription_modified_streams, + gather_new_streams, + handle_digest_email, +) from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured -from zerver.models import get_client, get_realm, flush_per_request_caches, \ - Realm, Message, UserActivity, UserProfile, RealmAuditLog, get_stream +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + UserActivity, + UserProfile, + flush_per_request_caches, + get_client, + get_realm, + get_stream, +) + class TestDigestEmailMessages(ZulipTestCase): diff --git a/zerver/tests/test_docs.py b/zerver/tests/test_docs.py index ee86e4041f..e6c401510f 100644 --- a/zerver/tests/test_docs.py +++ b/zerver/tests/test_docs.py @@ -1,12 +1,12 @@ import os -import ujson +from typing import Any, Dict, List from unittest import mock from urllib.parse import urlsplit +import ujson from django.conf import settings -from django.test import TestCase, override_settings from django.http import HttpResponse -from typing import Any, Dict, List +from django.test import TestCase, override_settings from zerver.lib.integrations import INTEGRATIONS from zerver.lib.test_classes import ZulipTestCase @@ -14,9 +14,8 @@ from zerver.lib.test_helpers import HostRequestMock from zerver.lib.test_runner import slow from zerver.lib.utils import split_by from zerver.models import Realm, get_realm -from zerver.views.documentation import ( - add_api_uri_context, -) +from zerver.views.documentation import add_api_uri_context + class DocPageTest(ZulipTestCase): def get_doc(self, url: str, subdomain: str) -> HttpResponse: diff --git a/zerver/tests/test_email_change.py b/zerver/tests/test_email_change.py index 6b8685774c..651dee7ed9 100644 --- a/zerver/tests/test_email_change.py +++ b/zerver/tests/test_email_change.py @@ -1,17 +1,22 @@ import datetime -from email.utils import parseaddr import re +from email.utils import parseaddr from django.core import mail from django.utils.timezone import now -from confirmation.models import Confirmation, generate_key, confirmation_url -from zerver.lib.actions import do_start_email_change_process, do_set_realm_property -from zerver.lib.test_classes import ( - ZulipTestCase, +from confirmation.models import Confirmation, confirmation_url, generate_key +from zerver.lib.actions import do_set_realm_property, do_start_email_change_process +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import ( + EmailChangeStatus, + Realm, + UserProfile, + get_realm, + get_user, + get_user_by_delivery_email, + get_user_profile_by_id, ) -from zerver.models import get_user_by_delivery_email, EmailChangeStatus, get_realm, \ - Realm, UserProfile, get_user, get_user_profile_by_id class EmailChangeTestCase(ZulipTestCase): diff --git a/zerver/tests/test_email_log.py b/zerver/tests/test_email_log.py index a720edf058..b21f9fcb66 100644 --- a/zerver/tests/test_email_log.py +++ b/zerver/tests/test_email_log.py @@ -1,10 +1,12 @@ import os from unittest import mock + from django.conf import settings from zerver.lib.test_classes import ZulipTestCase from zproject.email_backends import get_forward_address + class EmailLogTest(ZulipTestCase): def test_generate_and_clear_email_log(self) -> None: with self.settings(EMAIL_BACKEND='zproject.email_backends.EmailLogBackEnd'), \ diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py index c4205ff9bb..d516807b69 100644 --- a/zerver/tests/test_email_mirror.py +++ b/zerver/tests/test_email_mirror.py @@ -1,63 +1,50 @@ +import os import subprocess +from email import message_from_string +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from typing import Any, Callable, Dict, Mapping, Optional +from unittest import mock +import ujson +from django.conf import settings from django.http import HttpResponse -from zerver.lib.test_helpers import ( - most_recent_message, - most_recent_usermessage, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - get_display_recipient, - get_realm, - get_stream, - get_system_bot, - MissedMessageEmailAddress, - Recipient, - UserProfile, -) - -from zerver.lib.actions import ensure_stream, do_deactivate_realm, do_deactivate_user - +from zerver.lib.actions import do_deactivate_realm, do_deactivate_user, ensure_stream from zerver.lib.email_mirror import ( - process_message, - process_missed_message, + ZulipEmailForwardError, create_missed_message_address, + filter_footer, get_missed_message_token_from_address, - strip_from_subject, is_forwarded, is_missed_message_address, - filter_footer, log_and_report, + process_message, + process_missed_message, redact_email_address, - ZulipEmailForwardError, + strip_from_subject, ) - from zerver.lib.email_mirror_helpers import ( decode_email_address, encode_email_address, get_email_gateway_message_string_from_address, ) - from zerver.lib.email_notifications import convert_html_to_markdown from zerver.lib.send_email import FromAddress +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, most_recent_usermessage +from zerver.models import ( + MissedMessageEmailAddress, + Recipient, + UserProfile, + get_display_recipient, + get_realm, + get_stream, + get_system_bot, +) from zerver.worker.queue_processors import MirrorWorker -from email import message_from_string -from email.mime.text import MIMEText -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart - -import ujson -from unittest import mock -import os -from django.conf import settings - -from typing import Any, Callable, Dict, Mapping, Optional class TestEncodeDecode(ZulipTestCase): def _assert_options(self, options: Dict[str, bool], show_sender: bool=False, diff --git a/zerver/tests/test_email_notifications.py b/zerver/tests/test_email_notifications.py index b8720dd380..bbb134c7cd 100644 --- a/zerver/tests/test_email_notifications.py +++ b/zerver/tests/test_email_notifications.py @@ -1,27 +1,27 @@ -import ldap import random import re -import ujson +from email.utils import formataddr +from typing import List, Optional +from unittest.mock import patch +import ldap +import ujson from django.conf import settings from django.core import mail from django.test import override_settings from django_auth_ldap.config import LDAPSearch -from email.utils import formataddr -from unittest.mock import patch -from typing import List, Optional -from zerver.lib.email_notifications import fix_emojis, handle_missedmessage_emails, \ - enqueue_welcome_emails, relative_to_full_url from zerver.lib.actions import do_change_notification_settings, do_change_user_role -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.send_email import FromAddress, send_custom_email -from zerver.models import ( - get_realm, - get_stream, - UserProfile, - ScheduledEmail, +from zerver.lib.email_notifications import ( + enqueue_welcome_emails, + fix_emojis, + handle_missedmessage_emails, + relative_to_full_url, ) +from zerver.lib.send_email import FromAddress, send_custom_email +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import ScheduledEmail, UserProfile, get_realm, get_stream + class TestCustomEmails(ZulipTestCase): def test_send_custom_email_argument(self) -> None: diff --git a/zerver/tests/test_embedded_bot_system.py b/zerver/tests/test_embedded_bot_system.py index f70fdb5f8d..768237544d 100644 --- a/zerver/tests/test_embedded_bot_system.py +++ b/zerver/tests/test_embedded_bot_system.py @@ -1,13 +1,17 @@ from unittest.mock import patch +import ujson + from zerver.lib.bot_lib import EmbeddedBotQuitException from zerver.lib.test_classes import ZulipTestCase from zerver.models import ( - UserProfile, get_display_recipient, - get_service_profile, get_user, get_realm, + UserProfile, + get_display_recipient, + get_realm, + get_service_profile, + get_user, ) -import ujson class TestEmbeddedBotMessaging(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_event_queue.py b/zerver/tests/test_event_queue.py index f80c8b1eed..5a9cca4e83 100644 --- a/zerver/tests/test_event_queue.py +++ b/zerver/tests/test_event_queue.py @@ -1,18 +1,24 @@ -from unittest import mock import time -import ujson - -from django.http import HttpRequest, HttpResponse from typing import Any, Callable, Dict, Tuple +from unittest import mock -from zerver.lib.actions import do_mute_topic, do_change_subscription_property +import ujson +from django.http import HttpRequest, HttpResponse + +from zerver.lib.actions import do_change_subscription_property, do_mute_topic from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import POSTRequestMock from zerver.models import Recipient, Stream, Subscription, UserProfile, get_stream -from zerver.tornado.event_queue import maybe_enqueue_notifications, \ - allocate_client_descriptor, ClientDescriptor, \ - get_client_descriptor, missedmessage_hook, persistent_queue_filename -from zerver.tornado.views import get_events, cleanup_event_queue +from zerver.tornado.event_queue import ( + ClientDescriptor, + allocate_client_descriptor, + get_client_descriptor, + maybe_enqueue_notifications, + missedmessage_hook, + persistent_queue_filename, +) +from zerver.tornado.views import cleanup_event_queue, get_events + class MissedMessageNotificationsTest(ZulipTestCase): """Tests the logic for when missed-message notifications diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 85f36b5a9c..f2a817df4f 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -1,29 +1,26 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. -from typing import Any, Callable, Dict, List, Optional, Set, Tuple import copy import os import shutil import sys +import time +from io import StringIO +from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from unittest import mock +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now -from io import StringIO - -from zerver.models import ( - get_client, get_stream, get_realm, get_system_bot, - Message, RealmDomain, Recipient, UserMessage, UserPresence, UserProfile, - Realm, Subscription, Stream, flush_per_request_caches, UserGroup, Service, - Attachment, PreregistrationUser, get_user_by_delivery_email, MultiuseInvite, - RealmAuditLog, -) from zerver.lib.actions import ( - try_update_realm_custom_profile_field, + bulk_add_members_to_user_group, bulk_add_subscriptions, bulk_remove_subscriptions, check_add_realm_emoji, + check_add_user_group, + check_delete_user_group, check_send_message, check_send_typing_notification, do_add_alert_words, @@ -44,7 +41,6 @@ from zerver.lib.actions import ( do_change_full_name, do_change_icon_source, do_change_logo_source, - do_change_user_role, do_change_notification_settings, do_change_plan_type, do_change_realm_domain, @@ -53,9 +49,10 @@ from zerver.lib.actions import ( do_change_stream_post_policy, do_change_subscription_property, do_change_user_delivery_email, - do_create_user, + do_change_user_role, do_create_default_stream_group, do_create_multiuse_invite_link, + do_create_user, do_deactivate_stream, do_deactivate_user, do_delete_messages, @@ -78,10 +75,10 @@ from zerver.lib.actions import ( do_revoke_user_invite, do_set_realm_authentication_methods, do_set_realm_message_editing, - do_set_realm_property, - do_set_user_display_setting, do_set_realm_notifications_stream, + do_set_realm_property, do_set_realm_signup_notifications_stream, + do_set_user_display_setting, do_set_zoom_token, do_unmute_topic, do_update_embedded_data, @@ -89,18 +86,16 @@ from zerver.lib.actions import ( do_update_message_flags, do_update_outgoing_webhook_service, do_update_pointer, + do_update_user_custom_profile_data_if_changed, + do_update_user_group_description, + do_update_user_group_name, do_update_user_presence, do_update_user_status, log_event, lookup_default_stream_groups, notify_realm_custom_profile_fields, - check_add_user_group, - do_update_user_group_name, - do_update_user_group_description, - bulk_add_members_to_user_group, remove_members_from_user_group, - check_delete_user_group, - do_update_user_custom_profile_data_if_changed, + try_update_realm_custom_profile_field, ) from zerver.lib.bugdown import MentionData from zerver.lib.events import ( @@ -110,36 +105,65 @@ from zerver.lib.events import ( post_process_state, ) from zerver.lib.message import ( + MessageDict, + UnreadMessagesResult, aggregate_unread_data, apply_unread_message_event, get_raw_unread_data, render_markdown, - MessageDict, - UnreadMessagesResult, ) -from zerver.lib.test_helpers import POSTRequestMock, get_subscription, \ - get_test_image_file, stub_event_queue_user_events, queries_captured, \ - create_dummy_file, stdout_suppressed, reset_emails_in_zulip_realm -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + POSTRequestMock, + create_dummy_file, + get_subscription, + get_test_image_file, + queries_captured, + reset_emails_in_zulip_realm, + stdout_suppressed, + stub_event_queue_user_events, ) from zerver.lib.test_runner import slow -from zerver.lib.topic import ( - ORIG_TOPIC, - TOPIC_NAME, - TOPIC_LINKS, -) -from zerver.lib.topic_mutes import ( - add_topic_mute, -) -from zerver.lib.validator import ( - check_bool, check_dict, check_dict_only, check_float, check_int, check_list, check_string, - equals, check_none_or, Validator, check_url, check_int_in, -) +from zerver.lib.topic import ORIG_TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.topic_mutes import add_topic_mute from zerver.lib.users import get_api_key - -from zerver.views.events_register import _default_all_public_streams, _default_narrow - +from zerver.lib.validator import ( + Validator, + check_bool, + check_dict, + check_dict_only, + check_float, + check_int, + check_int_in, + check_list, + check_none_or, + check_string, + check_url, + equals, +) +from zerver.models import ( + Attachment, + Message, + MultiuseInvite, + PreregistrationUser, + Realm, + RealmAuditLog, + RealmDomain, + Recipient, + Service, + Stream, + Subscription, + UserGroup, + UserMessage, + UserPresence, + UserProfile, + flush_per_request_caches, + get_client, + get_realm, + get_stream, + get_system_bot, + get_user_by_delivery_email, +) from zerver.tornado.event_queue import ( allocate_client_descriptor, clear_client_event_queues_for_testing, @@ -147,10 +171,7 @@ from zerver.tornado.event_queue import ( process_message_event, ) from zerver.tornado.views import get_events - -from unittest import mock -import time -import ujson +from zerver.views.events_register import _default_all_public_streams, _default_narrow class LogEventsTest(ZulipTestCase): diff --git a/zerver/tests/test_external.py b/zerver/tests/test_external.py index 38a1886f34..41273b6439 100644 --- a/zerver/tests/test_external.py +++ b/zerver/tests/test_external.py @@ -1,28 +1,22 @@ +import time +from unittest import mock + +import DNS from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpResponse from zerver.forms import email_is_not_mit_mailing_list - from zerver.lib.rate_limiter import ( - add_ratelimit_rule, - remove_ratelimit_rule, RateLimitedUser, RateLimiterLockingException, + add_ratelimit_rule, + remove_ratelimit_rule, ) +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.zephyr import compute_mit_user_fullname +from zerver.models import UserProfile -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - UserProfile, -) - -import DNS -from unittest import mock -import time class MITNameTest(ZulipTestCase): def test_valid_hesiod(self) -> None: diff --git a/zerver/tests/test_gitter_importer.py b/zerver/tests/test_gitter_importer.py index a50ba1981d..89c217844e 100644 --- a/zerver/tests/test_gitter_importer.py +++ b/zerver/tests/test_gitter_importer.py @@ -1,24 +1,15 @@ -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.models import ( - get_realm, - UserProfile, - Message, -) -from zerver.data_import.gitter import ( - do_convert_data, - get_usermentions, -) - -import ujson import logging import os -from unittest import mock from typing import Any +from unittest import mock + +import ujson + +from zerver.data_import.gitter import do_convert_data, get_usermentions +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, UserProfile, get_realm + class GitterImporter(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_hipchat_importer.py b/zerver/tests/test_hipchat_importer.py index 709e2a2191..55657729f1 100644 --- a/zerver/tests/test_hipchat_importer.py +++ b/zerver/tests/test_hipchat_importer.py @@ -1,18 +1,10 @@ -from zerver.data_import.hipchat import ( - get_hipchat_sender_id, -) -from zerver.data_import.hipchat_user import ( - UserHandler, -) -from zerver.data_import.sequencer import ( - IdMapper, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) from typing import Any, Dict +from zerver.data_import.hipchat import get_hipchat_sender_id +from zerver.data_import.hipchat_user import UserHandler +from zerver.data_import.sequencer import IdMapper +from zerver.lib.test_classes import ZulipTestCase + class HipChatImporter(ZulipTestCase): def test_sender_ids(self) -> None: diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index b1fc3b62b5..e1b00d40c4 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -1,34 +1,38 @@ import calendar +import urllib from datetime import timedelta +from typing import Any, Dict +from unittest.mock import patch + import lxml.html import ujson - from django.conf import settings from django.http import HttpResponse from django.utils.timezone import now as timezone_now -from unittest.mock import patch -import urllib -from typing import Any, Dict -from zerver.lib.actions import ( - do_create_user, do_change_logo_source, -) + +from corporate.models import Customer, CustomerPlan +from zerver.lib.actions import do_change_logo_source, do_create_user from zerver.lib.events import add_realm_logo_fields -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - queries_captured, get_user_messages, -) from zerver.lib.soft_deactivation import do_soft_deactivate_users +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_user_messages, queries_captured from zerver.lib.test_runner import slow from zerver.lib.users import compute_show_invites_and_add_streams from zerver.models import ( - get_realm, get_stream, get_user, UserProfile, - flush_per_request_caches, DefaultStream, Realm, - get_system_bot, UserActivity, + DefaultStream, + Realm, + UserActivity, + UserProfile, + flush_per_request_caches, + get_realm, + get_stream, + get_system_bot, + get_user, ) from zerver.views.home import compute_navbar_logo_url, get_furthest_read_time -from corporate.models import Customer, CustomerPlan from zerver.worker.queue_processors import UserActivityWorker + class HomeTest(ZulipTestCase): def test_home(self) -> None: diff --git a/zerver/tests/test_hotspots.py b/zerver/tests/test_hotspots.py index c516307d04..59123353b4 100644 --- a/zerver/tests/test_hotspots.py +++ b/zerver/tests/test_hotspots.py @@ -1,9 +1,10 @@ -from zerver.lib.actions import do_mark_hotspot_as_read, do_create_user +import ujson + +from zerver.lib.actions import do_create_user, do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS, get_next_hotspots from zerver.lib.test_classes import ZulipTestCase -from zerver.models import UserProfile, UserHotspot, get_realm +from zerver.models import UserHotspot, UserProfile, get_realm -import ujson # Splitting this out, since I imagine this will eventually have most of the # complicated hotspots logic. diff --git a/zerver/tests/test_i18n.py b/zerver/tests/test_i18n.py index 6c550da043..804998b3fd 100644 --- a/zerver/tests/test_i18n.py +++ b/zerver/tests/test_i18n.py @@ -1,20 +1,19 @@ +from http.cookies import SimpleCookie from typing import Any - from unittest import mock + import ujson +from django.conf import settings +from django.core import mail +from django.http import HttpResponse from django.test import TestCase from django.utils import translation -from django.conf import settings -from django.http import HttpResponse -from django.core import mail -from http.cookies import SimpleCookie -from zerver.models import get_realm_stream -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.management.commands import makemessages from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.test_classes import ZulipTestCase +from zerver.management.commands import makemessages +from zerver.models import get_realm_stream + class EmailTranslationTestCase(ZulipTestCase): def test_email_translation(self) -> None: diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 809c2b345c..0fdacd442a 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -1,95 +1,68 @@ -from django.conf import settings - import os -import ujson - +from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple from unittest.mock import patch -from typing import Any, Dict, List, Set, Optional, Tuple, Callable, \ - FrozenSet + +import ujson +from django.conf import settings from django.db.models import Q from django.utils.timezone import now as timezone_now -from zerver.lib.export import ( - do_export_realm, - export_usermessages_batch, - do_export_user, -) -from zerver.lib.import_realm import ( - do_import_realm, - get_incoming_message_ids, -) -from zerver.lib.avatar_hash import ( - user_avatar_path, -) -from zerver.lib.upload import ( - claim_attachment, - upload_message_file, - upload_emoji_image, - upload_avatar_image, -) from zerver.lib import upload - -from zerver.lib.utils import ( - query_chunker, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - get_test_image_file, - use_s3_backend, - create_s3_buckets, -) - -from zerver.lib.topic_mutes import ( - add_topic_mute, -) -from zerver.lib.bot_lib import ( - StateHandler, -) -from zerver.lib.bot_config import ( - set_bot_config, -) from zerver.lib.actions import ( - do_create_user, do_add_reaction, do_change_icon_source, do_change_logo_source, - do_update_user_presence, do_change_plan_type, + do_create_user, + do_update_user_presence, ) +from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.bot_config import set_bot_config +from zerver.lib.bot_lib import StateHandler +from zerver.lib.export import do_export_realm, do_export_user, export_usermessages_batch +from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids from zerver.lib.streams import create_stream_if_needed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, use_s3_backend from zerver.lib.test_runner import slow - +from zerver.lib.topic_mutes import add_topic_mute +from zerver.lib.upload import ( + claim_attachment, + upload_avatar_image, + upload_emoji_image, + upload_message_file, +) +from zerver.lib.utils import query_chunker from zerver.models import ( - Message, - Realm, - Stream, - UserProfile, - Subscription, Attachment, - RealmEmoji, - Reaction, - Recipient, - UserMessage, + BotConfigData, + BotStorageData, CustomProfileField, CustomProfileFieldValue, - RealmAuditLog, Huddle, - UserHotspot, + Message, MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmEmoji, + Recipient, + Stream, + Subscription, UserGroup, UserGroupMembership, + UserHotspot, + UserMessage, UserPresence, - BotStorageData, - BotConfigData, + UserProfile, get_active_streams, get_client, + get_huddle_hash, get_realm, get_stream, - get_huddle_hash, ) + class QueryUtilTest(ZulipTestCase): def _create_messages(self) -> None: for name in ['cordelia', 'hamlet', 'iago']: diff --git a/zerver/tests/test_integrations.py b/zerver/tests/test_integrations.py index ae4b0e136a..ff0bb2b7d4 100644 --- a/zerver/tests/test_integrations.py +++ b/zerver/tests/test_integrations.py @@ -1,7 +1,15 @@ -from zerver.lib.test_classes import ZulipTestCase from zerver.lib.integrations import ( - split_fixture_path, get_fixture_and_image_paths, INTEGRATIONS, ScreenshotConfig, WebhookIntegration, - DOC_SCREENSHOT_CONFIG, WEBHOOK_INTEGRATIONS, NO_SCREENSHOT_WEBHOOKS) + DOC_SCREENSHOT_CONFIG, + INTEGRATIONS, + NO_SCREENSHOT_WEBHOOKS, + WEBHOOK_INTEGRATIONS, + ScreenshotConfig, + WebhookIntegration, + get_fixture_and_image_paths, + split_fixture_path, +) +from zerver.lib.test_classes import ZulipTestCase + class IntegrationsTestCase(ZulipTestCase): diff --git a/zerver/tests/test_integrations_dev_panel.py b/zerver/tests/test_integrations_dev_panel.py index 3a6dd4528f..7c7af870fa 100644 --- a/zerver/tests/test_integrations_dev_panel.py +++ b/zerver/tests/test_integrations_dev_panel.py @@ -1,7 +1,10 @@ -import ujson from unittest.mock import MagicMock, patch + +import ujson + from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_user, get_realm, Message, Stream +from zerver.models import Message, Stream, get_realm, get_user + class TestIntegrationsDevPanel(ZulipTestCase): diff --git a/zerver/tests/test_internet.py b/zerver/tests/test_internet.py index 0fee3306e5..6172c6f4c2 100644 --- a/zerver/tests/test_internet.py +++ b/zerver/tests/test_internet.py @@ -1,7 +1,8 @@ +import requests +import responses + from zerver.lib.test_classes import ZulipTestCase -import responses -import requests class ResponsesTest(ZulipTestCase): def test_responses(self) -> None: diff --git a/zerver/tests/test_legacy_subject.py b/zerver/tests/test_legacy_subject.py index 436a9de452..90702c7d69 100644 --- a/zerver/tests/test_legacy_subject.py +++ b/zerver/tests/test_legacy_subject.py @@ -1,6 +1,5 @@ -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase + class LegacySubjectTest(ZulipTestCase): def test_legacy_subject(self) -> None: diff --git a/zerver/tests/test_link_embed.py b/zerver/tests/test_link_embed.py index 14a1204bc5..c6b19a126a 100644 --- a/zerver/tests/test_link_embed.py +++ b/zerver/tests/test_link_embed.py @@ -1,22 +1,20 @@ -from unittest import mock -import ujson from typing import Any, Callable, Dict, Optional -from requests.exceptions import ConnectionError +from unittest import mock + +import ujson from django.test import override_settings from django.utils.html import escape +from requests.exceptions import ConnectionError -from zerver.models import Message, Realm, UserProfile from zerver.lib.actions import queue_json_publish +from zerver.lib.cache import NotFoundInCache, cache_set, preview_url_cache_key from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import MockPythonResponse -from zerver.worker.queue_processors import FetchLinksEmbedData -from zerver.lib.url_preview.preview import ( - get_link_embed_data, link_embed_data_from_cache) from zerver.lib.url_preview.oembed import get_oembed_data, strip_cdata -from zerver.lib.url_preview.parsers import ( - OpenGraphParser, GenericParser) -from zerver.lib.cache import cache_set, NotFoundInCache, preview_url_cache_key - +from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser +from zerver.lib.url_preview.preview import get_link_embed_data, link_embed_data_from_cache +from zerver.models import Message, Realm, UserProfile +from zerver.worker.queue_processors import FetchLinksEmbedData TEST_CACHES = { 'default': { diff --git a/zerver/tests/test_logging_handlers.py b/zerver/tests/test_logging_handlers.py index fd90ee3afd..d3a266b7ac 100644 --- a/zerver/tests/test_logging_handlers.py +++ b/zerver/tests/test_logging_handlers.py @@ -1,18 +1,18 @@ import logging import sys +from functools import wraps +from types import TracebackType +from typing import Any, Callable, Dict, Iterator, NoReturn, Optional, Tuple, Type +from unittest.mock import MagicMock, patch from django.conf import settings from django.contrib.auth.models import AnonymousUser from django.http import HttpRequest from django.test import TestCase from django.utils.log import AdminEmailHandler -from functools import wraps -from unittest.mock import MagicMock, patch -from types import TracebackType -from typing import Any, Callable, Dict, Iterator, NoReturn, Optional, Tuple, Type -from zerver.lib.types import ViewFuncT from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.types import ViewFuncT from zerver.logging_handlers import AdminNotifyHandler captured_request: Optional[HttpRequest] = None diff --git a/zerver/tests/test_management_commands.py b/zerver/tests/test_management_commands.py index cb47a55667..f0d9d7d26f 100644 --- a/zerver/tests/test_management_commands.py +++ b/zerver/tests/test_management_commands.py @@ -3,24 +3,32 @@ import os import re from datetime import timedelta from email.utils import parseaddr +from typing import Any, Dict, List, Optional from unittest import mock -from unittest.mock import MagicMock, patch, call -from typing import List, Dict, Any, Optional +from unittest.mock import MagicMock, call, patch from django.conf import settings from django.core.management import call_command from django.test import TestCase, override_settings -from zerver.lib.actions import do_create_user, do_add_reaction -from zerver.lib.management import ZulipBaseCommand, CommandError, check_config -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import stdout_suppressed -from zerver.lib.test_runner import slow -from zerver.models import Recipient, get_user_profile_by_email, get_stream from django.utils.timezone import now as timezone_now -from zerver.lib.test_helpers import most_recent_message -from zerver.models import get_realm, UserProfile, Realm, Reaction, Message from confirmation.models import RealmCreationKey, generate_realm_creation_url +from zerver.lib.actions import do_add_reaction, do_create_user +from zerver.lib.management import CommandError, ZulipBaseCommand, check_config +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, stdout_suppressed +from zerver.lib.test_runner import slow +from zerver.models import ( + Message, + Reaction, + Realm, + Recipient, + UserProfile, + get_realm, + get_stream, + get_user_profile_by_email, +) + class TestCheckConfig(ZulipTestCase): def test_check_config(self) -> None: diff --git a/zerver/tests/test_mattermost_importer.py b/zerver/tests/test_mattermost_importer.py index ff0978b3cd..f98784b4d0 100644 --- a/zerver/tests/test_mattermost_importer.py +++ b/zerver/tests/test_mattermost_importer.py @@ -1,27 +1,34 @@ -import os -import ujson import filecmp import logging +import os +from typing import Any, Dict, List -from typing import Dict, Any, List +import ujson -from zerver.lib.emoji import name_to_codepoint -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.data_import.mattermost_user import UserHandler -from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \ - create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \ - convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \ - build_reactions, do_convert_data, convert_huddle_data, \ - generate_huddle_name -from zerver.data_import.sequencer import IdMapper from zerver.data_import.import_util import SubscriberHandler -from zerver.models import Reaction, UserProfile, Message, get_realm, get_user, Recipient +from zerver.data_import.mattermost import ( + build_reactions, + check_user_in_team, + convert_channel_data, + convert_huddle_data, + convert_user_data, + create_username_to_user_mapping, + do_convert_data, + generate_huddle_name, + get_mentioned_user_ids, + label_mirror_dummy_users, + mattermost_data_file_to_dict, + process_user, + reset_mirror_dummy_users, + write_emoticon_data, +) +from zerver.data_import.mattermost_user import UserHandler +from zerver.data_import.sequencer import IdMapper +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, Reaction, Recipient, UserProfile, get_realm, get_user + class MatterMostImporter(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_message_edit_notifications.py b/zerver/tests/test_message_edit_notifications.py index c2f882b2a7..00800c1e78 100644 --- a/zerver/tests/test_message_edit_notifications.py +++ b/zerver/tests/test_message_edit_notifications.py @@ -1,25 +1,13 @@ from typing import Any, Dict, Mapping, Union - from unittest import mock from django.utils.timezone import now as timezone_now -from zerver.lib.actions import ( - get_client, -) +from zerver.lib.actions import get_client +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Subscription, UserPresence +from zerver.tornado.event_queue import maybe_enqueue_notifications -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - Subscription, - UserPresence, -) - -from zerver.tornado.event_queue import ( - maybe_enqueue_notifications, -) class EditMessageSideEffectsTest(ZulipTestCase): def _assert_update_does_not_notify_anybody(self, message_id: int, content: str) -> None: diff --git a/zerver/tests/test_messages.py b/zerver/tests/test_messages.py index 6a110fd8b8..bfec18fcbe 100644 --- a/zerver/tests/test_messages.py +++ b/zerver/tests/test_messages.py @@ -1,15 +1,22 @@ +import datetime +import time +from collections import namedtuple +from operator import itemgetter +from typing import Any, Dict, List, Set, Tuple, Union +from unittest import mock + +import ujson +from django.conf import settings from django.db import IntegrityError from django.db.models import Q -from django.conf import settings from django.http import HttpResponse from django.test import TestCase, override_settings from django.utils.timezone import now as timezone_now -from zerver.lib import bugdown +from analytics.lib.counts import COUNT_STATS +from analytics.models import RealmCount from zerver.decorator import JsonableError -from zerver.lib.test_runner import slow -from zerver.lib.addressee import Addressee - +from zerver.lib import bugdown from zerver.lib.actions import ( check_message, check_send_stream_message, @@ -22,8 +29,8 @@ from zerver.lib.actions import ( do_create_user, do_deactivate_user, do_send_messages, - do_update_message, do_set_realm_property, + do_update_message, extract_private_recipients, extract_stream_indicator, gather_subscriptions_helper, @@ -40,18 +47,9 @@ from zerver.lib.actions import ( internal_send_stream_message_by_name, send_rate_limited_pm_notification_to_bot_owner, ) - -from zerver.lib.cache import ( - cache_delete, - get_stream_cache_key, - to_dict_cache_key_id, -) - - -from zerver.lib.create_user import ( - create_user_profile, -) - +from zerver.lib.addressee import Addressee +from zerver.lib.cache import cache_delete, get_stream_cache_key, to_dict_cache_key_id +from zerver.lib.create_user import create_user_profile from zerver.lib.message import ( MessageDict, bulk_access_messages, @@ -64,7 +62,13 @@ from zerver.lib.message import ( sew_messages_and_reactions, update_first_visible_message_id, ) - +from zerver.lib.soft_deactivation import ( + add_missing_messages, + do_soft_activate_users, + do_soft_deactivate_users, + reactivate_user_if_soft_deactivated, +) +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( get_subscription, get_user_messages, @@ -75,55 +79,42 @@ from zerver.lib.test_helpers import ( queries_captured, reset_emails_in_zulip_realm, ) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.lib.topic import ( - LEGACY_PREV_TOPIC, - DB_TOPIC_NAME, - TOPIC_LINKS, - TOPIC_NAME, -) - -from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.lib.soft_deactivation import ( - add_missing_messages, - do_soft_activate_users, - do_soft_deactivate_users, - reactivate_user_if_soft_deactivated, -) - -from zerver.models import ( - MAX_MESSAGE_LENGTH, MAX_TOPIC_NAME_LENGTH, - Message, Realm, Recipient, Stream, UserMessage, UserProfile, Attachment, - RealmAuditLog, RealmDomain, get_realm, UserPresence, Subscription, - get_stream, get_system_bot, get_user, Reaction, - flush_per_request_caches, ScheduledMessage, get_huddle_recipient, - bulk_get_huddle_user_ids, get_huddle_user_ids, - get_display_recipient, RealmFilter, -) - - +from zerver.lib.test_runner import slow from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp from zerver.lib.timezone import get_timezone +from zerver.lib.topic import DB_TOPIC_NAME, LEGACY_PREV_TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient from zerver.lib.upload import create_attachment from zerver.lib.url_encoding import near_message_url +from zerver.models import ( + MAX_MESSAGE_LENGTH, + MAX_TOPIC_NAME_LENGTH, + Attachment, + Message, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmFilter, + Recipient, + ScheduledMessage, + Stream, + Subscription, + UserMessage, + UserPresence, + UserProfile, + bulk_get_huddle_user_ids, + flush_per_request_caches, + get_display_recipient, + get_huddle_recipient, + get_huddle_user_ids, + get_realm, + get_stream, + get_system_bot, + get_user, +) +from zerver.views.messages import InvalidMirrorInput, create_mirrored_message_users -from zerver.views.messages import create_mirrored_message_users, InvalidMirrorInput - -from analytics.lib.counts import COUNT_STATS -from analytics.models import RealmCount - -import datetime -from unittest import mock -from operator import itemgetter -import time -import ujson -from typing import Any, Dict, List, Set, Union, Tuple - -from collections import namedtuple class MiscMessageTest(ZulipTestCase): def test_get_last_message_id(self) -> None: diff --git a/zerver/tests/test_middleware.py b/zerver/tests/test_middleware.py index 1e5c0a440f..6506b73f04 100644 --- a/zerver/tests/test_middleware.py +++ b/zerver/tests/test_middleware.py @@ -1,13 +1,15 @@ import time from typing import List +from unittest.mock import patch from bs4 import BeautifulSoup -from unittest.mock import patch + from zerver.lib.realm_icon import get_realm_icon_url from zerver.lib.test_classes import ZulipTestCase from zerver.middleware import is_slow_query, write_log_line from zerver.models import get_realm + class SlowQueryTest(ZulipTestCase): SLOW_QUERY_TIME = 10 log_data = {'extra': '[transport=websocket]', diff --git a/zerver/tests/test_migrations.py b/zerver/tests/test_migrations.py index 7382f613a2..a83f190b9a 100644 --- a/zerver/tests/test_migrations.py +++ b/zerver/tests/test_migrations.py @@ -4,11 +4,10 @@ # You can also read # https://www.caktusgroup.com/blog/2016/02/02/writing-unit-tests-django-migrations/ # to get a tutorial on the framework that inspired this feature. +from django.db.migrations.state import StateApps from zerver.lib.test_classes import MigrationsTestCase from zerver.lib.test_helpers import use_db_models -from django.db.migrations.state import StateApps - from zerver.models import get_stream # Important note: These tests are very expensive, and details of diff --git a/zerver/tests/test_muting.py b/zerver/tests/test_muting.py index 5a64ee928e..91d72ff386 100644 --- a/zerver/tests/test_muting.py +++ b/zerver/tests/test_muting.py @@ -1,23 +1,19 @@ -from django.utils.timezone import now as timezone_now -from datetime import timedelta, datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Dict from unittest import mock -from zerver.lib.test_classes import ZulipTestCase +from django.utils.timezone import now as timezone_now + from zerver.lib.stream_topic import StreamTopicTarget - -from zerver.models import ( - get_stream, - UserProfile, - MutedTopic, -) - +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic_mutes import ( add_topic_mute, get_topic_mutes, remove_topic_mute, topic_is_muted, ) +from zerver.models import MutedTopic, UserProfile, get_stream + class MutedTopicsTests(ZulipTestCase): def test_user_ids_muting_topic(self) -> None: diff --git a/zerver/tests/test_narrow.py b/zerver/tests/test_narrow.py index a118e44491..0eaf11bae6 100644 --- a/zerver/tests/test_narrow.py +++ b/zerver/tests/test_narrow.py @@ -1,57 +1,47 @@ +import os +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union +from unittest import mock + +import ujson from django.db import connection from django.test import TestCase, override_settings -from sqlalchemy.sql import ( - and_, select, column, table, -) +from sqlalchemy.sql import and_, column, select, table from sqlalchemy.sql.elements import ClauseElement -from zerver.models import ( - Realm, Subscription, Recipient, Stream, - get_display_recipient, get_realm, get_stream, - UserMessage, Message, -) -from zerver.lib.actions import ( - do_set_realm_property, - do_deactivate_user, -) -from zerver.lib.message import ( - MessageDict, -) -from zerver.lib.narrow import ( - build_narrow_filter, - is_web_public_compatible, -) +from zerver.lib.actions import do_deactivate_user, do_set_realm_property +from zerver.lib.message import MessageDict +from zerver.lib.narrow import build_narrow_filter, is_web_public_compatible from zerver.lib.request import JsonableError from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import ( - POSTRequestMock, - get_user_messages, queries_captured, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.topic import ( - MATCH_TOPIC, - TOPIC_NAME, -) -from zerver.lib.topic_mutes import ( - set_topic_mutes, -) -from zerver.lib.types import DisplayRecipientT -from zerver.views.messages import ( - exclude_muting_conditions, - get_messages_backend, ok_to_include_history, - NarrowBuilder, BadNarrowOperator, Query, - post_process_limited_query, - find_first_unread_anchor, - LARGER_THAN_MAX_MESSAGE_ID, -) from zerver.lib.streams import create_streams_if_needed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import POSTRequestMock, get_user_messages, queries_captured +from zerver.lib.topic import MATCH_TOPIC, TOPIC_NAME +from zerver.lib.topic_mutes import set_topic_mutes +from zerver.lib.types import DisplayRecipientT +from zerver.models import ( + Message, + Realm, + Recipient, + Stream, + Subscription, + UserMessage, + get_display_recipient, + get_realm, + get_stream, +) +from zerver.views.messages import ( + LARGER_THAN_MAX_MESSAGE_ID, + BadNarrowOperator, + NarrowBuilder, + Query, + exclude_muting_conditions, + find_first_unread_anchor, + get_messages_backend, + ok_to_include_history, + post_process_limited_query, +) -from typing import Dict, Mapping, List, Sequence, Tuple, Union, Any, Optional -from unittest import mock -import os -import ujson def get_sqlalchemy_sql(query: ClauseElement) -> str: dialect = get_sqlalchemy_connection().dialect diff --git a/zerver/tests/test_new_users.py b/zerver/tests/test_new_users.py index ec1baccb00..9cc3778ab6 100644 --- a/zerver/tests/test_new_users.py +++ b/zerver/tests/test_new_users.py @@ -1,15 +1,17 @@ import datetime +from unittest import mock + from django.conf import settings from django.core import mail from django.test import override_settings -from zerver.lib.test_classes import ZulipTestCase -from zerver.signals import get_device_browser, get_device_os, JUST_CREATED_THRESHOLD -from zerver.lib.actions import notify_new_user, do_change_notification_settings -from zerver.models import Recipient, Stream, Realm +from zerver.lib.actions import do_change_notification_settings, notify_new_user from zerver.lib.initial_password import initial_password -from unittest import mock +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timezone import get_timezone +from zerver.models import Realm, Recipient, Stream +from zerver.signals import JUST_CREATED_THRESHOLD, get_device_browser, get_device_os + class SendLoginEmailTest(ZulipTestCase): """ diff --git a/zerver/tests/test_onboarding.py b/zerver/tests/test_onboarding.py index 4477257c20..ace78fcd29 100644 --- a/zerver/tests/test_onboarding.py +++ b/zerver/tests/test_onboarding.py @@ -1,8 +1,7 @@ -from zerver.models import Realm, UserProfile from zerver.lib.onboarding import create_if_missing_realm_internal_bots -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Realm, UserProfile + class TestRealmInternalBotCreation(ZulipTestCase): def test_create_if_missing_realm_internal_bots(self) -> None: diff --git a/zerver/tests/test_openapi.py b/zerver/tests/test_openapi.py index 7a7f8fff1a..03dc02bdd0 100644 --- a/zerver/tests/test_openapi.py +++ b/zerver/tests/test_openapi.py @@ -1,24 +1,43 @@ +import inspect import re import sys +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, +) from unittest import mock -import inspect -from typing import Dict, Any, Set, Union, List, Callable, Tuple, Optional, Iterable, Mapping, Sequence -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from django.http import HttpResponse -import zerver.openapi.openapi as openapi -from zerver.openapi.markdown_extension import generate_curl_example, \ - render_curl_example, parse_language_and_options -from zerver.lib.request import _REQ +from zerver.lib.request import _REQ, arguments_map from zerver.lib.test_classes import ZulipTestCase -from zerver.openapi.openapi import ( - get_openapi_fixture, get_openapi_parameters, - validate_against_openapi_schema, to_python_type, - SchemaError, openapi_spec, get_openapi_paths, - OpenAPISpec, OPENAPI_SPEC_PATH, +from zerver.openapi import openapi as openapi +from zerver.openapi.markdown_extension import ( + generate_curl_example, + parse_language_and_options, + render_curl_example, +) +from zerver.openapi.openapi import ( + OPENAPI_SPEC_PATH, + OpenAPISpec, + SchemaError, + get_openapi_fixture, + get_openapi_parameters, + get_openapi_paths, + openapi_spec, + to_python_type, + validate_against_openapi_schema, ) -from zerver.lib.request import arguments_map TEST_ENDPOINT = '/messages/{message_id}' TEST_METHOD = 'patch' @@ -525,7 +544,7 @@ do not match the types declared in the implementation of {}.\n""".format(functio in code. """ - import zproject.urls as urlconf + from zproject import urls as urlconf # We loop through all the API patterns, looking in particular # for those using the rest_dispatch decorator; we then parse diff --git a/zerver/tests/test_outgoing_webhook_interfaces.py b/zerver/tests/test_outgoing_webhook_interfaces.py index 5e141ec8b0..75b8ecdc87 100644 --- a/zerver/tests/test_outgoing_webhook_interfaces.py +++ b/zerver/tests/test_outgoing_webhook_interfaces.py @@ -1,25 +1,16 @@ -from typing import cast, Any, Dict - -from unittest import mock import json +from typing import Any, Dict, cast +from unittest import mock + import requests from zerver.lib.avatar import get_gravatar_url from zerver.lib.message import MessageDict -from zerver.lib.outgoing_webhook import ( - get_service_interface_class, - process_success_response, -) +from zerver.lib.outgoing_webhook import get_service_interface_class, process_success_response from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.topic import TOPIC_NAME -from zerver.models import ( - get_realm, - get_stream, - get_user, - Message, - SLACK_INTERFACE, -) +from zerver.models import SLACK_INTERFACE, Message, get_realm, get_stream, get_user class TestGenericOutgoingWebhookService(ZulipTestCase): diff --git a/zerver/tests/test_outgoing_webhook_system.py b/zerver/tests/test_outgoing_webhook_system.py index aee46890ae..b0c30cf3f4 100644 --- a/zerver/tests/test_outgoing_webhook_system.py +++ b/zerver/tests/test_outgoing_webhook_system.py @@ -1,33 +1,29 @@ -import ujson import logging -from unittest import mock -import requests - from typing import Any, Optional +from unittest import mock -from zerver.lib.actions import ( - do_create_user, -) +import requests +import ujson +from version import ZULIP_VERSION +from zerver.lib.actions import do_create_user from zerver.lib.outgoing_webhook import ( - do_rest_call, GenericOutgoingWebhookService, SlackOutgoingWebhookService, + do_rest_call, ) - from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic import TOPIC_NAME from zerver.lib.users import add_service from zerver.models import ( - get_display_recipient, - get_realm, - get_user, Recipient, Service, UserProfile, + get_display_recipient, + get_realm, + get_user, ) -from version import ZULIP_VERSION class ResponseMock: def __init__(self, status_code: int, content: Optional[Any]=None) -> None: diff --git a/zerver/tests/test_populate_db.py b/zerver/tests/test_populate_db.py index 31c946c050..1111d73858 100644 --- a/zerver/tests/test_populate_db.py +++ b/zerver/tests/test_populate_db.py @@ -1,8 +1,9 @@ -from zilencer.management.commands.populate_db import choose_date_sent -from zerver.lib.test_classes import ZulipTestCase - from django.utils.timezone import timedelta as timezone_timedelta +from zerver.lib.test_classes import ZulipTestCase +from zilencer.management.commands.populate_db import choose_date_sent + + class TestChoosePubDate(ZulipTestCase): def test_choose_date_sent_large_tot_messages(self) -> None: """ diff --git a/zerver/tests/test_presence.py b/zerver/tests/test_presence.py index 9a16db32d6..52cb221ee9 100644 --- a/zerver/tests/test_presence.py +++ b/zerver/tests/test_presence.py @@ -1,33 +1,26 @@ +import datetime from datetime import timedelta -from django.utils.timezone import now as timezone_now +from typing import Any, Dict from unittest import mock -from typing import Any, Dict +from django.utils.timezone import now as timezone_now + from zerver.lib.actions import do_deactivate_user -from zerver.lib.presence import ( - get_status_dict_by_realm, -) +from zerver.lib.presence import get_status_dict_by_realm from zerver.lib.statistics import seconds_usage_between -from zerver.lib.test_helpers import ( - make_client, - queries_captured, - reset_emails_in_zulip_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import make_client, queries_captured, reset_emails_in_zulip_realm from zerver.lib.timestamp import datetime_to_timestamp from zerver.models import ( Client, PushDeviceToken, UserActivity, UserActivityInterval, - UserProfile, UserPresence, + UserProfile, flush_per_request_caches, ) -import datetime class ActivityTest(ZulipTestCase): @mock.patch("stripe.Customer.list", return_value=[]) diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index b2097566fc..6255fbc1db 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -1,52 +1,35 @@ -from contextlib import contextmanager +import base64 import datetime import itertools -import requests +import os +import uuid +from contextlib import contextmanager +from typing import Any, Dict, Iterator, List, Optional from unittest import mock from unittest.mock import call -from typing import Any, Dict, Iterator, List, Optional -import base64 -import os +import requests import ujson -import uuid - -from django.test import override_settings from django.conf import settings -from django.http import HttpResponse from django.db import transaction from django.db.models import F +from django.http import HttpResponse +from django.test import override_settings from django.utils.crypto import get_random_string +from django.utils.timezone import now from analytics.lib.counts import CountStat, LoggingCountStat from analytics.models import InstallationCount, RealmCount -from zerver.models import ( - PushDeviceToken, - Message, - UserMessage, - receives_offline_email_notifications, - receives_offline_push_notifications, - receives_online_notifications, - receives_stream_notifications, - get_client, - get_realm, - get_stream, - Recipient, - RealmAuditLog, - Stream, - Subscription, -) from zerver.lib.actions import ( do_delete_messages, do_mark_stream_messages_as_read, do_regenerate_api_key, ) -from zerver.lib.soft_deactivation import do_soft_deactivate_users from zerver.lib.push_notifications import ( + DeviceToken, absolute_avatar_url, b64_to_hex, datetime_to_timestamp, - DeviceToken, get_apns_client, get_display_recipient, get_message_payload_apns, @@ -63,16 +46,38 @@ from zerver.lib.push_notifications import ( send_notifications_to_bouncer, send_to_push_bouncer, ) -from zerver.lib.remote_server import send_analytics_to_remote_server, \ - build_analytics_data, PushNotificationBouncerException, PushNotificationBouncerRetryLaterError -from zerver.lib.request import JsonableError -from zerver.lib.test_classes import ( - TestCase, ZulipTestCase, +from zerver.lib.remote_server import ( + PushNotificationBouncerException, + PushNotificationBouncerRetryLaterError, + build_analytics_data, + send_analytics_to_remote_server, +) +from zerver.lib.request import JsonableError +from zerver.lib.soft_deactivation import do_soft_deactivate_users +from zerver.lib.test_classes import TestCase, ZulipTestCase +from zerver.models import ( + Message, + PushDeviceToken, + RealmAuditLog, + Recipient, + Stream, + Subscription, + UserMessage, + get_client, + get_realm, + get_stream, + receives_offline_email_notifications, + receives_offline_push_notifications, + receives_online_notifications, + receives_stream_notifications, +) +from zilencer.models import ( + RemoteInstallationCount, + RemotePushDeviceToken, + RemoteRealmAuditLog, + RemoteRealmCount, + RemoteZulipServer, ) - -from zilencer.models import RemoteZulipServer, RemotePushDeviceToken, \ - RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog -from django.utils.timezone import now ZERVER_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -1510,6 +1515,7 @@ class TestSendToPushBouncer(ZulipTestCase): def test_400_error_invalid_server_key(self) -> None: from zerver.decorator import InvalidZulipServerError + # This is the exception our decorator uses for an invalid Zulip server error_obj = InvalidZulipServerError("testRole") with mock.patch('requests.request', diff --git a/zerver/tests/test_queue.py b/zerver/tests/test_queue.py index 48f36d0f7f..d08b43e172 100644 --- a/zerver/tests/test_queue.py +++ b/zerver/tests/test_queue.py @@ -1,13 +1,13 @@ -from unittest import mock from typing import Any, Dict +from unittest import mock from django.test import override_settings -from pika.exceptions import ConnectionClosed, AMQPConnectionError +from pika.exceptions import AMQPConnectionError, ConnectionClosed -from zerver.lib.queue import TornadoQueueClient, queue_json_publish, \ - get_queue_client +from zerver.lib.queue import TornadoQueueClient, get_queue_client, queue_json_publish from zerver.lib.test_classes import ZulipTestCase + class TestTornadoQueueClient(ZulipTestCase): @mock.patch('zerver.lib.queue.logging.getLogger', autospec=True) @mock.patch('zerver.lib.queue.ExceptionFreeTornadoConnection', autospec=True) diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index 2bde78a4c6..4bfff4fdf5 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -1,12 +1,12 @@ import os -import time -import ujson import smtplib +import time +from typing import Any, Callable, Dict, List, Mapping, Tuple +from unittest.mock import MagicMock, patch +import ujson from django.conf import settings from django.test import override_settings -from unittest.mock import patch, MagicMock -from typing import Any, Callable, Dict, List, Mapping, Tuple from zerver.lib.email_mirror import RateLimitedRealmMirror from zerver.lib.email_mirror_helpers import encode_email_address @@ -14,18 +14,17 @@ from zerver.lib.queue import MAX_REQUEST_RETRIES from zerver.lib.rate_limiter import RateLimiterLockingException from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError from zerver.lib.send_email import FromAddress -from zerver.lib.test_helpers import simulated_queue_client from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_client, UserActivity, PreregistrationUser, \ - get_stream, get_realm +from zerver.lib.test_helpers import simulated_queue_client +from zerver.models import PreregistrationUser, UserActivity, get_client, get_realm, get_stream from zerver.tornado.event_queue import build_offline_notification from zerver.worker import queue_processors from zerver.worker.queue_processors import ( - get_active_worker_queues, - QueueProcessingWorker, EmailSendingWorker, LoopQueueProcessingWorker, MissedMessageWorker, + QueueProcessingWorker, + get_active_worker_queues, ) Event = Dict[str, Any] diff --git a/zerver/tests/test_rate_limiter.py b/zerver/tests/test_rate_limiter.py index 7afa2379f4..898af5fb37 100644 --- a/zerver/tests/test_rate_limiter.py +++ b/zerver/tests/test_rate_limiter.py @@ -1,21 +1,19 @@ +import time +from typing import Dict, List, Tuple, Type +from unittest import mock + from zerver.lib.rate_limiter import ( - add_ratelimit_rule, - remove_ratelimit_rule, RateLimitedObject, RateLimitedUser, RateLimiterBackend, RedisRateLimiterBackend, TornadoInMemoryRateLimiterBackend, + add_ratelimit_rule, + remove_ratelimit_rule, ) - from zerver.lib.test_classes import ZulipTestCase from zerver.lib.utils import generate_random_token -from typing import Dict, List, Tuple, Type - -from unittest import mock -import time - RANDOM_KEY_PREFIX = generate_random_token(32) class RateLimitedTestObject(RateLimitedObject): diff --git a/zerver/tests/test_reactions.py b/zerver/tests/test_reactions.py index 9e1ad910ee..ff349a3fcc 100644 --- a/zerver/tests/test_reactions.py +++ b/zerver/tests/test_reactions.py @@ -1,15 +1,17 @@ -import ujson -from django.http import HttpResponse from typing import Any, Dict, List, Mapping from unittest import mock -from zerver.lib.cache import to_dict_cache_key_id, cache_get +import ujson +from django.http import HttpResponse + +from zerver.lib.cache import cache_get, to_dict_cache_key_id from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.lib.message import extract_message_dict from zerver.lib.request import JsonableError -from zerver.lib.test_helpers import tornado_redirected_to_list from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_realm, Message, Reaction, RealmEmoji, UserMessage +from zerver.lib.test_helpers import tornado_redirected_to_list +from zerver.models import Message, Reaction, RealmEmoji, UserMessage, get_realm + class ReactionEmojiTest(ZulipTestCase): def test_missing_emoji(self) -> None: diff --git a/zerver/tests/test_realm.py b/zerver/tests/test_realm.py index 89a514cceb..e571281bde 100644 --- a/zerver/tests/test_realm.py +++ b/zerver/tests/test_realm.py @@ -1,36 +1,43 @@ import datetime -import ujson import re -from unittest import mock from email.utils import parseaddr - -from django.conf import settings from typing import Any, Dict, List, Mapping +from unittest import mock +import ujson +from django.conf import settings + +from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.actions import ( + do_change_plan_type, do_change_realm_subdomain, - do_set_realm_property, + do_create_realm, do_deactivate_realm, do_deactivate_stream, - do_create_realm, do_scrub_realm, - do_change_plan_type, do_send_realm_reactivation_email, + do_set_realm_property, ) - -from confirmation.models import create_confirmation_link, Confirmation from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description from zerver.lib.send_email import send_future_email from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - reset_emails_in_zulip_realm, - tornado_redirected_to_list, -) +from zerver.lib.test_helpers import reset_emails_in_zulip_realm, tornado_redirected_to_list from zerver.lib.test_runner import slow -from zerver.models import get_realm, Realm, UserProfile, ScheduledEmail, get_stream, \ - CustomProfileField, Message, UserMessage, Attachment, get_user_profile_by_email, \ - get_user_profile_by_id +from zerver.models import ( + Attachment, + CustomProfileField, + Message, + Realm, + ScheduledEmail, + UserMessage, + UserProfile, + get_realm, + get_stream, + get_user_profile_by_email, + get_user_profile_by_id, +) + class RealmTest(ZulipTestCase): def assert_user_profile_cache_gets_new_name(self, user_profile: UserProfile, diff --git a/zerver/tests/test_realm_domains.py b/zerver/tests/test_realm_domains.py index 872fd78f9d..65a64c8d7e 100644 --- a/zerver/tests/test_realm_domains.py +++ b/zerver/tests/test_realm_domains.py @@ -1,16 +1,18 @@ +import ujson from django.core.exceptions import ValidationError from django.db.utils import IntegrityError -from zerver.lib.actions import do_change_user_role, \ - do_change_realm_domain, do_create_realm, \ - do_remove_realm_domain, do_set_realm_property -from zerver.lib.email_validation import email_allowed_for_realm +from zerver.lib.actions import ( + do_change_realm_domain, + do_change_user_role, + do_create_realm, + do_remove_realm_domain, + do_set_realm_property, +) from zerver.lib.domains import validate_domain +from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_realm, UserProfile, \ - RealmDomain, DomainNotAllowedForRealmError - -import ujson +from zerver.models import DomainNotAllowedForRealmError, RealmDomain, UserProfile, get_realm class RealmDomainTest(ZulipTestCase): diff --git a/zerver/tests/test_realm_emoji.py b/zerver/tests/test_realm_emoji.py index 0a672339c7..4874fb1fa9 100644 --- a/zerver/tests/test_realm_emoji.py +++ b/zerver/tests/test_realm_emoji.py @@ -1,11 +1,11 @@ from unittest import mock -from zerver.lib.actions import do_create_realm, do_create_user, \ - check_add_realm_emoji +from zerver.lib.actions import check_add_realm_emoji, do_create_realm, do_create_user from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file from zerver.models import Realm, RealmEmoji, UserProfile, get_realm + class RealmEmojiTest(ZulipTestCase): def create_test_emoji(self, name: str, author: UserProfile) -> RealmEmoji: diff --git a/zerver/tests/test_realm_export.py b/zerver/tests/test_realm_export.py index 0c0a040699..e9af3781dd 100644 --- a/zerver/tests/test_realm_export.py +++ b/zerver/tests/test_realm_export.py @@ -1,21 +1,23 @@ +import os from unittest.mock import patch -from analytics.models import RealmCount - -from django.utils.timezone import now as timezone_now +import botocore.exceptions +import ujson from django.conf import settings +from django.utils.timezone import now as timezone_now -from zerver.lib.test_classes import ZulipTestCase +from analytics.models import RealmCount from zerver.lib.exceptions import JsonableError -from zerver.lib.test_helpers import use_s3_backend, create_s3_buckets, \ - create_dummy_file, stdout_suppressed - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + create_dummy_file, + create_s3_buckets, + stdout_suppressed, + use_s3_backend, +) from zerver.models import RealmAuditLog from zerver.views.realm_export import export_realm -import os -import ujson -import botocore.exceptions class RealmExportTest(ZulipTestCase): """ diff --git a/zerver/tests/test_realm_filters.py b/zerver/tests/test_realm_filters.py index f94c2b1881..7b93ece2c5 100644 --- a/zerver/tests/test_realm_filters.py +++ b/zerver/tests/test_realm_filters.py @@ -4,6 +4,7 @@ from zerver.lib.actions import do_add_realm_filter from zerver.lib.test_classes import ZulipTestCase from zerver.models import RealmFilter, get_realm + class RealmFilterTest(ZulipTestCase): def test_list(self) -> None: diff --git a/zerver/tests/test_redis_utils.py b/zerver/tests/test_redis_utils.py index db15268f61..c69a4de40c 100644 --- a/zerver/tests/test_redis_utils.py +++ b/zerver/tests/test_redis_utils.py @@ -1,8 +1,15 @@ from unittest import mock +from zerver.lib.redis_utils import ( + MAX_KEY_LENGTH, + ZulipRedisKeyOfWrongFormatError, + ZulipRedisKeyTooLongError, + get_dict_from_redis, + get_redis_client, + put_dict_in_redis, +) from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.redis_utils import get_redis_client, get_dict_from_redis, put_dict_in_redis, \ - ZulipRedisKeyTooLongError, ZulipRedisKeyOfWrongFormatError, MAX_KEY_LENGTH + class RedisUtilsTest(ZulipTestCase): key_format = "test_redis_utils_{token}" diff --git a/zerver/tests/test_report.py b/zerver/tests/test_report.py index 714e70adfe..46d3b4f680 100644 --- a/zerver/tests/test_report.py +++ b/zerver/tests/test_report.py @@ -1,13 +1,12 @@ from typing import Any, Callable, Dict, Iterable, List, Tuple +from unittest import mock +import ujson from django.test import override_settings -from zerver.lib.test_classes import ( - ZulipTestCase, -) + +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.utils import statsd -from unittest import mock -import ujson def fix_params(raw_params: Dict[str, Any]) -> Dict[str, str]: # A few of our few legacy endpoints need their diff --git a/zerver/tests/test_retention.py b/zerver/tests/test_retention.py index 056909aec8..192ab15455 100644 --- a/zerver/tests/test_retention.py +++ b/zerver/tests/test_retention.py @@ -5,25 +5,39 @@ from unittest import mock from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.actions import internal_send_private_message, do_add_submessage, do_delete_messages +from zerver.lib.actions import do_add_submessage, do_delete_messages, internal_send_private_message +from zerver.lib.retention import ( + archive_messages, + clean_archived_data, + get_realms_and_streams_for_archiving, + move_messages_to_archive, + restore_all_data_from_archive, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured from zerver.lib.upload import create_attachment -from zerver.models import (Message, Realm, Stream, ArchivedUserMessage, SubMessage, - ArchivedMessage, Attachment, ArchivedAttachment, UserMessage, - Reaction, ArchivedReaction, ArchivedSubMessage, ArchiveTransaction, - get_realm, get_user_profile_by_email, get_stream, get_system_bot) -from zerver.lib.retention import ( - archive_messages, - move_messages_to_archive, - restore_all_data_from_archive, - clean_archived_data, - get_realms_and_streams_for_archiving, +from zerver.models import ( + ArchivedAttachment, + ArchivedMessage, + ArchivedReaction, + ArchivedSubMessage, + ArchivedUserMessage, + ArchiveTransaction, + Attachment, + Message, + Reaction, + Realm, + Stream, + SubMessage, + UserMessage, + get_realm, + get_stream, + get_system_bot, + get_user_profile_by_email, ) -from zerver.tornado.event_queue import send_event - # Class with helper functions useful for testing archiving of reactions: from zerver.tests.test_reactions import EmojiReactionBase +from zerver.tornado.event_queue import send_event ZULIP_REALM_DAYS = 30 MIT_REALM_DAYS = 100 diff --git a/zerver/tests/test_service_bot_system.py b/zerver/tests/test_service_bot_system.py index 8867ef75d2..6978640246 100644 --- a/zerver/tests/test_service_bot_system.py +++ b/zerver/tests/test_service_bot_system.py @@ -1,25 +1,16 @@ +from typing import Any, Callable, Mapping, Union from unittest import mock -from typing import Any, Union, Mapping, Callable +import ujson from django.conf import settings from django.test import override_settings -from zerver.lib.actions import ( - do_create_user, - get_service_bot_events, -) -from zerver.lib.bot_lib import StateHandler, EmbeddedBotHandler, \ - EmbeddedBotEmptyRecipientsList +from zerver.lib.actions import do_create_user, get_service_bot_events +from zerver.lib.bot_config import ConfigError, load_bot_config_template, set_bot_config +from zerver.lib.bot_lib import EmbeddedBotEmptyRecipientsList, EmbeddedBotHandler, StateHandler from zerver.lib.bot_storage import StateError -from zerver.lib.bot_config import set_bot_config, ConfigError, load_bot_config_template from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - get_realm, - UserProfile, - Recipient, -) - -import ujson +from zerver.models import Recipient, UserProfile, get_realm BOT_TYPE_TO_QUEUE_NAME = { UserProfile.OUTGOING_WEBHOOK_BOT: 'outgoing_webhooks', diff --git a/zerver/tests/test_sessions.py b/zerver/tests/test_sessions.py index 9b4daf3294..5b096201a4 100644 --- a/zerver/tests/test_sessions.py +++ b/zerver/tests/test_sessions.py @@ -1,25 +1,22 @@ from datetime import timedelta -from django.utils.timezone import now as timezone_now from typing import Any, Callable +from unittest import mock + +from django.utils.timezone import now as timezone_now from zerver.lib.sessions import ( - user_sessions, + delete_all_deactivated_user_sessions, + delete_all_user_sessions, + delete_realm_user_sessions, delete_session, delete_user_sessions, - delete_realm_user_sessions, - delete_all_user_sessions, - delete_all_deactivated_user_sessions, get_expirable_session_var, set_expirable_session_var, + user_sessions, ) - -from zerver.models import ( - get_realm, Realm, UserProfile, -) - from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Realm, UserProfile, get_realm -from unittest import mock class TestSessions(ZulipTestCase): diff --git a/zerver/tests/test_settings.py b/zerver/tests/test_settings.py index 4b70aa01d0..246ae6f687 100644 --- a/zerver/tests/test_settings.py +++ b/zerver/tests/test_settings.py @@ -1,20 +1,18 @@ -from unittest import mock import time -import ujson +from typing import Any, Dict +from unittest import mock +import ujson from django.http import HttpResponse from django.test import override_settings -from typing import Any, Dict from zerver.lib.initial_password import initial_password +from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file from zerver.lib.users import get_all_api_keys -from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule -from zerver.models import ( - UserProfile, - get_user_profile_by_api_key, -) +from zerver.models import UserProfile, get_user_profile_by_api_key + class ChangeSettingsTest(ZulipTestCase): diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.py index 242e007862..9bc375b8ac 100644 --- a/zerver/tests/test_signup.py +++ b/zerver/tests/test_signup.py @@ -1,86 +1,102 @@ import datetime -from email.utils import parseaddr - -from django.conf import settings -from django.contrib.auth.views import INTERNAL_RESET_URL_TOKEN -from django.contrib.contenttypes.models import ContentType -from django.http import HttpResponse -from django.test import TestCase, override_settings -from django.utils.timezone import now as timezone_now -from django.core.exceptions import ValidationError -from django.urls import reverse - -from unittest.mock import patch, MagicMock -from zerver.lib.test_helpers import ( - avatar_disk_path, - get_test_image_file, - reset_emails_in_zulip_realm, -) - -from confirmation.models import Confirmation, create_confirmation_link, MultiuseInvite, \ - generate_key, confirmation_url, get_object_from_key, ConfirmationKeyException, \ - one_click_unsubscribe_link -from confirmation import settings as confirmation_settings - -from zerver.forms import HomepageForm, check_subdomain_available -from zerver.decorator import do_two_factor_login -from zerver.views.auth import \ - redirect_and_log_into_subdomain, start_two_factor_auth -from zerver.views.invite import get_invitee_emails_set -from zerver.views.development.registration import confirmation_key - -from zerver.models import ( - get_realm, get_user, CustomProfileField, - CustomProfileFieldValue, DefaultStream, PreregistrationUser, - Realm, Recipient, Message, ScheduledEmail, UserProfile, UserMessage, - Stream, Subscription, flush_per_request_caches, get_system_bot, - get_user_by_delivery_email, -) -from zerver.lib.actions import ( - do_change_user_role, - get_stream, - do_create_default_stream_group, - do_add_default_stream, - do_create_realm, - get_default_streams_for_realm, - do_invite_users, do_create_user) -from zerver.lib.send_email import send_future_email, FromAddress, \ - deliver_email -from zerver.lib.initial_password import initial_password -from zerver.lib.actions import ( - do_get_user_invites, - do_change_full_name, - do_deactivate_realm, - do_deactivate_user, - do_set_realm_property, - add_new_user_history, -) -from zerver.lib.mobile_auth_otp import xor_hex_strings, ascii_to_hex, \ - otp_encrypt_api_key, is_valid_otp, hex_to_ascii, otp_decrypt_api_key -from zerver.lib.email_notifications import enqueue_welcome_emails, \ - followup_day2_email_delay -from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule -from zerver.lib.subdomains import is_root_domain_available -from zerver.lib.stream_subscription import get_stream_subscriptions_for_user -from zerver.lib.streams import create_stream_if_needed -from zerver.lib.test_helpers import find_key_by_email, queries_captured, \ - HostRequestMock, load_subdomain_token -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.name_restrictions import is_disposable_domain -from zerver.context_processors import common_context - -from zproject.backends import ExternalAuthResult, ExternalAuthDataDict - import re import smtplib import time -import ujson - -from typing import Any, List, Optional - import urllib +from email.utils import parseaddr +from typing import Any, List, Optional +from unittest.mock import MagicMock, patch + +import ujson +from django.conf import settings +from django.contrib.auth.views import INTERNAL_RESET_URL_TOKEN +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.http import HttpResponse +from django.test import TestCase, override_settings +from django.urls import reverse +from django.utils.timezone import now as timezone_now + +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + MultiuseInvite, + confirmation_url, + create_confirmation_link, + generate_key, + get_object_from_key, + one_click_unsubscribe_link, +) +from zerver.context_processors import common_context +from zerver.decorator import do_two_factor_login +from zerver.forms import HomepageForm, check_subdomain_available +from zerver.lib.actions import ( + add_new_user_history, + do_add_default_stream, + do_change_full_name, + do_change_user_role, + do_create_default_stream_group, + do_create_realm, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_get_user_invites, + do_invite_users, + do_set_realm_property, + get_default_streams_for_realm, + get_stream, +) +from zerver.lib.email_notifications import enqueue_welcome_emails, followup_day2_email_delay +from zerver.lib.initial_password import initial_password +from zerver.lib.mobile_auth_otp import ( + ascii_to_hex, + hex_to_ascii, + is_valid_otp, + otp_decrypt_api_key, + otp_encrypt_api_key, + xor_hex_strings, +) +from zerver.lib.name_restrictions import is_disposable_domain +from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule +from zerver.lib.send_email import FromAddress, deliver_email, send_future_email +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user +from zerver.lib.streams import create_stream_if_needed +from zerver.lib.subdomains import is_root_domain_available +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + HostRequestMock, + avatar_disk_path, + find_key_by_email, + get_test_image_file, + load_subdomain_token, + queries_captured, + reset_emails_in_zulip_realm, +) +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Message, + PreregistrationUser, + Realm, + Recipient, + ScheduledEmail, + Stream, + Subscription, + UserMessage, + UserProfile, + flush_per_request_caches, + get_realm, + get_system_bot, + get_user, + get_user_by_delivery_email, +) +from zerver.views.auth import redirect_and_log_into_subdomain, start_two_factor_auth +from zerver.views.development.registration import confirmation_key +from zerver.views.invite import get_invitee_emails_set +from zproject.backends import ExternalAuthDataDict, ExternalAuthResult + class RedirectAndLogIntoSubdomainTestCase(ZulipTestCase): def test_data(self) -> None: @@ -3968,6 +3984,7 @@ class TestFindMyTeam(ZulipTestCase): self.assertIn(self.example_email("iago"), content) self.assertIn(self.example_email("cordelia"), content) from django.core.mail import outbox + # 3 = 1 + 2 -- Cordelia gets an email each for the "zulip" and "lear" realms. self.assertEqual(len(outbox), 3) diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 3d9b5df545..3e0ec99122 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -1,64 +1,49 @@ +import logging +import os +import shutil +from typing import Any, Dict, Iterator, List, Set, Tuple +from unittest import mock +from unittest.mock import ANY, call + +import ujson from django.conf import settings from django.utils.timezone import now as timezone_now +from zerver.data_import.import_util import ( + build_defaultstream, + build_recipient, + build_subscription, + build_usermessages, + build_zerver_realm, +) +from zerver.data_import.sequencer import NEXT_ID from zerver.data_import.slack import ( - get_slack_api_data, - get_owner, - get_admin, - get_guest, - get_user_timezone, - fetch_shared_channel_users, - users_to_zerver_userprofile, - get_subscription, - channels_to_zerver_stream, - slack_workspace_to_realm, - get_message_sending_user, - channel_message_to_zerver_message, - convert_slack_workspace_messages, - do_convert_data, - process_message_files, AddedChannelsT, AddedMPIMsT, DMMembersT, ZerverFieldsT, + channel_message_to_zerver_message, + channels_to_zerver_stream, + convert_slack_workspace_messages, + do_convert_data, + fetch_shared_channel_users, + get_admin, + get_guest, + get_message_sending_user, + get_owner, + get_slack_api_data, + get_subscription, + get_user_timezone, + process_message_files, + slack_workspace_to_realm, + users_to_zerver_userprofile, ) -from zerver.data_import.import_util import ( - build_zerver_realm, - build_subscription, - build_recipient, - build_usermessages, - build_defaultstream, -) -from zerver.data_import.sequencer import ( - NEXT_ID, -) -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - get_test_image_file, -) -from zerver.lib.topic import ( - EXPORT_TOPIC_NAME, -) -from zerver.models import ( - Realm, - get_realm, - RealmAuditLog, - Recipient, - UserProfile, -) +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_test_image_file +from zerver.lib.topic import EXPORT_TOPIC_NAME +from zerver.models import Realm, RealmAuditLog, Recipient, UserProfile, get_realm -import ujson -import logging -import shutil -import os -from unittest import mock -from unittest.mock import ANY, call -from typing import Any, Dict, List, Set, Tuple, Iterator def remove_folder(path: str) -> None: if os.path.exists(path): diff --git a/zerver/tests/test_slack_message_conversion.py b/zerver/tests/test_slack_message_conversion.py index 11c6154590..2fc2063e0a 100644 --- a/zerver/tests/test_slack_message_conversion.py +++ b/zerver/tests/test_slack_message_conversion.py @@ -1,16 +1,16 @@ +import os +from typing import Any, Dict, List, Tuple + +import ujson + from zerver.data_import.slack_message_conversion import ( convert_to_zulip_markdown, get_user_full_name, ) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_runner import slow from zerver.lib import mdiff -import ujson +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_runner import slow -import os -from typing import Any, Dict, List, Tuple class SlackMessageConversion(ZulipTestCase): def assertEqual(self, first: Any, second: Any, msg: str="") -> None: diff --git a/zerver/tests/test_soft_deactivation.py b/zerver/tests/test_soft_deactivation.py index c7eda6dba8..45fb69fb79 100644 --- a/zerver/tests/test_soft_deactivation.py +++ b/zerver/tests/test_soft_deactivation.py @@ -3,18 +3,17 @@ from unittest import mock from django.utils.timezone import now as timezone_now from zerver.lib.soft_deactivation import ( + do_auto_soft_deactivate_users, + do_catch_up_soft_deactivated_users, + do_soft_activate_users, do_soft_deactivate_user, do_soft_deactivate_users, - get_users_for_soft_deactivation, - do_soft_activate_users, get_soft_deactivated_users_for_catch_up, - do_catch_up_soft_deactivated_users, - do_auto_soft_deactivate_users, + get_users_for_soft_deactivation, ) from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - Client, UserProfile, UserActivity, get_realm, UserMessage, -) +from zerver.models import Client, UserActivity, UserMessage, UserProfile, get_realm + class UserSoftDeactivationTests(ZulipTestCase): diff --git a/zerver/tests/test_subdomains.py b/zerver/tests/test_subdomains.py index 2b6a37f875..25ad1edd90 100644 --- a/zerver/tests/test_subdomains.py +++ b/zerver/tests/test_subdomains.py @@ -1,11 +1,12 @@ -from unittest import mock from typing import Any, Dict, List +from unittest import mock from django.test import TestCase from zerver.lib.subdomains import get_subdomain from zerver.models import Realm + class SubdomainsTest(TestCase): def test_get_subdomain(self) -> None: diff --git a/zerver/tests/test_submessage.py b/zerver/tests/test_submessage.py index e3785c599c..e58620005b 100644 --- a/zerver/tests/test_submessage.py +++ b/zerver/tests/test_submessage.py @@ -1,17 +1,10 @@ +from typing import Any, Dict, List from unittest import mock +from zerver.lib.message import MessageDict from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, SubMessage -from zerver.lib.message import ( - MessageDict, -) - -from zerver.models import ( - Message, - SubMessage, -) - -from typing import Any, Dict, List class TestBasics(ZulipTestCase): def test_get_raw_db_rows(self) -> None: diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py index 6257055dcb..ffcfda63fe 100644 --- a/zerver/tests/test_subs.py +++ b/zerver/tests/test_subs.py @@ -1,87 +1,89 @@ +import random +from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Set, Union +from unittest import mock +import ujson from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now +from zerver.decorator import JsonableError from zerver.lib import cache - -from zerver.lib.test_helpers import ( - get_subscription, queries_captured, tornado_redirected_to_list, - reset_emails_in_zulip_realm, +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_get_subscriber_user_ids, + bulk_remove_subscriptions, + can_access_stream_user_ids, + create_stream_if_needed, + do_add_default_stream, + do_add_streams_to_default_stream_group, + do_change_default_stream_group_description, + do_change_default_stream_group_name, + do_change_stream_post_policy, + do_change_user_role, + do_create_default_stream_group, + do_create_realm, + do_deactivate_stream, + do_deactivate_user, + do_get_streams, + do_remove_default_stream, + do_remove_default_stream_group, + do_remove_streams_from_default_stream_group, + do_set_realm_property, + ensure_stream, + gather_subscriptions, + gather_subscriptions_helper, + get_average_weekly_stream_traffic, + get_default_streams_for_realm, + get_stream, + lookup_default_stream_groups, + round_to_2_significant_digits, + validate_user_access_to_subscribers_helper, ) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.decorator import ( - JsonableError, -) - -from zerver.lib.response import ( - json_error, - json_success, -) - -from zerver.lib.streams import ( - access_stream_by_id, access_stream_by_name, filter_stream_authorization, - list_to_streams, create_streams_if_needed, -) - +from zerver.lib.message import aggregate_unread_data, get_raw_unread_data +from zerver.lib.response import json_error, json_success +from zerver.lib.stream_recipient import StreamRecipientMap from zerver.lib.stream_subscription import ( get_active_subscriptions_for_stream_id, num_subscribers_for_stream_id, ) - -from zerver.lib.test_runner import ( - slow, +from zerver.lib.streams import ( + access_stream_by_id, + access_stream_by_name, + create_streams_if_needed, + filter_stream_authorization, + list_to_streams, ) - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + get_subscription, + queries_captured, + reset_emails_in_zulip_realm, + tornado_redirected_to_list, +) +from zerver.lib.test_runner import slow from zerver.models import ( - Realm, Recipient, Stream, Subscription, - DefaultStream, UserProfile, active_non_guest_user_ids, - get_default_stream_groups, flush_per_request_caches, DefaultStreamGroup, - get_client, get_realm, get_user, get_user_profile_by_id_in_realm, Message, UserMessage, + DefaultStream, + DefaultStreamGroup, + Message, + Realm, + Recipient, + Stream, + Subscription, + UserMessage, + UserProfile, + active_non_guest_user_ids, + flush_per_request_caches, + get_client, + get_default_stream_groups, + get_realm, + get_user, + get_user_profile_by_id_in_realm, ) +from zerver.views.streams import compose_views -from zerver.lib.actions import ( - do_add_default_stream, do_change_user_role, do_set_realm_property, - do_create_realm, do_remove_default_stream, bulk_get_subscriber_user_ids, - gather_subscriptions_helper, bulk_add_subscriptions, bulk_remove_subscriptions, - gather_subscriptions, get_default_streams_for_realm, get_stream, - do_get_streams, - create_stream_if_needed, - ensure_stream, - do_deactivate_stream, - do_deactivate_user, - do_create_default_stream_group, - do_add_streams_to_default_stream_group, do_remove_streams_from_default_stream_group, - do_remove_default_stream_group, - do_change_default_stream_group_description, - do_change_default_stream_group_name, - lookup_default_stream_groups, - can_access_stream_user_ids, - validate_user_access_to_subscribers_helper, - get_average_weekly_stream_traffic, round_to_2_significant_digits, - do_change_stream_post_policy, -) - -from zerver.views.streams import ( - compose_views, -) - -from zerver.lib.message import ( - aggregate_unread_data, - get_raw_unread_data, -) -from zerver.lib.stream_recipient import StreamRecipientMap - -from datetime import timedelta -from unittest import mock -import random -import ujson class TestMiscStuff(ZulipTestCase): def test_empty_results(self) -> None: diff --git a/zerver/tests/test_templates.py b/zerver/tests/test_templates.py index a35d282ab4..b5cdc86ee8 100644 --- a/zerver/tests/test_templates.py +++ b/zerver/tests/test_templates.py @@ -1,9 +1,7 @@ from django.template.loader import get_template from zerver.lib.exceptions import InvalidMarkdownIncludeStatement -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase class TemplateTestCase(ZulipTestCase): diff --git a/zerver/tests/test_thumbnail.py b/zerver/tests/test_thumbnail.py index 6653093bea..d707a28cf4 100644 --- a/zerver/tests/test_thumbnail.py +++ b/zerver/tests/test_thumbnail.py @@ -1,19 +1,20 @@ +import base64 +import urllib +from io import StringIO + +import ujson from django.conf import settings from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( - use_s3_backend, create_s3_buckets, - override_settings, get_test_image_file, + override_settings, + use_s3_backend, ) from zerver.lib.upload import upload_backend, upload_emoji_image from zerver.lib.users import get_api_key -from io import StringIO -import ujson -import urllib -import base64 class ThumbnailTest(ZulipTestCase): diff --git a/zerver/tests/test_timestamp.py b/zerver/tests/test_timestamp.py index cd2aa739c5..5460d510b5 100644 --- a/zerver/tests/test_timestamp.py +++ b/zerver/tests/test_timestamp.py @@ -1,11 +1,19 @@ -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.timestamp import floor_to_hour, floor_to_day, ceiling_to_hour, \ - timestamp_to_datetime, datetime_to_timestamp, \ - TimezoneNotUTCException, convert_to_UTC - from datetime import timedelta, timezone + from dateutil import parser +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.timestamp import ( + TimezoneNotUTCException, + ceiling_to_hour, + convert_to_UTC, + datetime_to_timestamp, + floor_to_day, + floor_to_hour, + timestamp_to_datetime, +) + + class TestTimestamp(ZulipTestCase): def test_datetime_and_timestamp_conversions(self) -> None: timestamp = 1483228800 diff --git a/zerver/tests/test_tornado.py b/zerver/tests/test_tornado.py index 9b377dddf6..bdb519e4b2 100644 --- a/zerver/tests/test_tornado.py +++ b/zerver/tests/test_tornado.py @@ -1,22 +1,20 @@ +import urllib.parse +from typing import Any, Dict, List, Optional, cast + import ujson from django.conf import settings -from django.db import close_old_connections from django.core import signals +from django.db import close_old_connections from django.test import override_settings from tornado.httpclient import HTTPResponse - -from zerver.lib.test_classes import ZulipTestCase - from tornado.testing import AsyncHTTPTestCase from tornado.web import Application -from zerver.tornado.application import create_tornado_application +from zerver.lib.test_classes import ZulipTestCase from zerver.tornado import event_queue +from zerver.tornado.application import create_tornado_application from zerver.tornado.event_queue import process_event -import urllib.parse - -from typing import Any, Dict, Optional, List, cast class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_transfer.py b/zerver/tests/test_transfer.py index ec642b2b63..f1274e74d0 100644 --- a/zerver/tests/test_transfer.py +++ b/zerver/tests/test_transfer.py @@ -1,18 +1,22 @@ -from django.conf import settings - -from moto import mock_s3 -from unittest.mock import Mock, patch import logging +from unittest.mock import Mock, patch + +from django.conf import settings +from moto import mock_s3 -from zerver.lib.upload import upload_message_file, resize_emoji -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, \ - avatar_disk_path -from zerver.lib.transfer import transfer_avatars_to_s3, transfer_emoji_to_s3, \ - transfer_message_files_to_s3, transfer_uploads_to_s3 -from zerver.models import Attachment, RealmEmoji from zerver.lib.actions import check_add_realm_emoji from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import avatar_disk_path, create_s3_buckets, get_test_image_file +from zerver.lib.transfer import ( + transfer_avatars_to_s3, + transfer_emoji_to_s3, + transfer_message_files_to_s3, + transfer_uploads_to_s3, +) +from zerver.lib.upload import resize_emoji, upload_message_file +from zerver.models import Attachment, RealmEmoji + class TransferUploadsToS3Test(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_tutorial.py b/zerver/tests/test_tutorial.py index 1ac2f6e06f..7332a25ee5 100644 --- a/zerver/tests/test_tutorial.py +++ b/zerver/tests/test_tutorial.py @@ -1,11 +1,11 @@ +import ujson from django.conf import settings from zerver.lib.actions import internal_send_private_message from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import message_stream_count, most_recent_message -from zerver.models import get_system_bot, UserProfile +from zerver.models import UserProfile, get_system_bot -import ujson class TutorialTests(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_type_debug.py b/zerver/tests/test_type_debug.py index 7c66fc11e3..9bc9fbd6e2 100644 --- a/zerver/tests/test_type_debug.py +++ b/zerver/tests/test_type_debug.py @@ -1,11 +1,10 @@ import sys -from unittest import TestCase from io import StringIO +from typing import Any, Callable, Dict, Iterable, List, Tuple, TypeVar +from unittest import TestCase from zerver.lib.type_debug import print_types -from typing import Any, Callable, Dict, Iterable, Tuple, TypeVar, List - T = TypeVar('T') def add(x: Any=0, y: Any=0) -> Any: diff --git a/zerver/tests/test_typing.py b/zerver/tests/test_typing.py index e85ca32e6a..750b293b7a 100644 --- a/zerver/tests/test_typing.py +++ b/zerver/tests/test_typing.py @@ -1,17 +1,11 @@ -import ujson -from typing import Any, Mapping, List +from typing import Any, List, Mapping + +import ujson + +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured, tornado_redirected_to_list +from zerver.models import Huddle, get_huddle_hash -from zerver.lib.test_helpers import ( - tornado_redirected_to_list, - queries_captured, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.models import ( - Huddle, - get_huddle_hash, -) class TypingValidateOperatorTest(ZulipTestCase): def test_missing_parameter(self) -> None: diff --git a/zerver/tests/test_unread.py b/zerver/tests/test_unread.py index a427d4988f..803e1cf739 100644 --- a/zerver/tests/test_unread.py +++ b/zerver/tests/test_unread.py @@ -1,32 +1,15 @@ from typing import Any, List, Mapping +from unittest import mock +import ujson from django.db import connection -from zerver.models import ( - get_realm, - get_stream, - get_user, - Subscription, - UserMessage, - UserProfile, -) - -from zerver.lib.fix_unreads import ( - fix, - fix_pre_pointer, - fix_unsubscribed, -) -from zerver.lib.test_helpers import ( - get_subscription, - tornado_redirected_to_list, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.fix_unreads import fix, fix_pre_pointer, fix_unsubscribed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_subscription, tornado_redirected_to_list from zerver.lib.topic_mutes import add_topic_mute +from zerver.models import Subscription, UserMessage, UserProfile, get_realm, get_stream, get_user -from unittest import mock -import ujson class PointerTest(ZulipTestCase): diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.py index 1e08e1ceef..fa0084658c 100644 --- a/zerver/tests/test_upload.py +++ b/zerver/tests/test_upload.py @@ -1,63 +1,80 @@ -from django.conf import settings -from django.test import TestCase +import datetime +import io +import os +import re +import shutil +import time +import urllib +from io import StringIO +from unittest import mock from unittest.mock import patch -from zerver.lib.avatar import ( - avatar_url, - get_avatar_field, +import botocore.exceptions +import ujson +from django.conf import settings +from django.test import TestCase +from django.utils.timezone import now as timezone_now +from django_sendfile.sendfile import _get_sendfile +from PIL import Image + +import zerver.lib.upload +from scripts.lib.zulip_tools import get_dev_uuid_var_path +from zerver.lib.actions import ( + do_change_icon_source, + do_change_logo_source, + do_change_plan_type, + do_delete_old_unclaimed_attachments, + do_set_realm_property, + internal_send_private_message, ) +from zerver.lib.avatar import avatar_url, get_avatar_field from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.cache import cache_get, get_realm_used_upload_space_cache_key +from zerver.lib.create_user import copy_user_settings from zerver.lib.initial_password import initial_password from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_url -from zerver.lib.test_classes import ZulipTestCase, UploadSerializeMixin +from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase from zerver.lib.test_helpers import ( avatar_disk_path, - get_test_image_file, - use_s3_backend, create_s3_buckets, + get_test_image_file, queries_captured, + use_s3_backend, ) -from zerver.lib.upload import sanitize_name, S3UploadBackend, \ - upload_message_file, upload_emoji_image, delete_message_image, LocalUploadBackend, \ - ZulipUploadBackend, MEDIUM_AVATAR_SIZE, resize_avatar, \ - resize_emoji, BadImageError, get_realm_for_filename, \ - DEFAULT_AVATAR_SIZE, DEFAULT_EMOJI_SIZE, exif_rotate, \ - upload_export_tarball, delete_export_tarball -import zerver.lib.upload -from zerver.models import Attachment, get_user_by_delivery_email, \ - Message, UserProfile, Realm, \ - RealmDomain, RealmEmoji, get_realm, get_system_bot, \ - validate_attachment_request -from zerver.lib.actions import ( - do_change_plan_type, - do_change_icon_source, - do_change_logo_source, - do_delete_old_unclaimed_attachments, - internal_send_private_message, - do_set_realm_property, +from zerver.lib.upload import ( + DEFAULT_AVATAR_SIZE, + DEFAULT_EMOJI_SIZE, + MEDIUM_AVATAR_SIZE, + BadImageError, + LocalUploadBackend, + S3UploadBackend, + ZulipUploadBackend, + delete_export_tarball, + delete_message_image, + exif_rotate, + get_realm_for_filename, + resize_avatar, + resize_emoji, + sanitize_name, + upload_emoji_image, + upload_export_tarball, + upload_message_file, ) -from zerver.lib.cache import get_realm_used_upload_space_cache_key, cache_get -from zerver.lib.create_user import copy_user_settings from zerver.lib.users import get_api_key +from zerver.models import ( + Attachment, + Message, + Realm, + RealmDomain, + RealmEmoji, + UserProfile, + get_realm, + get_system_bot, + get_user_by_delivery_email, + validate_attachment_request, +) -from scripts.lib.zulip_tools import get_dev_uuid_var_path - -import urllib -import ujson -from PIL import Image -import botocore.exceptions - -from io import StringIO -from unittest import mock -import os -import io -import shutil -import re -import time -import datetime -from django.utils.timezone import now as timezone_now -from django_sendfile.sendfile import _get_sendfile def destroy_uploads() -> None: if os.path.exists(settings.LOCAL_UPLOADS_DIR): diff --git a/zerver/tests/test_urls.py b/zerver/tests/test_urls.py index 60b4555fa0..10a2de74f9 100644 --- a/zerver/tests/test_urls.py +++ b/zerver/tests/test_urls.py @@ -1,16 +1,17 @@ import importlib import os -import ujson +from typing import List, Optional import django.urls.resolvers -from django.test import TestCase, Client -from typing import List, Optional +import ujson +from django.test import Client, TestCase from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_runner import slow from zerver.models import Stream from zproject import urls + class PublicURLTest(ZulipTestCase): """ Account creation URLs are accessible even when not logged in. Authenticated diff --git a/zerver/tests/test_user_groups.py b/zerver/tests/test_user_groups.py index 423a533dcb..3ac492c780 100644 --- a/zerver/tests/test_user_groups.py +++ b/zerver/tests/test_user_groups.py @@ -1,26 +1,21 @@ -import ujson from unittest import mock -from zerver.lib.actions import ( - ensure_stream, -) +import ujson +from zerver.lib.actions import do_set_realm_property, ensure_stream from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - most_recent_usermessage, -) -from zerver.lib.actions import do_set_realm_property +from zerver.lib.test_helpers import most_recent_usermessage from zerver.lib.user_groups import ( check_add_user_to_user_group, check_remove_user_from_user_group, create_user_group, + get_memberships_of_users, get_user_groups, user_groups_in_realm, - get_memberships_of_users, user_groups_in_realm_serialized, ) -from zerver.models import UserGroup, get_realm, Realm, \ - UserGroupMembership +from zerver.models import Realm, UserGroup, UserGroupMembership, get_realm + class UserGroupTestCase(ZulipTestCase): def create_user_group_for_test(self, group_name: str, diff --git a/zerver/tests/test_user_status.py b/zerver/tests/test_user_status.py index 051b7606c1..5c178c64f7 100644 --- a/zerver/tests/test_user_status.py +++ b/zerver/tests/test_user_status.py @@ -1,24 +1,12 @@ +from typing import Any, Dict, Set + import ujson -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - EventInfo, - capture_event, -) -from zerver.lib.user_status import ( - get_user_info_dict, - update_user_status, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import EventInfo, capture_event +from zerver.lib.user_status import get_user_info_dict, update_user_status +from zerver.models import UserProfile, UserStatus, get_client -from zerver.models import ( - get_client, - UserProfile, - UserStatus, -) - -from typing import Any, Dict, Set def get_away_user_ids(realm_id: int) -> Set[int]: user_dict = get_user_info_dict(realm_id) diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py index cc4a7278e7..cf9213d117 100644 --- a/zerver/tests/test_users.py +++ b/zerver/tests/test_users.py @@ -1,8 +1,29 @@ +import datetime from email.utils import parseaddr +from typing import Any, Dict, Iterable, List, Mapping, Optional, TypeVar, Union +from unittest import mock -from typing import (Any, Dict, Iterable, List, Mapping, - Optional, TypeVar, Union) +import ujson +from django.conf import settings +from django.test import override_settings +from zerver.lib.actions import ( + create_users, + do_change_user_role, + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_set_realm_property, + get_emails_from_user_ids, + get_recipient_info, +) +from zerver.lib.avatar import avatar_url, get_gravatar_url +from zerver.lib.create_user import copy_user_settings +from zerver.lib.events import do_events_register +from zerver.lib.exceptions import JsonableError +from zerver.lib.send_email import clear_scheduled_emails, deliver_email, send_future_email +from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( get_subscription, most_recent_message, @@ -11,45 +32,28 @@ from zerver.lib.test_helpers import ( simulated_empty_cache, tornado_redirected_to_list, ) -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import UserProfile, Recipient, Realm, \ - RealmDomain, UserHotspot, get_client, \ - get_user, get_user_by_delivery_email, get_realm, get_stream, \ - get_source_profile, get_system_bot, \ - ScheduledEmail, check_valid_user_ids, \ - get_user_by_id_in_realm_including_cross_realm, CustomProfileField, \ - InvalidFakeEmailDomain, get_fake_email_domain - -from zerver.lib.avatar import avatar_url, get_gravatar_url -from zerver.lib.exceptions import JsonableError -from zerver.lib.send_email import send_future_email, clear_scheduled_emails, \ - deliver_email -from zerver.lib.actions import ( - create_users, - get_emails_from_user_ids, - get_recipient_info, - do_deactivate_user, - do_reactivate_user, - do_change_user_role, - do_create_user, - do_set_realm_property, -) -from zerver.lib.create_user import copy_user_settings -from zerver.lib.events import do_events_register from zerver.lib.topic_mutes import add_topic_mute -from zerver.lib.stream_topic import StreamTopicTarget -from zerver.lib.users import user_ids_to_users, access_user_by_id, \ - get_accounts_for_email - -from django.conf import settings -from django.test import override_settings - -import datetime -from unittest import mock -import ujson +from zerver.lib.users import access_user_by_id, get_accounts_for_email, user_ids_to_users +from zerver.models import ( + CustomProfileField, + InvalidFakeEmailDomain, + Realm, + RealmDomain, + Recipient, + ScheduledEmail, + UserHotspot, + UserProfile, + check_valid_user_ids, + get_client, + get_fake_email_domain, + get_realm, + get_source_profile, + get_stream, + get_system_bot, + get_user, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, +) K = TypeVar('K') V = TypeVar('V') diff --git a/zerver/tests/test_webhooks_common.py b/zerver/tests/test_webhooks_common.py index cf4b3ebeeb..a842b230b1 100644 --- a/zerver/tests/test_webhooks_common.py +++ b/zerver/tests/test_webhooks_common.py @@ -1,20 +1,24 @@ from types import SimpleNamespace -from unittest.mock import MagicMock, patch from typing import Dict +from unittest.mock import MagicMock, patch from django.http import HttpRequest from zerver.decorator import api_key_only_webhook_view from zerver.lib.exceptions import InvalidJSONError, JsonableError -from zerver.lib.test_classes import ZulipTestCase, WebhookTestCase -from zerver.lib.webhooks.common import \ - validate_extract_webhook_http_header, \ - MISSING_EVENT_HEADER_MESSAGE, MissingHTTPEventHeader, \ - INVALID_JSON_MESSAGE, get_fixture_http_headers, standardize_headers -from zerver.models import get_user, get_realm, UserProfile -from zerver.lib.users import get_api_key from zerver.lib.send_email import FromAddress +from zerver.lib.test_classes import WebhookTestCase, ZulipTestCase from zerver.lib.test_helpers import HostRequestMock +from zerver.lib.users import get_api_key +from zerver.lib.webhooks.common import ( + INVALID_JSON_MESSAGE, + MISSING_EVENT_HEADER_MESSAGE, + MissingHTTPEventHeader, + get_fixture_http_headers, + standardize_headers, + validate_extract_webhook_http_header, +) +from zerver.models import UserProfile, get_realm, get_user class WebhooksCommonTestCase(ZulipTestCase): diff --git a/zerver/tests/test_widgets.py b/zerver/tests/test_widgets.py index 9218277933..d362930c30 100644 --- a/zerver/tests/test_widgets.py +++ b/zerver/tests/test_widgets.py @@ -1,14 +1,12 @@ +from typing import Any, Dict + import ujson -from typing import Dict, Any - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.validator import check_widget_content +from zerver.lib.widget import get_widget_data from zerver.models import SubMessage -from zerver.lib.test_classes import ZulipTestCase - -from zerver.lib.widget import get_widget_data - -from zerver.lib.validator import check_widget_content class WidgetContentTestCase(ZulipTestCase): def test_validation(self) -> None: diff --git a/zerver/tests/test_zcommand.py b/zerver/tests/test_zcommand.py index 31165272e2..08c1ca7d16 100644 --- a/zerver/tests/test_zcommand.py +++ b/zerver/tests/test_zcommand.py @@ -1,6 +1,5 @@ -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase + class ZcommandTest(ZulipTestCase): diff --git a/zerver/tests/test_zephyr.py b/zerver/tests/test_zephyr.py index fef809b576..dae495d455 100644 --- a/zerver/tests/test_zephyr.py +++ b/zerver/tests/test_zephyr.py @@ -1,12 +1,12 @@ -import ujson - -from django.http import HttpResponse -from unittest.mock import patch from typing import Any +from unittest.mock import patch + +import ujson +from django.http import HttpResponse from zerver.lib.test_classes import ZulipTestCase from zerver.lib.users import get_api_key -from zerver.models import get_user, get_realm +from zerver.models import get_realm, get_user class ZephyrTest(ZulipTestCase): diff --git a/zerver/tornado/application.py b/zerver/tornado/application.py index 019cffbf23..a303b9a15f 100644 --- a/zerver/tornado/application.py +++ b/zerver/tornado/application.py @@ -2,11 +2,12 @@ import atexit import tornado.web from django.conf import settings -from zerver.tornado import autoreload from zerver.lib.queue import get_queue_client +from zerver.tornado import autoreload from zerver.tornado.handlers import AsyncDjangoHandler + def setup_tornado_rabbitmq() -> None: # nocoverage # When tornado is shut down, disconnect cleanly from rabbitmq if settings.USING_RABBITMQ: diff --git a/zerver/tornado/autoreload.py b/zerver/tornado/autoreload.py index a9d7a98e54..4aaec8f387 100644 --- a/zerver/tornado/autoreload.py +++ b/zerver/tornado/autoreload.py @@ -50,18 +50,17 @@ incorrectly. # code that didn't work. -import os -import sys import functools import importlib +import os +import subprocess +import sys import traceback import types -import subprocess import weakref -from tornado import ioloop +from tornado import ioloop, process from tornado.log import gen_log -from tornado import process try: import signal diff --git a/zerver/tornado/descriptors.py b/zerver/tornado/descriptors.py index 135ead63bc..402067a935 100644 --- a/zerver/tornado/descriptors.py +++ b/zerver/tornado/descriptors.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: from zerver.tornado.event_queue import ClientDescriptor diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.py index 4803ff7c40..2aea57b176 100644 --- a/zerver/tornado/event_queue.py +++ b/zerver/tornado/event_queue.py @@ -1,39 +1,57 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. -from typing import cast, AbstractSet, Any, Callable, Deque, Dict, List, \ - Mapping, MutableMapping, Optional, Iterable, Sequence, Set, Union +import atexit +import copy +import logging +import os +import random +import signal +import sys +import time +import traceback +from collections import deque +from typing import ( + AbstractSet, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + Union, + cast, +) + +import requests +import tornado.ioloop +import ujson +from django.conf import settings +from django.utils.translation import ugettext as _ from typing_extensions import TypedDict -from django.utils.translation import ugettext as _ -from django.conf import settings -from collections import deque -import os -import time -import logging -import ujson -import requests -import atexit -import sys -import signal -import traceback -import tornado.ioloop -import random -from zerver.models import UserProfile, Client, Realm from zerver.decorator import cachify -from zerver.tornado.handlers import clear_handler_by_id, get_handler_by_id, \ - finish_handler, handler_stats_string -from zerver.lib.utils import statsd -from zerver.middleware import async_request_timer_restart from zerver.lib.message import MessageDict from zerver.lib.narrow import build_narrow_filter from zerver.lib.queue import queue_json_publish, retry_event from zerver.lib.request import JsonableError +from zerver.lib.utils import statsd +from zerver.middleware import async_request_timer_restart +from zerver.models import Client, Realm, UserProfile +from zerver.tornado.autoreload import add_reload_hook from zerver.tornado.descriptors import clear_descriptor_by_handler_id, set_descriptor_by_handler_id from zerver.tornado.exceptions import BadEventQueueIdError -from zerver.tornado.sharding import get_tornado_uri, get_tornado_port, \ - notify_tornado_queue_name -from zerver.tornado.autoreload import add_reload_hook -import copy +from zerver.tornado.handlers import ( + clear_handler_by_id, + finish_handler, + get_handler_by_id, + handler_stats_string, +) +from zerver.tornado.sharding import get_tornado_port, get_tornado_uri, notify_tornado_queue_name requests_client = requests.Session() for host in ['127.0.0.1', 'localhost']: diff --git a/zerver/tornado/exceptions.py b/zerver/tornado/exceptions.py index 652dc579b6..9b194e6d78 100644 --- a/zerver/tornado/exceptions.py +++ b/zerver/tornado/exceptions.py @@ -2,6 +2,7 @@ from django.utils.translation import ugettext as _ from zerver.lib.exceptions import ErrorCode, JsonableError + class BadEventQueueIdError(JsonableError): code = ErrorCode.BAD_EVENT_QUEUE_ID data_fields = ['queue_id'] diff --git a/zerver/tornado/handlers.py b/zerver/tornado/handlers.py index e1af1f8a5c..74659ca757 100644 --- a/zerver/tornado/handlers.py +++ b/zerver/tornado/handlers.py @@ -7,8 +7,8 @@ from django import http from django.core import signals from django.core.handlers import base from django.core.handlers.wsgi import WSGIRequest, get_script_name -from django.urls import set_script_prefix from django.http import HttpRequest, HttpResponse +from django.urls import set_script_prefix from tornado.wsgi import WSGIContainer from zerver.lib.response import json_response diff --git a/zerver/tornado/sharding.py b/zerver/tornado/sharding.py index aeefb66738..7a7cb9588b 100644 --- a/zerver/tornado/sharding.py +++ b/zerver/tornado/sharding.py @@ -1,8 +1,10 @@ +import json +import os + from django.conf import settings from zerver.models import Realm -import json -import os + shard_map = {} if os.path.exists("/etc/zulip/sharding.json"): with open("/etc/zulip/sharding.json") as f: diff --git a/zerver/tornado/views.py b/zerver/tornado/views.py index fb8a24e9f7..9de1c1d3f2 100644 --- a/zerver/tornado/views.py +++ b/zerver/tornado/views.py @@ -5,16 +5,20 @@ import ujson from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import REQ, has_request_variables, internal_notify_view, \ - process_client +from zerver.decorator import REQ, has_request_variables, internal_notify_view, process_client from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_bool, check_int, check_list, check_string, \ - to_non_negative_int +from zerver.lib.validator import ( + check_bool, + check_int, + check_list, + check_string, + to_non_negative_int, +) from zerver.models import Client, UserProfile, get_client, get_user_profile_by_id -from zerver.tornado.event_queue import fetch_events, \ - get_client_descriptor, process_notification -from zerver.tornado.handlers import AsyncDjangoHandler +from zerver.tornado.event_queue import fetch_events, get_client_descriptor, process_notification from zerver.tornado.exceptions import BadEventQueueIdError +from zerver.tornado.handlers import AsyncDjangoHandler + @internal_notify_view(True) def notify(request: HttpRequest) -> HttpResponse: diff --git a/zerver/views/alert_words.py b/zerver/views/alert_words.py index c4821fd847..1f6ba2d23a 100644 --- a/zerver/views/alert_words.py +++ b/zerver/views/alert_words.py @@ -1,14 +1,14 @@ -from django.http import HttpResponse, HttpRequest - from typing import List -from zerver.models import UserProfile -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success -from zerver.lib.validator import check_list, check_string, check_capped_string +from django.http import HttpRequest, HttpResponse from zerver.lib.actions import do_add_alert_words, do_remove_alert_words from zerver.lib.alert_words import user_alert_words +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_success +from zerver.lib.validator import check_capped_string, check_list, check_string +from zerver.models import UserProfile + def list_alert_words(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return json_success({'alert_words': user_alert_words(user_profile)}) diff --git a/zerver/views/archive.py b/zerver/views/archive.py index 3b610a17fe..fbb7cbad18 100644 --- a/zerver/views/archive.py +++ b/zerver/views/archive.py @@ -3,17 +3,15 @@ from typing import List, Optional from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.template import loader -from zerver.lib.streams import get_stream_by_id -from zerver.models import Message, UserProfile from zerver.lib.avatar import get_gravatar_url -from zerver.lib.response import json_success -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - get_topic_history_for_web_public_stream, - messages_for_topic, -) from zerver.lib.exceptions import JsonableError +from zerver.lib.response import json_success +from zerver.lib.streams import get_stream_by_id +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.topic import get_topic_history_for_web_public_stream, messages_for_topic +from zerver.models import Message, UserProfile + def archive(request: HttpRequest, stream_id: int, diff --git a/zerver/views/attachments.py b/zerver/views/attachments.py index 250c004b87..e6a6cace8b 100644 --- a/zerver/views/attachments.py +++ b/zerver/views/attachments.py @@ -1,10 +1,9 @@ from django.http import HttpRequest, HttpResponse -from zerver.models import UserProfile from zerver.lib.actions import notify_attachment_update +from zerver.lib.attachments import access_attachment_by_id, remove_attachment, user_attachments from zerver.lib.response import json_success -from zerver.lib.attachments import user_attachments, remove_attachment, \ - access_attachment_by_id +from zerver.models import UserProfile def list_by_user(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: diff --git a/zerver/views/auth.py b/zerver/views/auth.py index 1fbe546f9f..86fc3148bc 100644 --- a/zerver/views/auth.py +++ b/zerver/views/auth.py @@ -1,39 +1,47 @@ +import logging import os +import urllib +from functools import wraps +from typing import Any, Dict, List, Mapping, Optional, cast + +import jwt from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from django.forms import Form from django.conf import settings from django.contrib.auth import authenticate -from django.contrib.auth.views import LoginView as DjangoLoginView, \ - logout_then_login as django_logout_then_login +from django.contrib.auth.views import LoginView as DjangoLoginView from django.contrib.auth.views import PasswordResetView as DjangoPasswordResetView -from django.urls import reverse -from zerver.decorator import require_post, \ - process_client, do_login, log_view_func -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, \ - HttpResponseServerError -from django.template.response import SimpleTemplateResponse +from django.contrib.auth.views import logout_then_login as django_logout_then_login +from django.forms import Form +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, HttpResponseServerError from django.shortcuts import redirect, render +from django.template.response import SimpleTemplateResponse +from django.urls import reverse +from django.utils.http import is_safe_url +from django.utils.translation import ugettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_safe from django.views.generic import TemplateView -from django.utils.translation import ugettext as _ -from django.utils.http import is_safe_url -from functools import wraps -import urllib -from typing import Any, Dict, List, Optional, Mapping, cast +from social_django.utils import load_backend, load_strategy +from two_factor.forms import BackupTokenForm +from two_factor.views import LoginView as BaseTwoFactorLoginView from confirmation.models import Confirmation, create_confirmation_link -from zerver.context_processors import zulip_default_context, get_realm_from_request, \ - login_context -from zerver.forms import HomepageForm, OurAuthenticationForm, \ - DEACTIVATED_ACCOUNT_ERROR, ZulipPasswordResetForm, \ - AuthenticationTokenForm +from version import API_FEATURE_LEVEL, ZULIP_VERSION +from zerver.context_processors import get_realm_from_request, login_context, zulip_default_context +from zerver.decorator import do_login, log_view_func, process_client, require_post +from zerver.forms import ( + DEACTIVATED_ACCOUNT_ERROR, + AuthenticationTokenForm, + HomepageForm, + OurAuthenticationForm, + ZulipPasswordResetForm, +) from zerver.lib.mobile_auth_otp import otp_encrypt_api_key from zerver.lib.push_notifications import push_notifications_enabled from zerver.lib.pysa import mark_sanitized from zerver.lib.realm_icon import realm_icon_url -from zerver.lib.request import REQ, has_request_variables, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.sessions import set_expirable_session_var from zerver.lib.subdomains import get_subdomain, is_subdomain_root_or_alias from zerver.lib.types import ViewFuncT @@ -42,23 +50,24 @@ from zerver.lib.user_agent import parse_user_agent from zerver.lib.users import get_api_key from zerver.lib.utils import has_api_key_format from zerver.lib.validator import validate_login_email -from zerver.models import PreregistrationUser, UserProfile, remote_user_to_email, Realm, \ - get_realm +from zerver.models import PreregistrationUser, Realm, UserProfile, get_realm, remote_user_to_email from zerver.signals import email_on_new_login -from zproject.backends import password_auth_enabled, dev_auth_enabled, \ - ldap_auth_enabled, ZulipLDAPConfigurationError, ZulipLDAPAuthBackend, \ - AUTH_BACKEND_NAME_MAP, auth_enabled_helper, saml_auth_enabled, SAMLAuthBackend, \ - redirect_to_config_error, ZulipRemoteUserBackend, validate_otp_params, ExternalAuthResult, \ - ExternalAuthDataDict -from version import ZULIP_VERSION, API_FEATURE_LEVEL - -import jwt -import logging - -from social_django.utils import load_backend, load_strategy - -from two_factor.forms import BackupTokenForm -from two_factor.views import LoginView as BaseTwoFactorLoginView +from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + ExternalAuthDataDict, + ExternalAuthResult, + SAMLAuthBackend, + ZulipLDAPAuthBackend, + ZulipLDAPConfigurationError, + ZulipRemoteUserBackend, + auth_enabled_helper, + dev_auth_enabled, + ldap_auth_enabled, + password_auth_enabled, + redirect_to_config_error, + saml_auth_enabled, + validate_otp_params, +) ExtraContext = Optional[Dict[str, Any]] diff --git a/zerver/views/camo.py b/zerver/views/camo.py index fecbdd868e..414eb6f602 100644 --- a/zerver/views/camo.py +++ b/zerver/views/camo.py @@ -1,13 +1,13 @@ +import binascii + from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import ( - HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound, -) + from zerver.lib.camo import is_camo_url_valid from zerver.lib.thumbnail import generate_thumbnail_url -import binascii def handle_camo_url(request: HttpRequest, digest: str, received_url: str) -> HttpResponse: diff --git a/zerver/views/compatibility.py b/zerver/views/compatibility.py index 6c0e08b158..55b9d4890e 100644 --- a/zerver/views/compatibility.py +++ b/zerver/views/compatibility.py @@ -1,13 +1,14 @@ -from django.http import HttpResponse, HttpRequest import re from typing import List, Optional, Tuple +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ +from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION from zerver.lib.response import json_error, json_success from zerver.lib.user_agent import parse_user_agent from zerver.signals import get_device_browser -from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION + def pop_numerals(ver: str) -> Tuple[List[int], str]: match = re.search(r'^( \d+ (?: \. \d+ )* ) (.*)', ver, re.X) diff --git a/zerver/views/custom_profile_fields.py b/zerver/views/custom_profile_fields.py index 850a11165f..96941837ef 100644 --- a/zerver/views/custom_profile_fields.py +++ b/zerver/views/custom_profile_fields.py @@ -1,29 +1,35 @@ -from typing import Union, List, Dict -import ujson +from typing import Dict, List, Union +import ujson from django.db import IntegrityError from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import require_realm_admin, human_users_only -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.actions import (try_add_realm_custom_profile_field, - do_remove_realm_custom_profile_field, - try_update_realm_custom_profile_field, - do_update_user_custom_profile_data_if_changed, - try_reorder_realm_custom_profile_fields, - try_add_realm_default_custom_profile_field, - check_remove_custom_profile_field_value) -from zerver.lib.response import json_success, json_error -from zerver.lib.types import ProfileFieldData -from zerver.lib.validator import (check_dict, check_list, check_int, - validate_choice_field_data, check_capped_string) - -from zerver.models import (UserProfile, - CustomProfileField, custom_profile_fields_for_realm) +from zerver.decorator import human_users_only, require_realm_admin +from zerver.lib.actions import ( + check_remove_custom_profile_field_value, + do_remove_realm_custom_profile_field, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_add_realm_default_custom_profile_field, + try_reorder_realm_custom_profile_fields, + try_update_realm_custom_profile_field, +) from zerver.lib.exceptions import JsonableError -from zerver.lib.users import validate_user_custom_profile_data from zerver.lib.external_accounts import validate_external_account_field_data +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.types import ProfileFieldData +from zerver.lib.users import validate_user_custom_profile_data +from zerver.lib.validator import ( + check_capped_string, + check_dict, + check_int, + check_list, + validate_choice_field_data, +) +from zerver.models import CustomProfileField, UserProfile, custom_profile_fields_for_realm + def list_realm_custom_profile_fields(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: fields = custom_profile_fields_for_realm(user_profile.realm_id) diff --git a/zerver/views/development/email_log.py b/zerver/views/development/email_log.py index 3cab8c4c29..bbfcbff344 100755 --- a/zerver/views/development/email_log.py +++ b/zerver/views/development/email_log.py @@ -1,25 +1,19 @@ -from django.conf import settings -from django.http import HttpRequest, HttpResponse -from django.shortcuts import render, redirect -from django.views.decorators.http import require_safe - -from zerver.models import ( - get_realm, get_user_by_delivery_email, get_realm_stream, Realm, -) -from zerver.lib.email_notifications import enqueue_welcome_emails -from zerver.lib.response import json_success -from zerver.lib.actions import do_change_user_delivery_email, \ - do_send_realm_reactivation_email -from zproject.email_backends import ( - get_forward_address, - set_forward_address, -) -import urllib -from confirmation.models import Confirmation, confirmation_url - import os import subprocess +import urllib + import ujson +from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.shortcuts import redirect, render +from django.views.decorators.http import require_safe + +from confirmation.models import Confirmation, confirmation_url +from zerver.lib.actions import do_change_user_delivery_email, do_send_realm_reactivation_email +from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.response import json_success +from zerver.models import Realm, get_realm, get_realm_stream, get_user_by_delivery_email +from zproject.email_backends import get_forward_address, set_forward_address ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') diff --git a/zerver/views/development/integrations.py b/zerver/views/development/integrations.py index 9da34f13f9..0b5701338a 100644 --- a/zerver/views/development/integrations.py +++ b/zerver/views/development/integrations.py @@ -1,19 +1,17 @@ import os -import ujson from typing import Any, Dict, List, Optional +import ujson from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.test import Client from zerver.lib.integrations import WEBHOOK_INTEGRATIONS -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.models import UserProfile, get_realm +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_bool -from zerver.lib.webhooks.common import get_fixture_http_headers, \ - standardize_headers - +from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers +from zerver.models import UserProfile, get_realm ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') diff --git a/zerver/views/development/registration.py b/zerver/views/development/registration.py index ee8a8047ba..467ea77605 100644 --- a/zerver/views/development/registration.py +++ b/zerver/views/development/registration.py @@ -1,14 +1,13 @@ +from typing import Any + from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.views.decorators.csrf import csrf_exempt from confirmation.models import Confirmation, create_confirmation_link - -from typing import Any - -from zerver.models import UserProfile from zerver.lib.response import json_success from zerver.lib.subdomains import get_subdomain +from zerver.models import UserProfile from zerver.views.auth import create_preregistration_user from zerver.views.registration import accounts_register diff --git a/zerver/views/digest.py b/zerver/views/digest.py index 524755f79c..d9eb84762f 100644 --- a/zerver/views/digest.py +++ b/zerver/views/digest.py @@ -1,12 +1,14 @@ import time +from datetime import timedelta + +from django.conf import settings from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.utils.timezone import now as timezone_now -from django.conf import settings -from zerver.lib.digest import handle_digest_email, DIGEST_CUTOFF from zerver.decorator import zulip_login_required -from datetime import timedelta +from zerver.lib.digest import DIGEST_CUTOFF, handle_digest_email + @zulip_login_required def digest_page(request: HttpRequest) -> HttpResponse: diff --git a/zerver/views/documentation.py b/zerver/views/documentation.py index f26c49b4bd..e93f04ce81 100644 --- a/zerver/views/documentation.py +++ b/zerver/views/documentation.py @@ -1,22 +1,22 @@ -from typing import Any, Dict, Tuple -from collections import OrderedDict -from django.views.generic import TemplateView -from django.conf import settings -from django.http import HttpRequest, HttpResponse, HttpResponseNotFound -from django.template import loader - import os import random import re +from collections import OrderedDict +from typing import Any, Dict, Tuple +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseNotFound +from django.template import loader +from django.views.generic import TemplateView + +from zerver.context_processors import zulip_default_context from zerver.decorator import add_google_analytics_context -from zerver.lib.integrations import CATEGORIES, INTEGRATIONS, HubotIntegration, \ - WebhookIntegration -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.integrations import CATEGORIES, INTEGRATIONS, HubotIntegration, WebhookIntegration +from zerver.lib.request import REQ, has_request_variables from zerver.lib.subdomains import get_subdomain from zerver.models import Realm from zerver.templatetags.app_filters import render_markdown_path -from zerver.context_processors import zulip_default_context + def add_api_uri_context(context: Dict[str, Any], request: HttpRequest) -> None: context.update(zulip_default_context(request)) diff --git a/zerver/views/email_mirror.py b/zerver/views/email_mirror.py index 8e8260111c..1f04e7f2c6 100644 --- a/zerver/views/email_mirror.py +++ b/zerver/views/email_mirror.py @@ -1,11 +1,11 @@ -import ujson - -from django.http import HttpRequest, HttpResponse from typing import Dict +import ujson +from django.http import HttpRequest, HttpResponse + from zerver.decorator import internal_notify_view from zerver.lib.email_mirror import mirror_email_message -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict, check_string diff --git a/zerver/views/events_register.py b/zerver/views/events_register.py index eab0f35a05..32b3277877 100644 --- a/zerver/views/events_register.py +++ b/zerver/views/events_register.py @@ -1,12 +1,14 @@ -from django.http import HttpRequest, HttpResponse from typing import Dict, Iterable, Optional, Sequence +from django.http import HttpRequest, HttpResponse + from zerver.lib.events import do_events_register from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.validator import check_dict, check_string, check_list, check_bool +from zerver.lib.validator import check_bool, check_dict, check_list, check_string from zerver.models import Stream, UserProfile + def _default_all_public_streams(user_profile: UserProfile, all_public_streams: Optional[bool]) -> bool: if all_public_streams is not None: diff --git a/zerver/views/home.py b/zerver/views/home.py index 5f15d5ddf8..a072ea2467 100644 --- a/zerver/views/home.py +++ b/zerver/views/home.py @@ -1,35 +1,36 @@ -from typing import Any, List, Dict, Optional, Tuple +import calendar +import logging +import time +from typing import Any, Dict, List, Optional, Tuple from django.conf import settings -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render +from django.urls import reverse from django.utils import translation from django.utils.cache import patch_cache_control +from two_factor.utils import default_device from zerver.decorator import zulip_login_required from zerver.forms import ToSForm -from zerver.models import Message, Stream, UserProfile, \ - Realm, PreregistrationUser +from zerver.lib.actions import do_change_tos_version, realm_user_count from zerver.lib.events import do_events_register -from zerver.lib.actions import do_change_tos_version, \ - realm_user_count -from zerver.lib.i18n import get_language_list, get_language_name, \ - get_language_list_for_templates, get_language_translation_data +from zerver.lib.i18n import ( + get_language_list, + get_language_list_for_templates, + get_language_name, + get_language_translation_data, +) from zerver.lib.push_notifications import num_push_devices_for_user from zerver.lib.streams import access_stream_by_name from zerver.lib.subdomains import get_subdomain from zerver.lib.users import compute_show_invites_and_add_streams -from zerver.lib.utils import statsd, generate_random_token -from zerver.views.compatibility import is_outdated_desktop_app, \ - is_unsupported_browser +from zerver.lib.utils import generate_random_token, statsd +from zerver.models import Message, PreregistrationUser, Realm, Stream, UserProfile +from zerver.views.compatibility import is_outdated_desktop_app, is_unsupported_browser from zerver.views.messages import get_latest_update_message_flag_activity from zerver.views.portico import hello_view -from two_factor.utils import default_device -import calendar -import logging -import time def need_accept_tos(user_profile: Optional[UserProfile]) -> bool: if user_profile is None: # nocoverage diff --git a/zerver/views/hotspots.py b/zerver/views/hotspots.py index 7efa693850..929ee2692e 100644 --- a/zerver/views/hotspots.py +++ b/zerver/views/hotspots.py @@ -4,11 +4,12 @@ from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only from zerver.lib.actions import do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_string from zerver.models import UserProfile + @human_users_only @has_request_variables def mark_hotspot_as_read(request: HttpRequest, user: UserProfile, diff --git a/zerver/views/invite.py b/zerver/views/invite.py index 25e4c5db8d..de93ac5d98 100644 --- a/zerver/views/invite.py +++ b/zerver/views/invite.py @@ -1,19 +1,25 @@ -from django.http import HttpRequest, HttpResponse -from django.utils.translation import ugettext as _ +import re from typing import List, Optional, Set -from zerver.decorator import require_realm_admin, require_member_or_admin -from zerver.lib.actions import do_invite_users, do_revoke_user_invite, \ - do_revoke_multi_use_invite, do_resend_user_invite_email, \ - do_get_user_invites, do_create_multiuse_invite_link -from zerver.lib.exceptions import OrganizationAdministratorRequired -from zerver.lib.request import REQ, has_request_variables, JsonableError -from zerver.lib.response import json_success, json_error -from zerver.lib.streams import access_stream_by_id -from zerver.lib.validator import check_list, check_int -from zerver.models import PreregistrationUser, Stream, UserProfile, MultiuseInvite +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + +from zerver.decorator import require_member_or_admin, require_realm_admin +from zerver.lib.actions import ( + do_create_multiuse_invite_link, + do_get_user_invites, + do_invite_users, + do_resend_user_invite_email, + do_revoke_multi_use_invite, + do_revoke_user_invite, +) +from zerver.lib.exceptions import OrganizationAdministratorRequired +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import access_stream_by_id +from zerver.lib.validator import check_int, check_list +from zerver.models import MultiuseInvite, PreregistrationUser, Stream, UserProfile -import re @require_member_or_admin @has_request_variables diff --git a/zerver/views/messages.py b/zerver/views/messages.py index a031d3e2ff..6ad5a5b34b 100644 --- a/zerver/views/messages.py +++ b/zerver/views/messages.py @@ -1,70 +1,119 @@ -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now +import datetime +import re +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union, cast + +import ujson +from dateutil.parser import parse as dateparser from django.conf import settings from django.core import validators from django.core.exceptions import ValidationError -from django.db import connection, IntegrityError +from django.db import IntegrityError, connection from django.http import HttpRequest, HttpResponse -from typing import Dict, List, Set, Any, Iterable, \ - Optional, Tuple, Union, Sequence, cast -from zerver.lib.exceptions import JsonableError, ErrorCode -from zerver.lib.html_diff import highlight_html_differences -from zerver.decorator import has_request_variables, REQ from django.utils.html import escape as escape_html +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from sqlalchemy import func +from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import ( + ColumnElement, + Selectable, + alias, + and_, + column, + join, + literal, + literal_column, + not_, + or_, + select, + table, + union_all, +) + +from zerver.decorator import REQ, has_request_variables from zerver.lib import bugdown -from zerver.lib.zcommand import process_zcommands -from zerver.lib.actions import recipient_for_user_profiles, do_update_message_flags, \ - compute_irc_user_fullname, compute_jabber_user_fullname, \ - create_mirror_user_if_needed, check_send_message, do_update_message, \ - extract_private_recipients, render_incoming_message, do_delete_messages, \ - do_mark_all_as_read, do_mark_stream_messages_as_read, extract_stream_indicator, \ - get_user_info_for_message_updates, check_schedule_message +from zerver.lib.actions import ( + check_schedule_message, + check_send_message, + compute_irc_user_fullname, + compute_jabber_user_fullname, + create_mirror_user_if_needed, + do_delete_messages, + do_mark_all_as_read, + do_mark_stream_messages_as_read, + do_update_message, + do_update_message_flags, + extract_private_recipients, + extract_stream_indicator, + get_user_info_for_message_updates, + recipient_for_user_profiles, + render_incoming_message, +) from zerver.lib.addressee import get_user_profiles, get_user_profiles_by_ids -from zerver.lib.queue import queue_json_publish +from zerver.lib.exceptions import ErrorCode, JsonableError +from zerver.lib.html_diff import highlight_html_differences from zerver.lib.message import ( access_message, + get_first_visible_message_id, messages_for_ids, render_markdown, - get_first_visible_message_id, truncate_body, ) -from zerver.lib.response import json_success, json_error +from zerver.lib.queue import queue_json_publish +from zerver.lib.response import json_error, json_success from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.streams import access_stream_by_id, get_public_streams_queryset, \ - can_access_stream_history_by_name, can_access_stream_history_by_id, \ - get_stream_by_narrow_operand_access_unchecked, get_stream_by_id -from zerver.lib.timestamp import datetime_to_timestamp, convert_to_UTC +from zerver.lib.streams import ( + access_stream_by_id, + can_access_stream_history_by_id, + can_access_stream_history_by_name, + get_public_streams_queryset, + get_stream_by_id, + get_stream_by_narrow_operand_access_unchecked, +) +from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp from zerver.lib.timezone import get_timezone from zerver.lib.topic import ( - topic_column_sa, - topic_match_sa, - user_message_exists_for_topic, DB_TOPIC_NAME, LEGACY_PREV_TOPIC, MATCH_TOPIC, REQ_topic, + topic_column_sa, + topic_match_sa, + user_message_exists_for_topic, ) from zerver.lib.topic_mutes import exclude_topic_mutes from zerver.lib.utils import statsd -from zerver.lib.validator import \ - check_list, check_int, check_dict, check_string, check_bool, \ - check_string_or_int_list, check_string_or_int, check_string_in, \ - check_required_string, to_non_negative_int +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_list, + check_required_string, + check_string, + check_string_in, + check_string_or_int, + check_string_or_int_list, + to_non_negative_int, +) +from zerver.lib.zcommand import process_zcommands from zerver.lib.zephyr import compute_mit_user_fullname -from zerver.models import Message, UserProfile, Stream, Subscription, Client,\ - Realm, RealmDomain, Recipient, UserMessage, UserActivity, \ - email_to_domain, get_realm, get_active_streams, get_user_including_cross_realm, \ - get_user_by_id_in_realm_including_cross_realm - -from sqlalchemy import func -from sqlalchemy.dialects import postgresql -from sqlalchemy.sql import select, join, column, literal_column, literal, and_, \ - or_, not_, union_all, alias, Selectable, ColumnElement, table - -from dateutil.parser import parse as dateparser -import re -import ujson -import datetime +from zerver.models import ( + Client, + Message, + Realm, + RealmDomain, + Recipient, + Stream, + Subscription, + UserActivity, + UserMessage, + UserProfile, + email_to_domain, + get_active_streams, + get_realm, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, +) LARGER_THAN_MAX_MESSAGE_ID = 10000000000000000 MAX_MESSAGES_PER_FETCH = 5000 diff --git a/zerver/views/muting.py b/zerver/views/muting.py index 77dbfdcd75..c4f46c5aee 100644 --- a/zerver/views/muting.py +++ b/zerver/views/muting.py @@ -1,13 +1,13 @@ -from django.http import HttpResponse, HttpRequest -from typing import Optional import datetime +from typing import Optional +from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ + from zerver.lib.actions import do_mute_topic, do_unmute_topic -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.lib.topic_mutes import topic_is_muted +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.streams import ( access_stream_by_id, access_stream_by_name, @@ -15,9 +15,11 @@ from zerver.lib.streams import ( access_stream_for_unmute_topic_by_name, check_for_exactly_one_stream_arg, ) +from zerver.lib.topic_mutes import topic_is_muted from zerver.lib.validator import check_int from zerver.models import UserProfile + def mute_topic(user_profile: UserProfile, stream_id: Optional[int], stream_name: Optional[str], diff --git a/zerver/views/pointer.py b/zerver/views/pointer.py index 924382098d..08e9a0756e 100644 --- a/zerver/views/pointer.py +++ b/zerver/views/pointer.py @@ -2,11 +2,12 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ from zerver.lib.actions import do_update_pointer -from zerver.lib.request import has_request_variables, JsonableError, REQ +from zerver.lib.request import REQ, JsonableError, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import to_non_negative_int from zerver.models import UserProfile, get_usermessage_by_message_id + def get_pointer_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return json_success({'pointer': user_profile.pointer}) diff --git a/zerver/views/portico.py b/zerver/views/portico.py index b6fdc13706..770dc7eaeb 100644 --- a/zerver/views/portico.py +++ b/zerver/views/portico.py @@ -1,12 +1,13 @@ +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.template.response import TemplateResponse -import ujson +from version import LATEST_DESKTOP_VERSION from zerver.context_processors import get_realm_from_request, latest_info_context from zerver.decorator import add_google_analytics, redirect_to_login from zerver.models import Realm -from version import LATEST_DESKTOP_VERSION + @add_google_analytics def apps_view(request: HttpRequest, _: str) -> HttpResponse: diff --git a/zerver/views/presence.py b/zerver/views/presence.py index 76525c7f5a..7d7e23d6be 100644 --- a/zerver/views/presence.py +++ b/zerver/views/presence.py @@ -1,27 +1,20 @@ import datetime - -from django.conf import settings from typing import Any, Dict, Optional +from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only -from zerver.lib.actions import ( - do_update_user_status, - update_user_presence, -) -from zerver.lib.presence import ( - get_presence_response, - get_presence_for_user, -) -from zerver.lib.request import has_request_variables, REQ, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.actions import do_update_user_status, update_user_presence +from zerver.lib.presence import get_presence_for_user, get_presence_response +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.validator import check_bool, check_capped_string -from zerver.models import UserActivity, UserPresence, UserProfile, \ - get_active_user +from zerver.models import UserActivity, UserPresence, UserProfile, get_active_user + def get_presence_backend(request: HttpRequest, user_profile: UserProfile, email: str) -> HttpResponse: diff --git a/zerver/views/push_notifications.py b/zerver/views/push_notifications.py index 2d4d88e3ad..67083211c7 100644 --- a/zerver/views/push_notifications.py +++ b/zerver/views/push_notifications.py @@ -3,12 +3,16 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only -from zerver.lib.push_notifications import add_push_device_token, \ - b64_to_hex, remove_push_device_token -from zerver.lib.request import has_request_variables, REQ, JsonableError +from zerver.lib.push_notifications import ( + add_push_device_token, + b64_to_hex, + remove_push_device_token, +) +from zerver.lib.request import REQ, JsonableError, has_request_variables from zerver.lib.response import json_success from zerver.models import PushDeviceToken, UserProfile + def validate_token(token_str: str, kind: int) -> None: if token_str == '' or len(token_str) > 4096: raise JsonableError(_('Empty or invalid length token')) diff --git a/zerver/views/reactions.py b/zerver/views/reactions.py index 31180adddb..951ad88a4f 100644 --- a/zerver/views/reactions.py +++ b/zerver/views/reactions.py @@ -1,8 +1,9 @@ +from typing import Optional + from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import \ - has_request_variables, REQ +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import do_add_reaction, do_remove_reaction from zerver.lib.emoji import check_emoji_request, emoji_name_to_emoji_code from zerver.lib.message import access_message @@ -10,7 +11,6 @@ from zerver.lib.request import JsonableError from zerver.lib.response import json_success from zerver.models import Message, Reaction, UserMessage, UserProfile -from typing import Optional def create_historical_message(user_profile: UserProfile, message: Message) -> None: # Users can see and react to messages sent to streams they diff --git a/zerver/views/realm.py b/zerver/views/realm.py index 40ec58ff1f..3dc1ecb8ef 100644 --- a/zerver/views/realm.py +++ b/zerver/views/realm.py @@ -1,31 +1,40 @@ from typing import Any, Dict, Optional + +from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.utils.translation import ugettext as _ -from django.core.exceptions import ValidationError from django.views.decorators.http import require_safe +from confirmation.models import Confirmation, ConfirmationKeyException, get_object_from_key from zerver.decorator import require_realm_admin, require_realm_owner +from zerver.forms import check_subdomain_available as check_subdomain from zerver.lib.actions import ( - do_set_realm_message_editing, - do_set_realm_message_deleting, - do_set_realm_authentication_methods, - do_set_realm_notifications_stream, - do_set_realm_signup_notifications_stream, - do_set_realm_property, do_deactivate_realm, do_reactivate_realm, + do_set_realm_authentication_methods, + do_set_realm_message_deleting, + do_set_realm_message_editing, + do_set_realm_notifications_stream, + do_set_realm_property, + do_set_realm_signup_notifications_stream, ) -from zerver.lib.i18n import get_available_language_codes -from zerver.lib.request import has_request_variables, REQ, JsonableError -from zerver.lib.response import json_success, json_error -from zerver.lib.validator import check_string, check_dict, check_bool, check_int, \ - check_int_in, to_positive_or_allowed_int, to_non_negative_int -from zerver.lib.streams import access_stream_by_id from zerver.lib.domains import validate_domain +from zerver.lib.i18n import get_available_language_codes +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import access_stream_by_id +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_int_in, + check_string, + to_non_negative_int, + to_positive_or_allowed_int, +) from zerver.models import Realm, UserProfile -from zerver.forms import check_subdomain_available as check_subdomain -from confirmation.models import get_object_from_key, Confirmation, ConfirmationKeyException + @require_realm_admin @has_request_variables diff --git a/zerver/views/realm_domains.py b/zerver/views/realm_domains.py index 85d014226a..60d682ccb7 100644 --- a/zerver/views/realm_domains.py +++ b/zerver/views/realm_domains.py @@ -3,10 +3,9 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ from zerver.decorator import require_realm_admin -from zerver.lib.actions import do_add_realm_domain, do_change_realm_domain, \ - do_remove_realm_domain +from zerver.lib.actions import do_add_realm_domain, do_change_realm_domain, do_remove_realm_domain from zerver.lib.domains import validate_domain -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_bool, check_string from zerver.models import RealmDomain, UserProfile, get_realm_domains diff --git a/zerver/views/realm_emoji.py b/zerver/views/realm_emoji.py index 526d70f090..0758caceaa 100644 --- a/zerver/views/realm_emoji.py +++ b/zerver/views/realm_emoji.py @@ -2,12 +2,12 @@ from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.models import RealmEmoji, UserProfile -from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name -from zerver.lib.request import JsonableError, REQ, has_request_variables -from zerver.lib.response import json_success, json_error -from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji from zerver.decorator import require_member_or_admin +from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji +from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.models import RealmEmoji, UserProfile def list_emoji(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: diff --git a/zerver/views/realm_export.py b/zerver/views/realm_export.py index 4b7b777fc8..29d7bdcc58 100644 --- a/zerver/views/realm_export.py +++ b/zerver/views/realm_export.py @@ -1,20 +1,19 @@ from datetime import timedelta -from analytics.models import RealmCount - +import ujson from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest +from analytics.models import RealmCount from zerver.decorator import require_realm_admin -from zerver.models import RealmAuditLog, UserProfile +from zerver.lib.actions import do_delete_realm_export, notify_realm_export +from zerver.lib.export import get_realm_exports_serialized from zerver.lib.queue import queue_json_publish from zerver.lib.response import json_error, json_success -from zerver.lib.export import get_realm_exports_serialized -from zerver.lib.actions import do_delete_realm_export, notify_realm_export +from zerver.models import RealmAuditLog, UserProfile -import ujson @require_realm_admin def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse: diff --git a/zerver/views/realm_filters.py b/zerver/views/realm_filters.py index 625ef7a173..366cb66bdb 100644 --- a/zerver/views/realm_filters.py +++ b/zerver/views/realm_filters.py @@ -4,9 +4,9 @@ from django.utils.translation import ugettext as _ from zerver.decorator import require_realm_admin from zerver.lib.actions import do_add_realm_filter, do_remove_realm_filter -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.models import realm_filters_for_realm, UserProfile, RealmFilter +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.models import RealmFilter, UserProfile, realm_filters_for_realm # Custom realm filters diff --git a/zerver/views/realm_icon.py b/zerver/views/realm_icon.py index 66054248dd..70dff5711b 100644 --- a/zerver/views/realm_icon.py +++ b/zerver/views/realm_icon.py @@ -1,7 +1,7 @@ from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest from zerver.decorator import require_realm_admin from zerver.lib.actions import do_change_icon_source diff --git a/zerver/views/realm_logo.py b/zerver/views/realm_logo.py index b7ab5ef03e..ea09ff8cd0 100644 --- a/zerver/views/realm_logo.py +++ b/zerver/views/realm_logo.py @@ -1,16 +1,16 @@ from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest -from zerver.lib.validator import check_bool -from zerver.lib.request import REQ, has_request_variables from zerver.decorator import require_realm_admin from zerver.lib.actions import do_change_logo_source from zerver.lib.realm_logo import get_realm_logo_url +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.upload import upload_logo_image from zerver.lib.url_encoding import add_query_arg_to_redirect_url +from zerver.lib.validator import check_bool from zerver.models import UserProfile diff --git a/zerver/views/registration.py b/zerver/views/registration.py index eb7dae16fb..290acf4cb4 100644 --- a/zerver/views/registration.py +++ b/zerver/views/registration.py @@ -1,55 +1,92 @@ -from typing import List, Dict, Optional +import logging +import smtplib +import urllib +from typing import Dict, List, Optional -from django.utils.translation import ugettext as _ from django.conf import settings from django.contrib.auth import authenticate, get_backends -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponse, HttpRequest -from django.shortcuts import redirect, render -from django.core.exceptions import ValidationError from django.core import validators -from zerver.context_processors import get_realm_from_request, login_context -from zerver.models import UserProfile, Realm, Stream, MultiuseInvite, \ - name_changes_disabled, email_to_username, \ - get_realm, get_user_by_delivery_email, get_default_stream_groups, DisposableEmailError, \ - DomainNotAllowedForRealmError, get_source_profile, EmailContainsPlusError -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm -from zerver.lib.send_email import send_email, FromAddress -from zerver.lib.actions import do_change_password, do_change_full_name, \ - do_activate_user, do_create_user, do_create_realm, \ - do_set_user_display_setting, lookup_default_stream_groups, bulk_add_subscriptions -from zerver.forms import RegistrationForm, HomepageForm, RealmCreationForm, \ - FindMyTeamForm, RealmRedirectForm +from django.core.exceptions import ValidationError +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import ugettext as _ from django_auth_ldap.backend import LDAPBackend, _LDAPUser -from zerver.decorator import require_post, \ - do_login + +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + RealmCreationKey, + create_confirmation_link, + get_object_from_key, + render_confirmation_key_error, + validate_key, +) +from zerver.context_processors import get_realm_from_request, login_context +from zerver.decorator import do_login, require_post +from zerver.forms import ( + FindMyTeamForm, + HomepageForm, + RealmCreationForm, + RealmRedirectForm, + RegistrationForm, +) +from zerver.lib.actions import ( + bulk_add_subscriptions, + do_activate_user, + do_change_full_name, + do_change_password, + do_create_realm, + do_create_user, + do_set_user_display_setting, + lookup_default_stream_groups, +) from zerver.lib.create_user import get_role_for_new_user +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm from zerver.lib.onboarding import send_initial_realm_messages, setup_realm_internal_bots -from zerver.lib.sessions import get_expirable_session_var from zerver.lib.pysa import mark_sanitized +from zerver.lib.send_email import FromAddress, send_email +from zerver.lib.sessions import get_expirable_session_var from zerver.lib.subdomains import get_subdomain, is_root_domain_available from zerver.lib.timezone import get_all_timezones from zerver.lib.url_encoding import add_query_to_redirect_url from zerver.lib.users import get_accounts_for_email from zerver.lib.zephyr import compute_mit_user_fullname -from zerver.views.auth import create_preregistration_user, redirect_and_log_into_subdomain, \ - redirect_to_deactivation_notice, get_safe_redirect_to, finish_desktop_flow, finish_mobile_flow +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + MultiuseInvite, + Realm, + Stream, + UserProfile, + email_to_username, + get_default_stream_groups, + get_realm, + get_source_profile, + get_user_by_delivery_email, + name_changes_disabled, +) +from zerver.views.auth import ( + create_preregistration_user, + finish_desktop_flow, + finish_mobile_flow, + get_safe_redirect_to, + redirect_and_log_into_subdomain, + redirect_to_deactivation_notice, +) +from zproject.backends import ( + ExternalAuthResult, + ZulipLDAPAuthBackend, + ZulipLDAPExceptionNoMatchingLDAPUser, + any_social_backend_enabled, + email_auth_enabled, + email_belongs_to_ldap, + ldap_auth_enabled, + password_auth_enabled, +) -from zproject.backends import ldap_auth_enabled, password_auth_enabled, \ - ZulipLDAPExceptionNoMatchingLDAPUser, email_auth_enabled, ZulipLDAPAuthBackend, \ - email_belongs_to_ldap, any_social_backend_enabled, ExternalAuthResult - -from confirmation.models import Confirmation, RealmCreationKey, ConfirmationKeyException, \ - validate_key, create_confirmation_link, get_object_from_key, \ - render_confirmation_key_error - -from confirmation import settings as confirmation_settings - -import logging -import smtplib - -import urllib def check_prereg_key_and_redirect(request: HttpRequest, confirmation_key: str) -> HttpResponse: confirmation = Confirmation.objects.filter(confirmation_key=confirmation_key).first() diff --git a/zerver/views/report.py b/zerver/views/report.py index de1d62d549..3e665c58a9 100644 --- a/zerver/views/report.py +++ b/zerver/views/report.py @@ -1,25 +1,24 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - +import logging +import subprocess from typing import Any, Dict, Optional from django.conf import settings from django.http import HttpRequest, HttpResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST + from zerver.decorator import human_users_only from zerver.lib.bugdown import privacy_clean_markdown -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success from zerver.lib.queue import queue_json_publish +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_success from zerver.lib.storage import static_path from zerver.lib.unminify import SourceMap from zerver.lib.utils import statsd, statsd_key from zerver.lib.validator import check_bool, check_dict, to_non_negative_int from zerver.models import UserProfile -import subprocess -import logging - js_source_map: Optional[SourceMap] = None # Read the source map information for decoding JavaScript backtraces. diff --git a/zerver/views/storage.py b/zerver/views/storage.py index 23e7214640..3772345cc0 100644 --- a/zerver/views/storage.py +++ b/zerver/views/storage.py @@ -1,17 +1,19 @@ +from typing import Dict, List, Optional + from django.http import HttpRequest, HttpResponse + +from zerver.decorator import REQ, has_request_variables from zerver.lib.bot_storage import ( - get_bot_storage, - set_bot_storage, - remove_bot_storage, - get_keys_in_bot_storage, StateError, + get_bot_storage, + get_keys_in_bot_storage, + remove_bot_storage, + set_bot_storage, ) -from zerver.decorator import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict, check_list, check_string from zerver.models import UserProfile -from typing import Dict, List, Optional @has_request_variables def update_storage(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/streams.py b/zerver/views/streams.py index f8513a3372..c8a36a24a4 100644 --- a/zerver/views/streams.py +++ b/zerver/views/streams.py @@ -1,39 +1,79 @@ -from typing import Any, Optional, Tuple, List, Set, Iterable, Mapping, Callable, Dict, \ - Union +from collections import defaultdict +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union -from django.utils.translation import ugettext as _ +import ujson from django.conf import settings from django.db import transaction from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ -from zerver.lib.exceptions import JsonableError, ErrorCode +from zerver.decorator import ( + authenticated_json_post_view, + require_non_guest_user, + require_realm_admin, +) +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_add_default_stream, + do_add_streams_to_default_stream_group, + do_change_default_stream_group_description, + do_change_default_stream_group_name, + do_change_stream_description, + do_change_stream_invite_only, + do_change_stream_post_policy, + do_change_subscription_property, + do_create_default_stream_group, + do_deactivate_stream, + do_delete_messages, + do_get_streams, + do_remove_default_stream, + do_remove_default_stream_group, + do_remove_streams_from_default_stream_group, + do_rename_stream, + do_send_messages, + gather_subscriptions, + get_subscriber_emails, + internal_prep_private_message, + internal_prep_stream_message, +) +from zerver.lib.exceptions import ErrorCode, JsonableError from zerver.lib.request import REQ, has_request_variables -from zerver.decorator import authenticated_json_post_view, \ - require_realm_admin, require_non_guest_user -from zerver.lib.actions import bulk_remove_subscriptions, \ - do_change_subscription_property, internal_prep_private_message, \ - internal_prep_stream_message, \ - gather_subscriptions, \ - bulk_add_subscriptions, do_send_messages, get_subscriber_emails, do_rename_stream, \ - do_deactivate_stream, do_change_stream_invite_only, do_add_default_stream, \ - do_change_stream_description, do_get_streams, \ - do_remove_default_stream, do_change_stream_post_policy, do_delete_messages, \ - do_create_default_stream_group, do_add_streams_to_default_stream_group, \ - do_remove_streams_from_default_stream_group, do_remove_default_stream_group, \ - do_change_default_stream_group_description, do_change_default_stream_group_name -from zerver.lib.response import json_success, json_error -from zerver.lib.streams import access_stream_by_id, access_stream_by_name, \ - check_stream_name, check_stream_name_available, filter_stream_authorization, \ - list_to_streams, access_stream_for_delete_or_update, access_default_stream_group_by_id +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import ( + access_default_stream_group_by_id, + access_stream_by_id, + access_stream_by_name, + access_stream_for_delete_or_update, + check_stream_name, + check_stream_name_available, + filter_stream_authorization, + list_to_streams, +) from zerver.lib.topic import get_topic_history_for_stream, messages_for_topic -from zerver.lib.validator import check_string, check_int, check_list, check_dict, \ - check_bool, check_variable_type, check_capped_string, check_color, check_dict_only, \ - check_int_in, to_non_negative_int -from zerver.models import UserProfile, Stream, Realm, UserMessage, \ - get_system_bot, get_active_user, get_active_user_profile_by_id_in_realm +from zerver.lib.validator import ( + check_bool, + check_capped_string, + check_color, + check_dict, + check_dict_only, + check_int, + check_int_in, + check_list, + check_string, + check_variable_type, + to_non_negative_int, +) +from zerver.models import ( + Realm, + Stream, + UserMessage, + UserProfile, + get_active_user, + get_active_user_profile_by_id_in_realm, + get_system_bot, +) -from collections import defaultdict -import ujson class PrincipalError(JsonableError): code = ErrorCode.UNAUTHORIZED_PRINCIPAL diff --git a/zerver/views/submessage.py b/zerver/views/submessage.py index 014cb80b90..0c0346949a 100644 --- a/zerver/views/submessage.py +++ b/zerver/views/submessage.py @@ -1,21 +1,15 @@ import ujson - from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import ( - has_request_variables, - REQ, -) +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import do_add_submessage from zerver.lib.message import access_message +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_int -from zerver.lib.response import ( - json_error, - json_success, -) from zerver.models import UserProfile + @has_request_variables def process_submessage(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/thumbnail.py b/zerver/views/thumbnail.py index 2156d07720..7d5bdf2ff5 100644 --- a/zerver/views/thumbnail.py +++ b/zerver/views/thumbnail.py @@ -1,11 +1,14 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html +from typing import Optional + +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden -from typing import Optional -from zerver.models import UserProfile, validate_attachment_request -from zerver.lib.request import has_request_variables, REQ + +from zerver.lib.request import REQ, has_request_variables from zerver.lib.thumbnail import generate_thumbnail_url +from zerver.models import UserProfile, validate_attachment_request + def validate_thumbnail_request(user_profile: UserProfile, path: str) -> Optional[bool]: # path here does not have a leading / as it is parsed from request hitting the diff --git a/zerver/views/tutorial.py b/zerver/views/tutorial.py index 5b647af7f3..93ff426061 100644 --- a/zerver/views/tutorial.py +++ b/zerver/views/tutorial.py @@ -1,11 +1,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import human_users_only -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_string from zerver.models import UserProfile + @human_users_only @has_request_variables def set_tutorial_status(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/typing.py b/zerver/views/typing.py index 08f5ca1b01..1b2d143a71 100644 --- a/zerver/views/typing.py +++ b/zerver/views/typing.py @@ -1,12 +1,14 @@ -from django.http import HttpRequest, HttpResponse from typing import List -from zerver.decorator import has_request_variables, REQ +from django.http import HttpRequest, HttpResponse + +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import check_send_typing_notification from zerver.lib.response import json_success from zerver.lib.validator import check_int, check_list from zerver.models import UserProfile + @has_request_variables def send_notification_backend( request: HttpRequest, diff --git a/zerver/views/unsubscribe.py b/zerver/views/unsubscribe.py index 4b5b561281..d22a2e8190 100644 --- a/zerver/views/unsubscribe.py +++ b/zerver/views/unsubscribe.py @@ -1,13 +1,14 @@ -from django.http import HttpRequest, HttpResponse -from django.shortcuts import render from typing import Callable -from confirmation.models import Confirmation, get_object_from_key, \ - ConfirmationKeyException +from django.http import HttpRequest, HttpResponse +from django.shortcuts import render + +from confirmation.models import Confirmation, ConfirmationKeyException, get_object_from_key +from zerver.context_processors import common_context from zerver.lib.actions import do_change_notification_settings from zerver.lib.send_email import clear_scheduled_emails -from zerver.models import UserProfile, ScheduledEmail -from zerver.context_processors import common_context +from zerver.models import ScheduledEmail, UserProfile + def process_unsubscribe(request: HttpRequest, confirmation_key: str, subscription_type: str, unsubscribe_function: Callable[[UserProfile], None]) -> HttpResponse: diff --git a/zerver/views/upload.py b/zerver/views/upload.py index 22ff8389b4..eaa724e29b 100644 --- a/zerver/views/upload.py +++ b/zerver/views/upload.py @@ -1,17 +1,24 @@ -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, \ - HttpResponseNotFound +from mimetypes import guess_type + +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound from django.shortcuts import redirect from django.utils.cache import patch_cache_control from django.utils.translation import ugettext as _ - -from zerver.lib.response import json_success, json_error -from zerver.lib.upload import upload_message_image_from_request, get_local_file_path, \ - get_signed_upload_url, check_upload_within_quota, INLINE_MIME_TYPES, \ - generate_unauthed_file_access_url, get_local_file_path_id_from_token -from zerver.models import UserProfile, validate_attachment_request -from django.conf import settings from django_sendfile import sendfile -from mimetypes import guess_type + +from zerver.lib.response import json_error, json_success +from zerver.lib.upload import ( + INLINE_MIME_TYPES, + check_upload_within_quota, + generate_unauthed_file_access_url, + get_local_file_path, + get_local_file_path_id_from_token, + get_signed_upload_url, + upload_message_image_from_request, +) +from zerver.models import UserProfile, validate_attachment_request + def serve_s3(request: HttpRequest, url_path: str, url_only: bool) -> HttpResponse: url = get_signed_upload_url(url_path) diff --git a/zerver/views/user_groups.py b/zerver/views/user_groups.py index 2aaddeda87..852823ee63 100644 --- a/zerver/views/user_groups.py +++ b/zerver/views/user_groups.py @@ -1,21 +1,31 @@ -from django.http import HttpResponse, HttpRequest -from django.utils.translation import ugettext as _ - from typing import List +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + from zerver.decorator import require_member_or_admin, require_user_group_edit_permission -from zerver.lib.actions import check_add_user_group, do_update_user_group_name, \ - do_update_user_group_description, bulk_add_members_to_user_group, \ - remove_members_from_user_group, check_delete_user_group +from zerver.lib.actions import ( + bulk_add_members_to_user_group, + check_add_user_group, + check_delete_user_group, + do_update_user_group_description, + do_update_user_group_name, + remove_members_from_user_group, +) from zerver.lib.exceptions import JsonableError -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.user_groups import ( + access_user_group_by_id, + get_memberships_of_users, + get_user_group_members, + user_groups_in_realm_serialized, +) from zerver.lib.users import user_ids_to_users -from zerver.lib.validator import check_list, check_int -from zerver.lib.user_groups import access_user_group_by_id, get_memberships_of_users, \ - get_user_group_members, user_groups_in_realm_serialized +from zerver.lib.validator import check_int, check_list from zerver.models import UserProfile -from zerver.views.streams import compose_views, FuncKwargPair +from zerver.views.streams import FuncKwargPair, compose_views + @require_user_group_edit_permission @has_request_variables diff --git a/zerver/views/user_settings.py b/zerver/views/user_settings.py index 86999d063a..b19fe08ff3 100644 --- a/zerver/views/user_settings.py +++ b/zerver/views/user_settings.py @@ -1,35 +1,47 @@ -from typing import Optional, Any, Dict +from typing import Any, Dict, Optional -from django.core.exceptions import ValidationError -from django.utils.translation import ugettext as _ from django.conf import settings from django.contrib.auth import authenticate, update_session_auth_hash +from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.shortcuts import render +from django.utils.translation import ugettext as _ -from zerver.decorator import has_request_variables, \ - REQ, human_users_only -from zerver.lib.actions import do_change_password, do_change_notification_settings, \ - do_change_enter_sends, do_regenerate_api_key, do_change_avatar_fields, \ - do_set_user_display_setting, do_change_user_delivery_email, \ - do_start_email_change_process, check_change_full_name, \ - get_available_notification_sounds, validate_email_is_valid +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + get_object_from_key, + render_confirmation_key_error, +) +from zerver.decorator import REQ, has_request_variables, human_users_only +from zerver.lib.actions import ( + check_change_full_name, + do_change_avatar_fields, + do_change_enter_sends, + do_change_notification_settings, + do_change_password, + do_change_user_delivery_email, + do_regenerate_api_key, + do_set_user_display_setting, + do_start_email_change_process, + get_available_notification_sounds, + validate_email_is_valid, +) from zerver.lib.avatar import avatar_url -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_not_already_in_realm -from zerver.lib.send_email import send_email, FromAddress +from zerver.lib.email_validation import ( + get_realm_email_validator, + validate_email_not_already_in_realm, +) from zerver.lib.i18n import get_available_language_codes -from zerver.lib.response import json_success, json_error -from zerver.lib.upload import upload_avatar_image -from zerver.lib.validator import check_bool, check_string, check_int, check_int_in, \ - check_string_in from zerver.lib.rate_limiter import RateLimited from zerver.lib.request import JsonableError +from zerver.lib.response import json_error, json_success +from zerver.lib.send_email import FromAddress, send_email from zerver.lib.timezone import get_all_timezones -from zerver.models import UserProfile, name_changes_disabled, avatar_changes_disabled -from confirmation.models import get_object_from_key, render_confirmation_key_error, \ - ConfirmationKeyException, Confirmation -from zproject.backends import email_belongs_to_ldap, check_password_strength +from zerver.lib.upload import upload_avatar_image +from zerver.lib.validator import check_bool, check_int, check_int_in, check_string, check_string_in +from zerver.models import UserProfile, avatar_changes_disabled, name_changes_disabled +from zproject.backends import check_password_strength, email_belongs_to_ldap AVATAR_CHANGES_DISABLED_ERROR = _("Avatar changes are disabled in this organization.") diff --git a/zerver/views/users.py b/zerver/views/users.py index 6e8af2d747..fd5e926f3a 100644 --- a/zerver/views/users.py +++ b/zerver/views/users.py @@ -1,44 +1,84 @@ -from typing import Union, Optional, Dict, Any, List +from typing import Any, Dict, List, Optional, Union -from django.http import HttpRequest, HttpResponse - -from django.utils.translation import ugettext as _ -from django.shortcuts import redirect from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.shortcuts import redirect +from django.utils.translation import ugettext as _ -from zerver.decorator import require_realm_admin, require_member_or_admin -from zerver.forms import CreateUserForm, PASSWORD_TOO_WEAK_ERROR -from zerver.lib.actions import do_change_avatar_fields, do_change_bot_owner, \ - do_change_user_role, do_change_default_all_public_streams, \ - do_change_default_events_register_stream, do_change_default_sending_stream, \ - do_create_user, do_deactivate_user, do_reactivate_user, do_regenerate_api_key, \ - check_change_full_name, notify_created_bot, do_update_outgoing_webhook_service, \ - do_update_bot_config_data, check_change_bot_full_name, \ - do_update_user_custom_profile_data_if_changed, check_remove_custom_profile_field_value +from zerver.decorator import require_member_or_admin, require_realm_admin +from zerver.forms import PASSWORD_TOO_WEAK_ERROR, CreateUserForm +from zerver.lib.actions import ( + check_change_bot_full_name, + check_change_full_name, + check_remove_custom_profile_field_value, + do_change_avatar_fields, + do_change_bot_owner, + do_change_default_all_public_streams, + do_change_default_events_register_stream, + do_change_default_sending_stream, + do_change_user_role, + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_regenerate_api_key, + do_update_bot_config_data, + do_update_outgoing_webhook_service, + do_update_user_custom_profile_data_if_changed, + notify_created_bot, +) from zerver.lib.avatar import avatar_url, get_gravatar_url from zerver.lib.bot_config import set_bot_config from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.exceptions import CannotDeactivateLastUserError from zerver.lib.integrations import EMBEDDED_BOTS -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.streams import access_stream_by_name, access_stream_by_id, subscribed_to_stream +from zerver.lib.streams import access_stream_by_id, access_stream_by_name, subscribed_to_stream from zerver.lib.upload import upload_avatar_image -from zerver.lib.validator import check_bool, check_string, check_int, check_url, check_dict, check_list, \ - check_int_in from zerver.lib.url_encoding import add_query_arg_to_redirect_url -from zerver.lib.users import check_valid_bot_type, check_bot_creation_policy, \ - check_full_name, check_short_name, check_valid_interface_type, check_valid_bot_config, \ - access_bot_by_id, add_service, access_user_by_id, check_bot_name_available, \ - validate_user_custom_profile_data, get_raw_user_data, get_api_key +from zerver.lib.users import ( + access_bot_by_id, + access_user_by_id, + add_service, + check_bot_creation_policy, + check_bot_name_available, + check_full_name, + check_short_name, + check_valid_bot_config, + check_valid_bot_type, + check_valid_interface_type, + get_api_key, + get_raw_user_data, + validate_user_custom_profile_data, +) from zerver.lib.utils import generate_api_key -from zerver.models import UserProfile, Stream, Message, \ - get_user_by_delivery_email, Service, get_user_including_cross_realm, \ - DomainNotAllowedForRealmError, DisposableEmailError, get_user_profile_by_id_in_realm, \ - EmailContainsPlusError, get_user_by_id_in_realm_including_cross_realm, Realm, \ - InvalidFakeEmailDomain +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_int_in, + check_list, + check_string, + check_url, +) +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + InvalidFakeEmailDomain, + Message, + Realm, + Service, + Stream, + UserProfile, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, + get_user_profile_by_id_in_realm, +) from zproject.backends import check_password_strength + def check_last_owner(user_profile: UserProfile) -> bool: owners = set(user_profile.realm.get_human_owner_users()) return user_profile.is_realm_owner and not user_profile.is_bot and len(owners) == 1 diff --git a/zerver/views/video_calls.py b/zerver/views/video_calls.py index a1c43f084b..f36373702f 100644 --- a/zerver/views/video_calls.py +++ b/zerver/views/video_calls.py @@ -1,10 +1,11 @@ -from functools import partial import json +from functools import partial from typing import Dict from urllib.parse import urljoin +import requests from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.middleware import csrf from django.shortcuts import redirect, render from django.utils.crypto import constant_time_compare, salted_hmac @@ -13,7 +14,6 @@ from django.views.decorators.cache import never_cache from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST from oauthlib.oauth2 import OAuth2Error -import requests from requests_oauthlib import OAuth2Session from zerver.decorator import REQ, has_request_variables, zulip_login_required diff --git a/zerver/views/zephyr.py b/zerver/views/zephyr.py index 99db0796ae..71c12ce588 100644 --- a/zerver/views/zephyr.py +++ b/zerver/views/zephyr.py @@ -1,22 +1,21 @@ -from django.conf import settings -from django.http import HttpResponse, HttpRequest -from django.utils.translation import ugettext as _ -from zerver.decorator import authenticated_json_view -from zerver.lib.ccache import make_ccache -from zerver.lib.pysa import mark_sanitized -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.lib.users import get_api_key -from zerver.models import UserProfile - import base64 import logging import re import subprocess -import ujson - from typing import Optional +import ujson +from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + +from zerver.decorator import authenticated_json_view +from zerver.lib.ccache import make_ccache +from zerver.lib.pysa import mark_sanitized +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.users import get_api_key +from zerver.models import UserProfile # Hack for mit.edu users whose Kerberos usernames don't match what they zephyr # as. The key is for Kerberos and the value is for zephyr. diff --git a/zerver/webhooks/alertmanager/view.py b/zerver/webhooks/alertmanager/view.py index 4156da5e72..11177d1008 100644 --- a/zerver/webhooks/alertmanager/view.py +++ b/zerver/webhooks/alertmanager/view.py @@ -9,6 +9,7 @@ from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile + @api_key_only_webhook_view('AlertManager') @has_request_variables def api_alertmanager_webhook(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/webhooks/ansibletower/view.py b/zerver/webhooks/ansibletower/view.py index b41329c8f5..77d0676d18 100644 --- a/zerver/webhooks/ansibletower/view.py +++ b/zerver/webhooks/ansibletower/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict, List from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/appveyor/view.py b/zerver/webhooks/appveyor/view.py index 9caadc9fe8..40914e1ef2 100644 --- a/zerver/webhooks/appveyor/view.py +++ b/zerver/webhooks/appveyor/view.py @@ -2,8 +2,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/basecamp/view.py b/zerver/webhooks/basecamp/view.py index 8ba5412070..74793d3826 100644 --- a/zerver/webhooks/basecamp/view.py +++ b/zerver/webhooks/basecamp/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile from .support_event import SUPPORT_EVENTS diff --git a/zerver/webhooks/beanstalk/tests.py b/zerver/webhooks/beanstalk/tests.py index 162ce7b708..fe6f56e022 100644 --- a/zerver/webhooks/beanstalk/tests.py +++ b/zerver/webhooks/beanstalk/tests.py @@ -1,5 +1,4 @@ from typing import Dict - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/beanstalk/view.py b/zerver/webhooks/beanstalk/view.py index 114642cdc3..2e23e45788 100644 --- a/zerver/webhooks/beanstalk/view.py +++ b/zerver/webhooks/beanstalk/view.py @@ -12,8 +12,7 @@ from zerver.lib.response import json_success from zerver.lib.types import ViewFuncT from zerver.lib.validator import check_dict from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - get_push_commits_event_message +from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, get_push_commits_event_message from zerver.models import UserProfile diff --git a/zerver/webhooks/bitbucket/tests.py b/zerver/webhooks/bitbucket/tests.py index b1979283ad..61fe6cf6ac 100644 --- a/zerver/webhooks/bitbucket/tests.py +++ b/zerver/webhooks/bitbucket/tests.py @@ -1,5 +1,4 @@ from typing import Dict, Union - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/bitbucket/view.py b/zerver/webhooks/bitbucket/view.py index 583ef1ce71..e2d9579add 100644 --- a/zerver/webhooks/bitbucket/view.py +++ b/zerver/webhooks/bitbucket/view.py @@ -7,8 +7,7 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_dict from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - get_push_commits_event_message +from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, get_push_commits_event_message from zerver.models import UserProfile diff --git a/zerver/webhooks/bitbucket2/view.py b/zerver/webhooks/bitbucket2/view.py index 1333b62f77..35bae5d0ba 100644 --- a/zerver/webhooks/bitbucket2/view.py +++ b/zerver/webhooks/bitbucket2/view.py @@ -10,13 +10,22 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, validate_extract_webhook_http_header -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, get_commits_comment_action_message, \ - get_force_push_commits_event_message, get_issue_event_message, \ - get_pull_request_event_message, get_push_commits_event_message, \ - get_push_tag_event_message, get_remove_branch_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_force_push_commits_event_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile BITBUCKET_TOPIC_TEMPLATE = '{repository_name}' diff --git a/zerver/webhooks/bitbucket3/view.py b/zerver/webhooks/bitbucket3/view.py index 29dc805f03..cb6bae92d7 100644 --- a/zerver/webhooks/bitbucket3/view.py +++ b/zerver/webhooks/bitbucket3/view.py @@ -8,16 +8,23 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message -from zerver.lib.webhooks.git import CONTENT_MESSAGE_TEMPLATE, \ - TOPIC_WITH_BRANCH_TEMPLATE, TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, \ - get_commits_comment_action_message, get_create_branch_event_message, \ - get_pull_request_event_message, get_push_tag_event_message, \ - get_remove_branch_event_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message +from zerver.lib.webhooks.git import ( + CONTENT_MESSAGE_TEMPLATE, + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_create_branch_event_message, + get_pull_request_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile -from zerver.webhooks.bitbucket2.view import BITBUCKET_FORK_BODY, \ - BITBUCKET_REPO_UPDATED_CHANGED, BITBUCKET_TOPIC_TEMPLATE +from zerver.webhooks.bitbucket2.view import ( + BITBUCKET_FORK_BODY, + BITBUCKET_REPO_UPDATED_CHANGED, + BITBUCKET_TOPIC_TEMPLATE, +) BRANCH_UPDATED_MESSAGE_TEMPLATE = "{user_name} pushed to branch {branch_name}. Head is now {head}." PULL_REQUEST_MARKED_AS_NEEDS_WORK_TEMPLATE = "{user_name} marked [PR #{number}]({url}) as \"needs work\"." diff --git a/zerver/webhooks/clubhouse/tests.py b/zerver/webhooks/clubhouse/tests.py index 72f802c430..76494504ea 100644 --- a/zerver/webhooks/clubhouse/tests.py +++ b/zerver/webhooks/clubhouse/tests.py @@ -1,5 +1,4 @@ import json - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/clubhouse/view.py b/zerver/webhooks/clubhouse/view.py index 68628753e9..2987cddd72 100644 --- a/zerver/webhooks/clubhouse/view.py +++ b/zerver/webhooks/clubhouse/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile EPIC_NAME_TEMPLATE = "**{name}**" diff --git a/zerver/webhooks/flock/view.py b/zerver/webhooks/flock/view.py index 99f2232303..dd8938387d 100644 --- a/zerver/webhooks/flock/view.py +++ b/zerver/webhooks/flock/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/github/view.py b/zerver/webhooks/github/view.py index 7fd1510f04..1e677e0c67 100644 --- a/zerver/webhooks/github/view.py +++ b/zerver/webhooks/github/view.py @@ -8,15 +8,24 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header -from zerver.lib.webhooks.git import CONTENT_MESSAGE_TEMPLATE, \ - TOPIC_WITH_BRANCH_TEMPLATE, TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, \ - get_commits_comment_action_message, get_issue_event_message, \ - get_pull_request_event_message, get_push_commits_event_message, \ - get_push_tag_event_message, get_setup_webhook_message, \ - get_release_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + CONTENT_MESSAGE_TEMPLATE, + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_release_event_message, + get_setup_webhook_message, +) from zerver.models import UserProfile fixture_to_headers = get_http_headers_from_filename("HTTP_X_GITHUB_EVENT") diff --git a/zerver/webhooks/gitlab/view.py b/zerver/webhooks/gitlab/view.py index 80a6178070..6e10e2e19c 100644 --- a/zerver/webhooks/gitlab/view.py +++ b/zerver/webhooks/gitlab/view.py @@ -8,13 +8,21 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, validate_extract_webhook_http_header -from zerver.lib.webhooks.git import EMPTY_SHA, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, get_commits_comment_action_message, \ - get_issue_event_message, get_pull_request_event_message, \ - get_push_commits_event_message, get_push_tag_event_message, \ - get_remove_branch_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + EMPTY_SHA, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile diff --git a/zerver/webhooks/gogs/view.py b/zerver/webhooks/gogs/view.py index e43f2b75ad..2f4a46300f 100644 --- a/zerver/webhooks/gogs/view.py +++ b/zerver/webhooks/gogs/view.py @@ -6,14 +6,22 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, TOPIC_WITH_RELEASE_TEMPLATE, \ - get_create_branch_event_message, \ - get_issue_event_message, get_pull_request_event_message, \ - get_push_commits_event_message, get_release_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + TOPIC_WITH_RELEASE_TEMPLATE, + get_create_branch_event_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_release_event_message, +) from zerver.models import UserProfile fixture_to_headers = get_http_headers_from_filename("HTTP_X_GOGS_EVENT") diff --git a/zerver/webhooks/gosquared/view.py b/zerver/webhooks/gosquared/view.py index ec6aacd5a3..0c9affba7d 100644 --- a/zerver/webhooks/gosquared/view.py +++ b/zerver/webhooks/gosquared/view.py @@ -5,8 +5,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile TRAFFIC_SPIKE_TEMPLATE = '[{website_name}]({website_url}) has {user_num} visitors online.' diff --git a/zerver/webhooks/grafana/view.py b/zerver/webhooks/grafana/view.py index 9051d7aaec..ebd70a8b66 100644 --- a/zerver/webhooks/grafana/view.py +++ b/zerver/webhooks/grafana/view.py @@ -3,9 +3,9 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view -from zerver.lib.webhooks.common import check_send_webhook_message from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success +from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile GRAFANA_TOPIC_TEMPLATE = '{alert_title}' diff --git a/zerver/webhooks/groove/view.py b/zerver/webhooks/groove/view.py index 76b508056f..038003bdfa 100644 --- a/zerver/webhooks/groove/view.py +++ b/zerver/webhooks/groove/view.py @@ -7,9 +7,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile TICKET_STARTED_TEMPLATE = """ diff --git a/zerver/webhooks/harbor/view.py b/zerver/webhooks/harbor/view.py index 64a6846cb7..3dd7a35c05 100644 --- a/zerver/webhooks/harbor/view.py +++ b/zerver/webhooks/harbor/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import Realm, UserProfile IGNORED_EVENTS = [ diff --git a/zerver/webhooks/hellosign/tests.py b/zerver/webhooks/hellosign/tests.py index 2cadf71664..79738fddac 100644 --- a/zerver/webhooks/hellosign/tests.py +++ b/zerver/webhooks/hellosign/tests.py @@ -1,6 +1,8 @@ -from zerver.lib.test_classes import WebhookTestCase from typing import Dict +from zerver.lib.test_classes import WebhookTestCase + + class HelloSignHookTests(WebhookTestCase): STREAM_NAME = 'hellosign' URL_TEMPLATE = "/api/v1/external/hellosign?stream={stream}&api_key={api_key}" diff --git a/zerver/webhooks/hellosign/view.py b/zerver/webhooks/hellosign/view.py index 6dd775af05..27018dd6b3 100644 --- a/zerver/webhooks/hellosign/view.py +++ b/zerver/webhooks/hellosign/view.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List +import ujson from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view @@ -7,7 +8,6 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile -import ujson IS_AWAITING_SIGNATURE = "is awaiting the signature of {awaiting_recipients}" WAS_JUST_SIGNED_BY = "was just signed by {signed_recipients}" diff --git a/zerver/webhooks/insping/view.py b/zerver/webhooks/insping/view.py index 152de3ad44..3acea44451 100644 --- a/zerver/webhooks/insping/view.py +++ b/zerver/webhooks/insping/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/intercom/view.py b/zerver/webhooks/intercom/view.py index 35960af125..fcee857de7 100644 --- a/zerver/webhooks/intercom/view.py +++ b/zerver/webhooks/intercom/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile COMPANY_CREATED = """ diff --git a/zerver/webhooks/jira/view.py b/zerver/webhooks/jira/view.py index 05f8c6aef7..ab095687df 100644 --- a/zerver/webhooks/jira/view.py +++ b/zerver/webhooks/jira/view.py @@ -9,8 +9,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import Realm, UserProfile, get_user_by_delivery_email IGNORED_EVENTS = [ diff --git a/zerver/webhooks/netlify/view.py b/zerver/webhooks/netlify/view.py index 1f06303fca..1c256b1dfe 100644 --- a/zerver/webhooks/netlify/view.py +++ b/zerver/webhooks/netlify/view.py @@ -5,9 +5,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile EVENTS = ['deploy_failed', 'deploy_locked', 'deploy_unlocked', 'deploy_building', 'deploy_created'] diff --git a/zerver/webhooks/newrelic/view.py b/zerver/webhooks/newrelic/view.py index 7848c811ab..8ef9bf0f2e 100644 --- a/zerver/webhooks/newrelic/view.py +++ b/zerver/webhooks/newrelic/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_dict -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile ALERT_TEMPLATE = "{long_description} ([view alert]({alert_url}))." diff --git a/zerver/webhooks/pagerduty/view.py b/zerver/webhooks/pagerduty/view.py index 69576c38df..9267bb9811 100644 --- a/zerver/webhooks/pagerduty/view.py +++ b/zerver/webhooks/pagerduty/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile PAGER_DUTY_EVENT_NAMES = { diff --git a/zerver/webhooks/pingdom/view.py b/zerver/webhooks/pingdom/view.py index 1cb1ec5aed..16ac895043 100644 --- a/zerver/webhooks/pingdom/view.py +++ b/zerver/webhooks/pingdom/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile PINGDOM_TOPIC_TEMPLATE = '{name} status.' diff --git a/zerver/webhooks/pivotal/view.py b/zerver/webhooks/pivotal/view.py index 3a56f4b2dd..6fd45a8275 100644 --- a/zerver/webhooks/pivotal/view.py +++ b/zerver/webhooks/pivotal/view.py @@ -10,8 +10,7 @@ from django.utils.translation import ugettext as _ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/raygun/view.py b/zerver/webhooks/raygun/view.py index cace93e5bb..497c9c9e27 100644 --- a/zerver/webhooks/raygun/view.py +++ b/zerver/webhooks/raygun/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/reviewboard/view.py b/zerver/webhooks/reviewboard/view.py index e89d2122b1..e5e024d3ef 100644 --- a/zerver/webhooks/reviewboard/view.py +++ b/zerver/webhooks/reviewboard/view.py @@ -5,9 +5,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile REVIEW_REQUEST_PUBLISHED = """ diff --git a/zerver/webhooks/semaphore/tests.py b/zerver/webhooks/semaphore/tests.py index bcb487f17a..83356b5601 100644 --- a/zerver/webhooks/semaphore/tests.py +++ b/zerver/webhooks/semaphore/tests.py @@ -1,5 +1,7 @@ -import ujson from unittest.mock import patch + +import ujson + from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/semaphore/view.py b/zerver/webhooks/semaphore/view.py index 77b0979a86..4fe0fc3242 100644 --- a/zerver/webhooks/semaphore/view.py +++ b/zerver/webhooks/semaphore/view.py @@ -1,6 +1,5 @@ # Webhooks for external integrations. -from typing import Any, Dict, Tuple, Optional - +from typing import Any, Dict, Optional, Tuple from urllib.parse import urlparse from django.http import HttpRequest, HttpResponse diff --git a/zerver/webhooks/sentry/view.py b/zerver/webhooks/sentry/view.py index 9d2aa7977c..4e42b6c31c 100644 --- a/zerver/webhooks/sentry/view.py +++ b/zerver/webhooks/sentry/view.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Optional, Tuple from django.http import HttpRequest, HttpResponse @@ -8,7 +8,6 @@ from zerver.lib.response import json_success from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile - DEPRECATED_EXCEPTION_MESSAGE_TEMPLATE = """ New [issue]({url}) (level: {level}): diff --git a/zerver/webhooks/slack_incoming/view.py b/zerver/webhooks/slack_incoming/view.py index 418953c657..67eb9d9003 100644 --- a/zerver/webhooks/slack_incoming/view.py +++ b/zerver/webhooks/slack_incoming/view.py @@ -1,17 +1,18 @@ # Webhooks for external integrations. +import re from typing import Any, Dict, Optional +import ujson from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ from zerver.decorator import api_key_only_webhook_view +from zerver.lib.exceptions import InvalidJSONError from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.exceptions import InvalidJSONError -from django.utils.translation import ugettext as _ from zerver.models import UserProfile -import re -import ujson + @api_key_only_webhook_view('SlackIncoming') @has_request_variables diff --git a/zerver/webhooks/statuspage/view.py b/zerver/webhooks/statuspage/view.py index e5fa6b3f3b..cf8e820b31 100644 --- a/zerver/webhooks/statuspage/view.py +++ b/zerver/webhooks/statuspage/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/stripe/view.py b/zerver/webhooks/stripe/view.py index 57d542f748..4b2ed311b9 100644 --- a/zerver/webhooks/stripe/view.py +++ b/zerver/webhooks/stripe/view.py @@ -8,8 +8,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.timestamp import timestamp_to_datetime -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/teamcity/tests.py b/zerver/webhooks/teamcity/tests.py index 268c90d078..db3e8acee7 100644 --- a/zerver/webhooks/teamcity/tests.py +++ b/zerver/webhooks/teamcity/tests.py @@ -3,8 +3,7 @@ import ujson from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import WebhookTestCase from zerver.models import Recipient, get_realm, get_user_by_delivery_email -from zerver.webhooks.teamcity.view import \ - MISCONFIGURED_PAYLOAD_TYPE_ERROR_MESSAGE +from zerver.webhooks.teamcity.view import MISCONFIGURED_PAYLOAD_TYPE_ERROR_MESSAGE class TeamcityHookTests(WebhookTestCase): diff --git a/zerver/webhooks/teamcity/view.py b/zerver/webhooks/teamcity/view.py index 2dc4dcecf7..b83cfb0c82 100644 --- a/zerver/webhooks/teamcity/view.py +++ b/zerver/webhooks/teamcity/view.py @@ -6,8 +6,10 @@ from django.db.models import Q from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view -from zerver.lib.actions import check_send_private_message, \ - send_rate_limited_pm_notification_to_bot_owner +from zerver.lib.actions import ( + check_send_private_message, + send_rate_limited_pm_notification_to_bot_owner, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.send_email import FromAddress diff --git a/zerver/webhooks/transifex/view.py b/zerver/webhooks/transifex/view.py index 8614807fa0..7f22564963 100644 --- a/zerver/webhooks/transifex/view.py +++ b/zerver/webhooks/transifex/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_int -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/trello/view/__init__.py b/zerver/webhooks/trello/view/__init__.py index c455383037..e6a01d0c3c 100644 --- a/zerver/webhooks/trello/view/__init__.py +++ b/zerver/webhooks/trello/view/__init__.py @@ -4,17 +4,14 @@ from typing import Any, Mapping, Optional, Tuple import ujson from django.http import HttpRequest, HttpResponse -from zerver.decorator import api_key_only_webhook_view, \ - return_success_on_head_request +from zerver.decorator import api_key_only_webhook_view, return_success_on_head_request from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile from .board_actions import SUPPORTED_BOARD_ACTIONS, process_board_action -from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, \ - process_card_action +from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, process_card_action @api_key_only_webhook_view('Trello') diff --git a/zerver/webhooks/updown/view.py b/zerver/webhooks/updown/view.py index 8151518822..d0c49b2eca 100644 --- a/zerver/webhooks/updown/view.py +++ b/zerver/webhooks/updown/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile TOPIC_TEMPLATE = "{service_url}" diff --git a/zerver/webhooks/zabbix/view.py b/zerver/webhooks/zabbix/view.py index 846dcc4bbd..5a59bd7c82 100644 --- a/zerver/webhooks/zabbix/view.py +++ b/zerver/webhooks/zabbix/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.actions import send_rate_limited_pm_notification_to_bot_owner from zerver.lib.response import json_error, json_success from zerver.lib.send_email import FromAddress diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index 1aa7cb202f..b647706469 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -1,61 +1,97 @@ # Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Mapping, MutableSequence, Optional, cast, Tuple, TypeVar, Type - import copy +import datetime +import email +import logging +import os import signal -import tempfile -from functools import wraps -from threading import Timer - import smtplib import socket +import tempfile +import time +import urllib +from abc import ABC, abstractmethod +from collections import defaultdict, deque +from functools import wraps +from threading import Timer +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + MutableSequence, + Optional, + Tuple, + Type, + TypeVar, + cast, +) +import requests +import ujson from django.conf import settings from django.db import connection from django.utils.timezone import now as timezone_now -from zerver.models import \ - get_client, get_system_bot, PreregistrationUser, \ - get_user_profile_by_id, Message, Realm, UserMessage, UserProfile, \ - Client, flush_per_request_caches -from zerver.lib.context_managers import lockfile -from zerver.lib.error_notify import do_report_error -from zerver.lib.queue import SimpleQueueClient, retry_event -from zerver.lib.timestamp import timestamp_to_datetime -from zerver.lib.email_notifications import handle_missedmessage_emails -from zerver.lib.push_notifications import handle_push_notification, handle_remove_push_notification, \ - initialize_push_notifications, clear_push_device_tokens -from zerver.lib.actions import do_send_confirmation_email, \ - do_update_user_activity, do_update_user_activity_interval, do_update_user_presence, \ - internal_send_private_message, notify_realm_export, \ - render_incoming_message, do_update_embedded_data, do_mark_stream_messages_as_read -from zerver.lib.url_preview import preview as url_preview -from zerver.lib.digest import handle_digest_email -from zerver.lib.send_email import send_future_email, send_email_from_dict, \ - FromAddress, EmailNotDeliveredException, handle_send_email_format_changes -from zerver.lib.email_mirror import process_message as mirror_email, rate_limit_mirror_by_realm, \ - is_missed_message_address, decode_stream_email_address -from zerver.lib.streams import access_stream_by_id -from zerver.lib.db import reset_queries -from zerver.context_processors import common_context -from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler -from zerver.models import get_bot_services, RealmAuditLog from zulip_bots.lib import ExternalBotHandler, extract_query_without_mention -from zerver.lib.bot_lib import EmbeddedBotHandler, get_bot_handler, EmbeddedBotQuitException + +from zerver.context_processors import common_context +from zerver.lib.actions import ( + do_mark_stream_messages_as_read, + do_send_confirmation_email, + do_update_embedded_data, + do_update_user_activity, + do_update_user_activity_interval, + do_update_user_presence, + internal_send_private_message, + notify_realm_export, + render_incoming_message, +) +from zerver.lib.bot_lib import EmbeddedBotHandler, EmbeddedBotQuitException, get_bot_handler +from zerver.lib.context_managers import lockfile +from zerver.lib.db import reset_queries +from zerver.lib.digest import handle_digest_email +from zerver.lib.email_mirror import decode_stream_email_address, is_missed_message_address +from zerver.lib.email_mirror import process_message as mirror_email +from zerver.lib.email_mirror import rate_limit_mirror_by_realm +from zerver.lib.email_notifications import handle_missedmessage_emails +from zerver.lib.error_notify import do_report_error from zerver.lib.exceptions import RateLimited from zerver.lib.export import export_realm_wrapper -from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError +from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler +from zerver.lib.push_notifications import ( + clear_push_device_tokens, + handle_push_notification, + handle_remove_push_notification, + initialize_push_notifications, +) from zerver.lib.pysa import mark_sanitized - -import os -import ujson -from collections import defaultdict, deque -import email -import time -import datetime -import logging -import requests -import urllib +from zerver.lib.queue import SimpleQueueClient, retry_event +from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError +from zerver.lib.send_email import ( + EmailNotDeliveredException, + FromAddress, + handle_send_email_format_changes, + send_email_from_dict, + send_future_email, +) +from zerver.lib.streams import access_stream_by_id +from zerver.lib.timestamp import timestamp_to_datetime +from zerver.lib.url_preview import preview as url_preview +from zerver.models import ( + Client, + Message, + PreregistrationUser, + Realm, + RealmAuditLog, + UserMessage, + UserProfile, + flush_per_request_caches, + get_bot_services, + get_client, + get_system_bot, + get_user_profile_by_id, +) logger = logging.getLogger(__name__) diff --git a/zilencer/forms.py b/zilencer/forms.py index 4a2352e749..b6f90c3832 100644 --- a/zilencer/forms.py +++ b/zilencer/forms.py @@ -1,5 +1,6 @@ from django import forms + class EnterpriseToSForm(forms.Form): full_name = forms.CharField(max_length=100) company = forms.CharField(max_length=100) diff --git a/zilencer/management/commands/add_mock_conversation.py b/zilencer/management/commands/add_mock_conversation.py index 69f40877e2..ae50776598 100644 --- a/zilencer/management/commands/add_mock_conversation.py +++ b/zilencer/management/commands/add_mock_conversation.py @@ -2,9 +2,15 @@ from typing import Any, Dict, List from django.core.management.base import BaseCommand -from zerver.lib.actions import bulk_add_subscriptions, do_add_reaction, \ - do_change_avatar_fields, do_create_user, do_send_messages, ensure_stream, \ - internal_prep_stream_message +from zerver.lib.actions import ( + bulk_add_subscriptions, + do_add_reaction, + do_change_avatar_fields, + do_create_user, + do_send_messages, + ensure_stream, + internal_prep_stream_message, +) from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.lib.upload import upload_avatar_image from zerver.models import Message, UserProfile, get_realm diff --git a/zilencer/management/commands/add_new_realm.py b/zilencer/management/commands/add_new_realm.py index a78723114d..170952ce56 100644 --- a/zilencer/management/commands/add_new_realm.py +++ b/zilencer/management/commands/add_new_realm.py @@ -1,7 +1,6 @@ from typing import Any -from zerver.lib.actions import bulk_add_subscriptions, do_create_realm, \ - do_create_user +from zerver.lib.actions import bulk_add_subscriptions, do_create_realm, do_create_user from zerver.lib.management import ZulipBaseCommand from zerver.lib.onboarding import send_initial_realm_messages from zerver.models import Realm, UserProfile diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 86c3421ad8..0676eaa773 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -1,11 +1,11 @@ import itertools import os import random -from typing import Any, Callable, Dict, List, \ - Mapping, Sequence, Tuple - -import ujson from datetime import datetime +from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple + +import pylibmc +import ujson from django.conf import settings from django.contrib.sessions.models import Session from django.core.management import call_command @@ -13,11 +13,17 @@ from django.core.management.base import BaseCommand, CommandParser from django.db.models import F, Max from django.utils.timezone import now as timezone_now from django.utils.timezone import timedelta as timezone_timedelta -import pylibmc -from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, check_add_realm_emoji, \ - do_change_user_role, do_send_messages, do_update_user_custom_profile_data_if_changed, \ - try_add_realm_custom_profile_field, try_add_realm_default_custom_profile_field +from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.lib.actions import ( + STREAM_ASSIGNMENT_COLORS, + check_add_realm_emoji, + do_change_user_role, + do_send_messages, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_add_realm_default_custom_profile_field, +) from zerver.lib.bulk_create import bulk_create_streams from zerver.lib.cache import cache_set from zerver.lib.generate_test_data import create_test_data, generate_topics @@ -25,18 +31,35 @@ from zerver.lib.onboarding import create_if_missing_realm_internal_bots from zerver.lib.push_notifications import logger as push_notifications_logger from zerver.lib.server_initialization import create_internal_realm, create_users from zerver.lib.storage import static_path -from zerver.lib.users import add_service +from zerver.lib.types import ProfileFieldData from zerver.lib.url_preview.preview import CACHE_NAME as PREVIEW_CACHE_NAME from zerver.lib.user_groups import create_user_group +from zerver.lib.users import add_service from zerver.lib.utils import generate_api_key -from zerver.models import CustomProfileField, DefaultStream, Message, Realm, RealmAuditLog, \ - RealmDomain, Recipient, Service, Stream, Subscription, \ - UserMessage, UserPresence, UserProfile, Huddle, Client, \ - get_client, get_huddle, get_realm, get_stream, \ - get_user, get_user_by_delivery_email, get_user_profile_by_id -from zerver.lib.types import ProfileFieldData - -from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.models import ( + Client, + CustomProfileField, + DefaultStream, + Huddle, + Message, + Realm, + RealmAuditLog, + RealmDomain, + Recipient, + Service, + Stream, + Subscription, + UserMessage, + UserPresence, + UserProfile, + get_client, + get_huddle, + get_realm, + get_stream, + get_user, + get_user_by_delivery_email, + get_user_profile_by_id, +) settings.TORNADO_SERVER = None # Disable using memcached caches to avoid 'unsupported pickle diff --git a/zilencer/models.py b/zilencer/models.py index 91f93606cb..34481bbeb5 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -2,8 +2,9 @@ import datetime from django.db import models -from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog from analytics.models import BaseCount +from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog + def get_remote_server_by_uuid(uuid: str) -> 'RemoteZulipServer': return RemoteZulipServer.objects.get(uuid=uuid) diff --git a/zilencer/views.py b/zilencer/views.py index fc137a4149..ad659c4f1c 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -1,29 +1,46 @@ -from typing import Any, Dict, List, Optional, Union import datetime import logging +from typing import Any, Dict, List, Optional, Union from django.core.exceptions import ValidationError -from django.core.validators import validate_email, URLValidator +from django.core.validators import URLValidator, validate_email from django.db import IntegrityError, transaction from django.http import HttpRequest, HttpResponse from django.utils import timezone -from django.utils.translation import ugettext as _, ugettext as err_ +from django.utils.translation import ugettext as _ +from django.utils.translation import ugettext as err_ from django.views.decorators.csrf import csrf_exempt from analytics.lib.counts import COUNT_STATS -from zerver.decorator import require_post, InvalidZulipServerKeyError +from zerver.decorator import InvalidZulipServerKeyError, require_post from zerver.lib.exceptions import JsonableError -from zerver.lib.push_notifications import send_android_push_notification, \ - send_apple_push_notification +from zerver.lib.push_notifications import ( + send_android_push_notification, + send_apple_push_notification, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_int, check_string, \ - check_capped_string, check_string_fixed_length, check_float, check_none_or, \ - check_dict_only, check_list, check_bool +from zerver.lib.validator import ( + check_bool, + check_capped_string, + check_dict_only, + check_float, + check_int, + check_list, + check_none_or, + check_string, + check_string_fixed_length, +) from zerver.models import UserProfile from zerver.views.push_notifications import validate_token -from zilencer.models import RemotePushDeviceToken, RemoteZulipServer, \ - RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog +from zilencer.models import ( + RemoteInstallationCount, + RemotePushDeviceToken, + RemoteRealmAuditLog, + RemoteRealmCount, + RemoteZulipServer, +) + def validate_entity(entity: Union[UserProfile, RemoteZulipServer]) -> RemoteZulipServer: if not isinstance(entity, RemoteZulipServer): diff --git a/zproject/backends.py b/zproject/backends.py index 56e6b11fe4..b53aac21b9 100644 --- a/zproject/backends.py +++ b/zproject/backends.py @@ -15,67 +15,81 @@ import binascii import copy import logging +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, cast + import jwt import magic import ujson -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, \ - cast -from typing_extensions import TypedDict -from zxcvbn import zxcvbn - -from django_auth_ldap.backend import LDAPBackend, LDAPReverseEmailSearch, \ - _LDAPUser, ldap_error from decorator import decorator - +from django.conf import settings from django.contrib.auth import authenticate, get_backends from django.contrib.auth.backends import RemoteUserBackend -from django.conf import settings from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.dispatch import receiver, Signal -from django.http import HttpResponse, HttpResponseRedirect, HttpRequest +from django.dispatch import Signal, receiver +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.urls import reverse from django.utils.translation import ugettext as _ +from django_auth_ldap.backend import LDAPBackend, LDAPReverseEmailSearch, _LDAPUser, ldap_error from jwt.algorithms import RSAAlgorithm from jwt.exceptions import PyJWTError from lxml.etree import XMLSyntaxError -from requests import HTTPError from onelogin.saml2.errors import OneLogin_Saml2_Error from onelogin.saml2.response import OneLogin_Saml2_Response - -from social_core.backends.github import GithubOAuth2, GithubOrganizationOAuth2, \ - GithubTeamOAuth2 -from social_core.backends.azuread import AzureADOAuth2 -from social_core.backends.gitlab import GitLabOAuth2 -from social_core.backends.base import BaseAuth -from social_core.backends.google import GoogleOAuth2 +from requests import HTTPError from social_core.backends.apple import AppleIdAuth +from social_core.backends.azuread import AzureADOAuth2 +from social_core.backends.base import BaseAuth +from social_core.backends.github import GithubOAuth2, GithubOrganizationOAuth2, GithubTeamOAuth2 +from social_core.backends.gitlab import GitLabOAuth2 +from social_core.backends.google import GoogleOAuth2 from social_core.backends.saml import SAMLAuth +from social_core.exceptions import ( + AuthFailed, + AuthMissingParameter, + AuthStateForbidden, + SocialAuthBaseException, +) from social_core.pipeline.partial import partial -from social_core.exceptions import AuthFailed, SocialAuthBaseException, \ - AuthMissingParameter, AuthStateForbidden +from typing_extensions import TypedDict +from zxcvbn import zxcvbn from zerver.decorator import client_is_exempt_from_rate_limiting -from zerver.lib.actions import do_create_user, do_reactivate_user, do_deactivate_user, \ - do_update_user_custom_profile_data_if_changed -from zerver.lib.avatar import is_avatar_new, avatar_url +from zerver.lib.actions import ( + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_update_user_custom_profile_data_if_changed, +) +from zerver.lib.avatar import avatar_url, is_avatar_new from zerver.lib.avatar_hash import user_avatar_content_hash from zerver.lib.create_user import get_role_for_new_user from zerver.lib.dev_ldap_directory import init_fakeldap -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm from zerver.lib.mobile_auth_otp import is_valid_otp from zerver.lib.rate_limiter import RateLimitedObject +from zerver.lib.redis_utils import get_dict_from_redis, get_redis_client, put_dict_in_redis from zerver.lib.request import JsonableError -from zerver.lib.users import check_full_name, validate_user_custom_profile_field -from zerver.lib.redis_utils import get_redis_client, get_dict_from_redis, put_dict_in_redis from zerver.lib.subdomains import get_subdomain -from zerver.models import CustomProfileField, DisposableEmailError, DomainNotAllowedForRealmError, \ - EmailContainsPlusError, PreregistrationUser, UserProfile, Realm, custom_profile_fields_for_realm, \ - get_user_profile_by_id, remote_user_to_email, \ - email_to_username, get_realm, get_user_by_delivery_email, supported_auth_backends +from zerver.lib.users import check_full_name, validate_user_custom_profile_field +from zerver.models import ( + CustomProfileField, + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + PreregistrationUser, + Realm, + UserProfile, + custom_profile_fields_for_realm, + email_to_username, + get_realm, + get_user_by_delivery_email, + get_user_profile_by_id, + remote_user_to_email, + supported_auth_backends, +) redis_client = get_redis_client() @@ -505,10 +519,11 @@ class ZulipLDAPAuthBackendBase(ZulipAuthMixin, LDAPBackend): def sync_avatar_from_ldap(self, user: UserProfile, ldap_user: _LDAPUser) -> None: if 'avatar' in settings.AUTH_LDAP_USER_ATTR_MAP: # We do local imports here to avoid import loops - from zerver.lib.upload import upload_avatar_image - from zerver.lib.actions import do_change_avatar_fields from io import BytesIO + from zerver.lib.actions import do_change_avatar_fields + from zerver.lib.upload import upload_avatar_image + avatar_attr_name = settings.AUTH_LDAP_USER_ATTR_MAP['avatar'] if avatar_attr_name not in ldap_user.attrs: # nocoverage # If this specific user doesn't have e.g. a @@ -1236,8 +1251,7 @@ def social_auth_finish(backend: Any, comments below as well as login_or_register_remote_user in `zerver/views/auth.py` for the details on how that dispatch works. """ - from zerver.views.auth import (login_or_register_remote_user, - redirect_and_log_into_subdomain) + from zerver.views.auth import login_or_register_remote_user, redirect_and_log_into_subdomain user_profile = kwargs['user_profile'] return_data = kwargs['return_data'] diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 7b2be1e10b..f456f6293a 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -1,21 +1,28 @@ -from copy import deepcopy import os -import time import sys +import time +from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import urljoin -from zerver.lib.db import TimeTrackingConnection import zerver.lib.logging_util +from zerver.lib.db import TimeTrackingConnection -from .config import DEPLOY_ROOT, PRODUCTION, DEVELOPMENT, get_secret, get_config, get_from_file_if_exists +from .config import ( + DEPLOY_ROOT, + DEVELOPMENT, + PRODUCTION, + get_config, + get_from_file_if_exists, + get_secret, +) from .configured_settings import ( ADMINS, ALLOWED_HOSTS, - AUTHENTICATION_BACKENDS, AUTH_LDAP_BIND_DN, AUTH_LDAP_CONNECTION_OPTIONS, AUTH_LDAP_SERVER_URI, + AUTHENTICATION_BACKENDS, CAMO_URI, DEBUG, DEBUG_ERROR_REPORTING, @@ -156,6 +163,8 @@ ALLOWED_HOSTS += [EXTERNAL_HOST.split(":")[0], ALLOWED_HOSTS += REALM_HOSTS.values() from django.template.loaders import app_directories + + class TwoFactorLoader(app_directories.Loader): def get_dirs(self) -> List[str]: dirs = super().get_dirs() diff --git a/zproject/config.py b/zproject/config.py index f55c616c1c..1a10b7f1f8 100644 --- a/zproject/config.py +++ b/zproject/config.py @@ -1,6 +1,6 @@ +import configparser import os from typing import Optional, overload -import configparser DEPLOY_ROOT = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) diff --git a/zproject/configured_settings.py b/zproject/configured_settings.py index 10997c8193..fc4f128e83 100644 --- a/zproject/configured_settings.py +++ b/zproject/configured_settings.py @@ -10,6 +10,7 @@ from .default_settings import * # noqa: F401,F403 isort: skip # Import variables like secrets from the prod_settings file # Import prod_settings after determining the deployment/machine type from .config import PRODUCTION + if PRODUCTION: from .prod_settings import * # noqa: F401,F403 isort: skip else: diff --git a/zproject/default_settings.py b/zproject/default_settings.py index 84285801a9..e9cccd61d6 100644 --- a/zproject/default_settings.py +++ b/zproject/default_settings.py @@ -1,10 +1,11 @@ -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: from django_auth_ldap.config import LDAPSearch from typing_extensions import TypedDict -from .config import PRODUCTION, DEVELOPMENT, get_secret +from .config import DEVELOPMENT, PRODUCTION, get_secret + if PRODUCTION: from .prod_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR else: diff --git a/zproject/dev_settings.py b/zproject/dev_settings.py index 3cd628a886..c551372800 100644 --- a/zproject/dev_settings.py +++ b/zproject/dev_settings.py @@ -113,6 +113,7 @@ FAKE_LDAP_MODE = None # type: Optional[str] if FAKE_LDAP_MODE: import ldap from django_auth_ldap.config import LDAPSearch + # To understand these parameters, read the docs in # prod_settings_template.py and on ReadTheDocs. LDAP_APPEND_DOMAIN = None diff --git a/zproject/dev_urls.py b/zproject/dev_urls.py index 254b17c981..2b0ea06b62 100644 --- a/zproject/dev_urls.py +++ b/zproject/dev_urls.py @@ -1,16 +1,18 @@ +import os from urllib.parse import urlsplit -from django.conf.urls import url + from django.conf import settings +from django.conf.urls import url from django.conf.urls.static import static from django.contrib.staticfiles.views import serve as staticfiles_serve from django.http import HttpRequest, HttpResponse from django.views.generic import TemplateView -import os from django.views.static import serve -import zerver.views.development.registration + import zerver.views.auth import zerver.views.development.email_log import zerver.views.development.integrations +import zerver.views.development.registration # These URLs are available only in the development environment diff --git a/zproject/email_backends.py b/zproject/email_backends.py index 0215694353..9a082088cc 100644 --- a/zproject/email_backends.py +++ b/zproject/email_backends.py @@ -1,17 +1,16 @@ -import logging - -from typing import List import configparser - +import logging import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from typing import List from django.conf import settings -from django.core.mail.backends.base import BaseEmailBackend from django.core.mail import EmailMultiAlternatives +from django.core.mail.backends.base import BaseEmailBackend from django.template import loader + def get_forward_address() -> str: config = configparser.ConfigParser() config.read(settings.FORWARD_ADDRESS_CONFIG_FILE) diff --git a/zproject/jinja2/__init__.py b/zproject/jinja2/__init__.py index 138a4ae683..fb8582a2fe 100644 --- a/zproject/jinja2/__init__.py +++ b/zproject/jinja2/__init__.py @@ -2,7 +2,7 @@ from typing import Any from django.conf import settings from django.contrib.staticfiles.storage import staticfiles_storage -from django.template.defaultfilters import slugify, pluralize +from django.template.defaultfilters import pluralize, slugify from django.urls import reverse from django.utils import translation from django.utils.timesince import timesince diff --git a/zproject/legacy_urls.py b/zproject/legacy_urls.py index b7f7a70f36..1366f13be0 100644 --- a/zproject/legacy_urls.py +++ b/zproject/legacy_urls.py @@ -1,9 +1,10 @@ from django.urls import path + import zerver.views -import zerver.views.streams import zerver.views.auth -import zerver.views.tutorial import zerver.views.report +import zerver.views.streams +import zerver.views.tutorial # Future endpoints should add to urls.py, which includes these legacy urls diff --git a/zproject/test_extra_settings.py b/zproject/test_extra_settings.py index 6a6bfd2753..d7d414f275 100644 --- a/zproject/test_extra_settings.py +++ b/zproject/test_extra_settings.py @@ -5,6 +5,7 @@ import ldap from django_auth_ldap.config import LDAPSearch from zerver.lib.db import TimeTrackingConnection + from .config import DEPLOY_ROOT, get_from_file_if_exists from .settings import ( AUTHENTICATION_BACKENDS, diff --git a/zproject/test_settings.py b/zproject/test_settings.py index 28522199e6..0f10b85840 100644 --- a/zproject/test_settings.py +++ b/zproject/test_settings.py @@ -1,4 +1,5 @@ import os + # test_settings.py works differently from # dev_settings.py/prod_settings.py; it actually is directly referenced # by the test suite as DJANGO_SETTINGS_MODULE and imports settings.py diff --git a/zproject/urls.py b/zproject/urls.py index a3f6bc1ba7..8349b02c26 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -1,49 +1,51 @@ -from django.conf import settings -from django.conf.urls import url, include -from django.conf.urls.i18n import i18n_patterns -from django.views.generic import TemplateView, RedirectView -from django.utils.module_loading import import_string import os + +from django.conf import settings +from django.conf.urls import include, url +from django.conf.urls.i18n import i18n_patterns +from django.contrib.auth.views import ( + LoginView, + PasswordResetCompleteView, + PasswordResetConfirmView, + PasswordResetDoneView, +) +from django.utils.module_loading import import_string +from django.views.generic import RedirectView, TemplateView + import zerver.forms -from zproject import dev_urls -from zproject.legacy_urls import legacy_urls -from zerver.views.documentation import IntegrationView, MarkdownDirectoryView -from zerver.lib.integrations import WEBHOOK_INTEGRATIONS - - -from django.contrib.auth.views import (LoginView, PasswordResetDoneView, - PasswordResetConfirmView, PasswordResetCompleteView) - import zerver.tornado.views import zerver.views -import zerver.views.auth import zerver.views.archive +import zerver.views.auth import zerver.views.camo import zerver.views.compatibility -import zerver.views.home -import zerver.views.email_mirror -import zerver.views.registration -import zerver.views.portico -import zerver.views.zephyr -import zerver.views.users -import zerver.views.unsubscribe +import zerver.views.digest import zerver.views.documentation +import zerver.views.email_mirror +import zerver.views.home +import zerver.views.messages +import zerver.views.muting +import zerver.views.portico +import zerver.views.realm +import zerver.views.realm_export +import zerver.views.registration +import zerver.views.streams +import zerver.views.unsubscribe +import zerver.views.upload import zerver.views.user_groups import zerver.views.user_settings -import zerver.views.muting -import zerver.views.streams -import zerver.views.realm -import zerver.views.digest -import zerver.views.messages -import zerver.views.realm_export -import zerver.views.upload +import zerver.views.users import zerver.views.video_calls - +import zerver.views.zephyr +from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.rest import rest_dispatch +from zerver.views.documentation import IntegrationView, MarkdownDirectoryView +from zproject import dev_urls +from zproject.legacy_urls import legacy_urls if settings.TWO_FACTOR_AUTHENTICATION_ENABLED: - from two_factor.urls import urlpatterns as tf_urls from two_factor.gateways.twilio.urls import urlpatterns as tf_twilio_urls + from two_factor.urls import urlpatterns as tf_urls # NB: There are several other pieces of code which route requests by URL: # diff --git a/zproject/wsgi.py b/zproject/wsgi.py index 0d1253c779..75144d5732 100644 --- a/zproject/wsgi.py +++ b/zproject/wsgi.py @@ -25,6 +25,7 @@ setup_path() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings") import django + try: django.setup() except Exception as e: @@ -45,10 +46,12 @@ except Exception as e: # need to import zerver.models first before the middleware tries to import it. import zerver.models + zerver.models # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. from django.core.wsgi import get_wsgi_application + application = get_wsgi_application() diff --git a/zthumbor/loaders/helpers.py b/zthumbor/loaders/helpers.py index 615e4d77c8..b595256dec 100644 --- a/zthumbor/loaders/helpers.py +++ b/zthumbor/loaders/helpers.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import os import re import sys -from typing import Any, Text, Tuple, Optional +from typing import Any, Optional, Text, Tuple ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) diff --git a/zthumbor/loaders/zloader.py b/zthumbor/loaders/zloader.py index 98bd506d77..74f694273a 100644 --- a/zthumbor/loaders/zloader.py +++ b/zthumbor/loaders/zloader.py @@ -1,20 +1,24 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html from __future__ import absolute_import -from six.moves import urllib -from tornado.concurrent import return_future -from thumbor.loaders import LoaderResult, file_loader, https_loader -from tc_aws.loaders import s3_loader -from thumbor.context import Context -from .helpers import ( - separate_url_and_source_type, - THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE, -) - -from typing import Any, Callable - import base64 import logging +from typing import Any, Callable + +from six.moves import urllib +from tornado.concurrent import return_future + +from tc_aws.loaders import s3_loader +from thumbor.context import Context +from thumbor.loaders import LoaderResult, file_loader, https_loader + +from .helpers import ( + THUMBOR_EXTERNAL_TYPE, + THUMBOR_LOCAL_FILE_TYPE, + THUMBOR_S3_TYPE, + separate_url_and_source_type, +) + def get_not_found_result(): # type: () -> LoaderResult