From 4aa2d76beab11fd2ddbf1f7fb33c1b149bc5f0e6 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 14 Dec 2023 18:57:04 -0800 Subject: [PATCH] models: Extract zerver.models.streams. Signed-off-by: Anders Kaseorg --- tools/semgrep-py.yml | 2 +- web/src/settings_config.ts | 2 +- zerver/actions/default_streams.py | 9 +- zerver/actions/message_edit.py | 2 +- zerver/actions/message_send.py | 3 +- zerver/data_import/rocketchat.py | 2 +- zerver/lib/cache.py | 2 +- zerver/lib/digest.py | 2 +- zerver/lib/email_mirror.py | 2 +- zerver/lib/events.py | 2 +- zerver/lib/mention.py | 3 +- zerver/lib/narrow.py | 11 +- zerver/lib/streams.py | 4 +- zerver/lib/subscription_info.py | 3 +- zerver/lib/test_classes.py | 3 +- zerver/lib/test_helpers.py | 2 +- zerver/lib/user_topics.py | 3 +- zerver/management/commands/merge_streams.py | 2 +- .../commands/remove_users_from_stream.py | 2 +- .../commands/send_to_email_mirror.py | 3 +- zerver/migrations/0001_initial.py | 2 +- .../0212_make_stream_email_token_unique.py | 2 +- zerver/models/__init__.py | 379 +---------------- zerver/models/streams.py | 387 ++++++++++++++++++ zerver/tests/test_audit_log.py | 2 +- zerver/tests/test_bots.py | 2 +- zerver/tests/test_digest.py | 11 +- zerver/tests/test_email_mirror.py | 3 +- zerver/tests/test_event_queue.py | 3 +- zerver/tests/test_event_system.py | 10 +- zerver/tests/test_events.py | 2 +- zerver/tests/test_example.py | 3 +- zerver/tests/test_home.py | 3 +- zerver/tests/test_i18n.py | 2 +- zerver/tests/test_import_export.py | 3 +- zerver/tests/test_invite.py | 2 +- zerver/tests/test_management_commands.py | 3 +- zerver/tests/test_markdown.py | 2 +- zerver/tests/test_message_dict.py | 12 +- zerver/tests/test_message_edit.py | 12 +- .../tests/test_message_edit_notifications.py | 3 +- zerver/tests/test_message_fetch.py | 2 +- zerver/tests/test_message_flags.py | 2 +- .../tests/test_message_notification_emails.py | 3 +- zerver/tests/test_message_send.py | 2 +- zerver/tests/test_message_topics.py | 3 +- .../tests/test_outgoing_webhook_interfaces.py | 3 +- zerver/tests/test_outgoing_webhook_system.py | 3 +- zerver/tests/test_push_notifications.py | 2 +- zerver/tests/test_queue_worker.py | 2 +- zerver/tests/test_realm.py | 2 +- zerver/tests/test_retention.py | 2 +- zerver/tests/test_signup.py | 2 +- zerver/tests/test_soft_deactivation.py | 2 +- zerver/tests/test_subs.py | 3 +- zerver/tests/test_user_topics.py | 3 +- zerver/tests/test_users.py | 2 +- zerver/views/development/email_log.py | 3 +- zerver/views/registration.py | 2 +- zilencer/management/commands/populate_db.py | 2 +- 60 files changed, 474 insertions(+), 483 deletions(-) create mode 100644 zerver/models/streams.py diff --git a/tools/semgrep-py.yml b/tools/semgrep-py.yml index 3fd1030545..8f48e394fc 100644 --- a/tools/semgrep-py.yml +++ b/tools/semgrep-py.yml @@ -49,7 +49,7 @@ rules: - pattern-not: from zerver.lib.utils import generate_api_key - pattern-not: from zerver.models.linkifiers import filter_pattern_validator - pattern-not: from zerver.models.linkifiers import url_template_validator - - pattern-not: from zerver.models import generate_email_token_for_stream + - pattern-not: from zerver.models.streams import generate_email_token_for_stream - pattern-not: from zerver.models.realms import generate_realm_uuid_owner_secret - pattern-either: - pattern: from zerver import $X diff --git a/web/src/settings_config.ts b/web/src/settings_config.ts index 8868941bcf..c739771e6d 100644 --- a/web/src/settings_config.ts +++ b/web/src/settings_config.ts @@ -1006,7 +1006,7 @@ export const stream_privacy_policy_values = { export const stream_post_policy_values = { // These strings should match the strings in the - // Stream.POST_POLICIES object in zerver/models/__init__.py. + // Stream.POST_POLICIES object in zerver/models/streams.py. everyone: { code: StreamPostPolicy.EVERYONE, description: $t({defaultMessage: "Everyone"}), diff --git a/zerver/actions/default_streams.py b/zerver/actions/default_streams.py index f298b2fc99..cc04538a8f 100644 --- a/zerver/actions/default_streams.py +++ b/zerver/actions/default_streams.py @@ -8,13 +8,8 @@ from zerver.lib.default_streams import ( get_default_streams_for_realm_as_dicts, ) from zerver.lib.exceptions import JsonableError -from zerver.models import ( - DefaultStream, - DefaultStreamGroup, - Realm, - Stream, - get_default_stream_groups, -) +from zerver.models import DefaultStream, DefaultStreamGroup, Realm, Stream +from zerver.models.streams import get_default_stream_groups from zerver.models.users import active_non_guest_user_ids from zerver.tornado.django_api import send_event_on_commit diff --git a/zerver/actions/message_edit.py b/zerver/actions/message_edit.py index c9014ed0ef..ecb45b6a6b 100644 --- a/zerver/actions/message_edit.py +++ b/zerver/actions/message_edit.py @@ -76,8 +76,8 @@ from zerver.models import ( UserMessage, UserProfile, UserTopic, - get_stream_by_id_in_realm, ) +from zerver.models.streams import get_stream_by_id_in_realm from zerver.models.users import get_system_bot from zerver.tornado.django_api import send_event diff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.py index 29ccf9458d..7afc85d857 100644 --- a/zerver/actions/message_send.py +++ b/zerver/actions/message_send.py @@ -103,12 +103,11 @@ from zerver.models import ( UserProfile, UserTopic, get_client, - get_stream, - get_stream_by_id_in_realm, query_for_ids, ) from zerver.models.groups import SystemGroups from zerver.models.recipients import get_huddle_user_ids +from zerver.models.streams import get_stream, get_stream_by_id_in_realm from zerver.models.users import get_system_bot, get_user_by_delivery_email, is_cross_realm_bot_email from zerver.tornado.django_api import send_event diff --git a/zerver/data_import/rocketchat.py b/zerver/data_import/rocketchat.py index 8013ecdde2..4d6e7e181d 100644 --- a/zerver/data_import/rocketchat.py +++ b/zerver/data_import/rocketchat.py @@ -182,7 +182,7 @@ def convert_channel_data( # should be allowed to post in the converted Zulip stream. # For more details: https://zulip.com/help/stream-sending-policy # - # See `Stream` model in `zerver/models/__init__.py` to know about what each + # See `Stream` model in `zerver/models/streams.py` to know about what each # number represent. stream_post_policy = 4 if channel_dict.get("ro", False) else 1 diff --git a/zerver/lib/cache.py b/zerver/lib/cache.py index 4f74a1a874..4fbc05c69b 100644 --- a/zerver/lib/cache.py +++ b/zerver/lib/cache.py @@ -639,7 +639,7 @@ def realm_text_description_cache_key(realm: "Realm") -> str: return f"realm_text_description:{realm.string_id}" -# Called by models/__init__.py to flush the stream cache whenever we save a stream +# Called by models/streams.py to flush the stream cache whenever we save a stream # object. def flush_stream( *, diff --git a/zerver/lib/digest.py b/zerver/lib/digest.py index 12413ac943..a79b7f0ed1 100644 --- a/zerver/lib/digest.py +++ b/zerver/lib/digest.py @@ -29,8 +29,8 @@ from zerver.models import ( Subscription, UserActivityInterval, UserProfile, - get_active_streams, ) +from zerver.models.streams import get_active_streams logger = logging.getLogger(__name__) log_to_file(logger, settings.DIGEST_LOG_PATH) diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py index f26e5b4c8b..13631e6bf4 100644 --- a/zerver/lib/email_mirror.py +++ b/zerver/lib/email_mirror.py @@ -37,8 +37,8 @@ from zerver.models import ( Stream, UserProfile, get_client, - get_stream_by_id_in_realm, ) +from zerver.models.streams import get_stream_by_id_in_realm from zerver.models.users import get_system_bot, get_user_profile_by_id from zproject.backends import is_user_active diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 2ba72c941c..5b683d62fe 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -80,13 +80,13 @@ from zerver.models import ( UserStatus, UserTopic, custom_profile_fields_for_realm, - get_default_stream_groups, ) from zerver.models.constants import MAX_TOPIC_NAME_LENGTH from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realm_emoji import get_all_custom_emoji_for_realm from zerver.models.realm_playgrounds import get_realm_playgrounds from zerver.models.realms import get_realm_domains +from zerver.models.streams import get_default_stream_groups from zerver.tornado.django_api import get_user_events, request_event_queue from zproject.backends import email_auth_enabled, password_auth_enabled diff --git a/zerver/lib/mention.py b/zerver/lib/mention.py index a67f696f99..18d0ded238 100644 --- a/zerver/lib/mention.py +++ b/zerver/lib/mention.py @@ -7,7 +7,8 @@ from django.conf import settings from django.db.models import Q from zerver.lib.users import get_inaccessible_user_ids -from zerver.models import UserGroup, UserProfile, get_linkable_streams +from zerver.models import UserGroup, UserProfile +from zerver.models.streams import get_linkable_streams BEFORE_MENTION_ALLOWED_REGEX = r"(? SubscriptionInfo: diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index dc767291ef..821139d612 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -107,11 +107,10 @@ from zerver.models import ( UserMessage, UserProfile, UserStatus, - get_realm_stream, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.realms import clear_supported_auth_backends_cache, get_realm +from zerver.models.streams import get_realm_stream, get_stream from zerver.models.users import get_system_bot, get_user, get_user_by_delivery_email from zerver.openapi.openapi import validate_against_openapi_schema, validate_request from zerver.tornado.event_queue import clear_client_event_queues_for_testing diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index ef5021e712..42c947b09f 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -60,9 +60,9 @@ from zerver.models import ( UserMessage, UserProfile, get_client, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.tornado.handlers import AsyncDjangoHandler, allocate_handler_id from zilencer.models import RemoteZulipServer from zproject.backends import ExternalAuthDataDict, ExternalAuthResult diff --git a/zerver/lib/user_topics.py b/zerver/lib/user_topics.py index a5b64aa2b0..8f3e6e499c 100644 --- a/zerver/lib/user_topics.py +++ b/zerver/lib/user_topics.py @@ -12,7 +12,8 @@ from sqlalchemy.types import Integer from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.topic import topic_match_sa from zerver.lib.types import UserTopicDict -from zerver.models import UserProfile, UserTopic, get_stream +from zerver.models import UserProfile, UserTopic +from zerver.models.streams import get_stream def get_user_topics( diff --git a/zerver/management/commands/merge_streams.py b/zerver/management/commands/merge_streams.py index 1f72b551cb..b85a7c9e98 100644 --- a/zerver/management/commands/merge_streams.py +++ b/zerver/management/commands/merge_streams.py @@ -5,7 +5,7 @@ from typing_extensions import override from zerver.actions.streams import merge_streams from zerver.lib.management import ZulipBaseCommand -from zerver.models import get_stream +from zerver.models.streams import get_stream class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/remove_users_from_stream.py b/zerver/management/commands/remove_users_from_stream.py index 0849f0b331..c47db24d20 100644 --- a/zerver/management/commands/remove_users_from_stream.py +++ b/zerver/management/commands/remove_users_from_stream.py @@ -5,7 +5,7 @@ from typing_extensions import override from zerver.actions.streams import bulk_remove_subscriptions from zerver.lib.management import ZulipBaseCommand -from zerver.models import get_stream +from zerver.models.streams import get_stream class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/send_to_email_mirror.py b/zerver/management/commands/send_to_email_mirror.py index 02c65368de..1bdee441a7 100644 --- a/zerver/management/commands/send_to_email_mirror.py +++ b/zerver/management/commands/send_to_email_mirror.py @@ -13,8 +13,9 @@ from typing_extensions import override from zerver.lib.email_mirror import mirror_email_message from zerver.lib.email_mirror_helpers import encode_email_address from zerver.lib.management import ZulipBaseCommand -from zerver.models import Realm, get_stream +from zerver.models import Realm from zerver.models.realms import get_realm +from zerver.models.streams import get_stream # This command loads an email from a specified file and sends it # to the email mirror. Simple emails can be passed in a JSON file, diff --git a/zerver/migrations/0001_initial.py b/zerver/migrations/0001_initial.py index ee29c5468c..c5145b64b3 100644 --- a/zerver/migrations/0001_initial.py +++ b/zerver/migrations/0001_initial.py @@ -12,7 +12,7 @@ from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models.functions import Upper -from zerver.models import generate_email_token_for_stream +from zerver.models.streams import generate_email_token_for_stream def migrate_existing_attachment_data( diff --git a/zerver/migrations/0212_make_stream_email_token_unique.py b/zerver/migrations/0212_make_stream_email_token_unique.py index aa63c49326..2e77c08631 100644 --- a/zerver/migrations/0212_make_stream_email_token_unique.py +++ b/zerver/migrations/0212_make_stream_email_token_unique.py @@ -2,7 +2,7 @@ from django.db import migrations, models -from zerver.models import generate_email_token_for_stream +from zerver.models.streams import generate_email_token_for_stream class Migration(migrations.Migration): diff --git a/zerver/models/__init__.py b/zerver/models/__init__.py index 1eac732724..5aceaf36f3 100644 --- a/zerver/models/__init__.py +++ b/zerver/models/__init__.py @@ -2,10 +2,9 @@ # mypy: disable-error-code="explicit-override" import hashlib -import secrets import time from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypedDict, TypeVar, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union import orjson from bitfield import BitField @@ -34,7 +33,6 @@ from zerver.lib.cache import ( cache_with_key, flush_message, flush_muting_users_cache, - flush_stream, flush_submessage, flush_used_upload_space_cache, realm_alert_words_automaton_cache_key, @@ -44,11 +42,9 @@ from zerver.lib.display_recipient import get_recipient_ids from zerver.lib.exceptions import RateLimitedError from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.types import ( - DefaultStreamDict, ExtendedFieldElement, ExtendedValidator, FieldElement, - GroupPermissionSetting, ProfileDataElementBase, ProfileDataElementValue, RealmUserValidator, @@ -66,7 +62,6 @@ from zerver.lib.validator import ( ) from zerver.models.constants import MAX_TOPIC_NAME_LENGTH from zerver.models.groups import GroupGroupMembership as GroupGroupMembership -from zerver.models.groups import SystemGroups from zerver.models.groups import UserGroup as UserGroup from zerver.models.groups import UserGroupMembership as UserGroupMembership from zerver.models.linkifiers import RealmFilter as RealmFilter @@ -84,6 +79,10 @@ from zerver.models.realms import RealmAuthenticationMethod as RealmAuthenticatio from zerver.models.realms import RealmDomain as RealmDomain from zerver.models.recipients import Huddle as Huddle from zerver.models.recipients import Recipient as Recipient +from zerver.models.streams import DefaultStream as DefaultStream +from zerver.models.streams import DefaultStreamGroup as DefaultStreamGroup +from zerver.models.streams import Stream as Stream +from zerver.models.streams import Subscription as Subscription from zerver.models.users import RealmUserDefault as RealmUserDefault from zerver.models.users import UserBaseSettings as UserBaseSettings from zerver.models.users import UserProfile as UserProfile @@ -143,202 +142,6 @@ def query_for_ids( return query -def generate_email_token_for_stream() -> str: - return secrets.token_hex(16) - - -class Stream(models.Model): - MAX_NAME_LENGTH = 60 - MAX_DESCRIPTION_LENGTH = 1024 - - name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True) - realm = models.ForeignKey(Realm, db_index=True, on_delete=CASCADE) - date_created = models.DateTimeField(default=timezone_now) - deactivated = models.BooleanField(default=False) - description = models.CharField(max_length=MAX_DESCRIPTION_LENGTH, default="") - rendered_description = models.TextField(default="") - - # Foreign key to the Recipient object for STREAM type messages to this stream. - recipient = models.ForeignKey(Recipient, null=True, on_delete=models.SET_NULL) - - # Various permission policy configurations - PERMISSION_POLICIES: Dict[str, Dict[str, Any]] = { - "web_public": { - "invite_only": False, - "history_public_to_subscribers": True, - "is_web_public": True, - "policy_name": gettext_lazy("Web-public"), - }, - "public": { - "invite_only": False, - "history_public_to_subscribers": True, - "is_web_public": False, - "policy_name": gettext_lazy("Public"), - }, - "private_shared_history": { - "invite_only": True, - "history_public_to_subscribers": True, - "is_web_public": False, - "policy_name": gettext_lazy("Private, shared history"), - }, - "private_protected_history": { - "invite_only": True, - "history_public_to_subscribers": False, - "is_web_public": False, - "policy_name": gettext_lazy("Private, protected history"), - }, - # Public streams with protected history are currently only - # available in Zephyr realms - "public_protected_history": { - "invite_only": False, - "history_public_to_subscribers": False, - "is_web_public": False, - "policy_name": gettext_lazy("Public, protected history"), - }, - } - invite_only = models.BooleanField(default=False) - history_public_to_subscribers = models.BooleanField(default=True) - - # Whether this stream's content should be published by the web-public archive features - is_web_public = models.BooleanField(default=False) - - STREAM_POST_POLICY_EVERYONE = 1 - STREAM_POST_POLICY_ADMINS = 2 - STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS = 3 - STREAM_POST_POLICY_MODERATORS = 4 - # TODO: Implement policy to restrict posting to a user group or admins. - - # Who in the organization has permission to send messages to this stream. - stream_post_policy = models.PositiveSmallIntegerField(default=STREAM_POST_POLICY_EVERYONE) - POST_POLICIES: Dict[int, StrPromise] = { - # These strings should match the strings in the - # stream_post_policy_values object in stream_data.js. - STREAM_POST_POLICY_EVERYONE: gettext_lazy("All stream members can post"), - STREAM_POST_POLICY_ADMINS: gettext_lazy("Only organization administrators can post"), - STREAM_POST_POLICY_MODERATORS: gettext_lazy( - "Only organization administrators and moderators can post" - ), - STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS: gettext_lazy( - "Only organization full members can post" - ), - } - STREAM_POST_POLICY_TYPES = list(POST_POLICIES.keys()) - - # The unique thing about Zephyr public streams is that we never list their - # users. We may try to generalize this concept later, but for now - # we just use a concrete field. (Zephyr public streams aren't exactly like - # invite-only streams--while both are private in terms of listing users, - # for Zephyr we don't even list users to stream members, yet membership - # is more public in the sense that you don't need a Zulip invite to join. - # This field is populated directly from UserProfile.is_zephyr_mirror_realm, - # and the reason for denormalizing field is performance. - is_in_zephyr_realm = models.BooleanField(default=False) - - # Used by the e-mail forwarder. The e-mail RFC specifies a maximum - # e-mail length of 254, and our max stream length is 30, so we - # have plenty of room for the token. - email_token = models.CharField( - max_length=32, - default=generate_email_token_for_stream, - unique=True, - ) - - # For old messages being automatically deleted. - # Value NULL means "use retention policy of the realm". - # Value -1 means "disable retention policy for this stream unconditionally". - # Non-negative values have the natural meaning of "archive messages older than days". - MESSAGE_RETENTION_SPECIAL_VALUES_MAP = { - "unlimited": -1, - "realm_default": None, - } - message_retention_days = models.IntegerField(null=True, default=None) - - # on_delete field here is set to RESTRICT because we don't want to allow - # deleting a user group in case it is referenced by this setting. - # We are not using PROTECT since we want to allow deletion of user groups - # when realm itself is deleted. - can_remove_subscribers_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT) - - # The very first message ID in the stream. Used to help clients - # determine whether they might need to display "more topics" for a - # stream based on what messages they have cached. - first_message_id = models.IntegerField(null=True, db_index=True) - - stream_permission_group_settings = { - "can_remove_subscribers_group": GroupPermissionSetting( - require_system_group=True, - allow_internet_group=False, - allow_owners_group=False, - allow_nobody_group=False, - allow_everyone_group=True, - default_group_name=SystemGroups.ADMINISTRATORS, - id_field_name="can_remove_subscribers_group_id", - ), - } - - class Meta: - indexes = [ - models.Index(Upper("name"), name="upper_stream_name_idx"), - ] - - @override - def __str__(self) -> str: - return self.name - - def is_public(self) -> bool: - # All streams are private in Zephyr mirroring realms. - return not self.invite_only and not self.is_in_zephyr_realm - - def is_history_realm_public(self) -> bool: - return self.is_public() - - def is_history_public_to_subscribers(self) -> bool: - return self.history_public_to_subscribers - - # Stream fields included whenever a Stream object is provided to - # Zulip clients via the API. A few details worth noting: - # * "id" is represented as "stream_id" in most API interfaces. - # * "email_token" is not realm-public and thus is not included here. - # * is_in_zephyr_realm is a backend-only optimization. - # * "deactivated" streams are filtered from the API entirely. - # * "realm" and "recipient" are not exposed to clients via the API. - API_FIELDS = [ - "date_created", - "description", - "first_message_id", - "history_public_to_subscribers", - "id", - "invite_only", - "is_web_public", - "message_retention_days", - "name", - "rendered_description", - "stream_post_policy", - "can_remove_subscribers_group_id", - ] - - def to_dict(self) -> DefaultStreamDict: - return DefaultStreamDict( - can_remove_subscribers_group=self.can_remove_subscribers_group_id, - date_created=datetime_to_timestamp(self.date_created), - description=self.description, - first_message_id=self.first_message_id, - history_public_to_subscribers=self.history_public_to_subscribers, - invite_only=self.invite_only, - is_web_public=self.is_web_public, - message_retention_days=self.message_retention_days, - name=self.name, - rendered_description=self.rendered_description, - stream_id=self.id, - stream_post_policy=self.stream_post_policy, - is_announcement_only=self.stream_post_policy == Stream.STREAM_POST_POLICY_ADMINS, - ) - - -post_save.connect(flush_stream, sender=Stream) -post_delete.connect(flush_stream, sender=Stream) - - class UserTopic(models.Model): user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) stream = models.ForeignKey(Stream, on_delete=CASCADE) @@ -489,61 +292,6 @@ def get_client_remote_cache(name: str) -> Client: return client -def get_realm_stream(stream_name: str, realm_id: int) -> Stream: - return Stream.objects.get(name__iexact=stream_name.strip(), realm_id=realm_id) - - -def get_active_streams(realm: Realm) -> QuerySet[Stream]: - """ - Return all streams (including invite-only streams) that have not been deactivated. - """ - return Stream.objects.filter(realm=realm, deactivated=False) - - -def get_linkable_streams(realm_id: int) -> QuerySet[Stream]: - """ - This returns the streams that we are allowed to linkify using - something like "#frontend" in our markup. For now the business - rule is that you can link any stream in the realm that hasn't - been deactivated (similar to how get_active_streams works). - """ - return Stream.objects.filter(realm_id=realm_id, deactivated=False) - - -def get_stream(stream_name: str, realm: Realm) -> Stream: - """ - Callers that don't have a Realm object already available should use - get_realm_stream directly, to avoid unnecessarily fetching the - Realm object. - """ - return get_realm_stream(stream_name, realm.id) - - -def get_stream_by_id_in_realm(stream_id: int, realm: Realm) -> Stream: - return Stream.objects.select_related("realm", "recipient").get(id=stream_id, realm=realm) - - -def bulk_get_streams(realm: Realm, stream_names: Set[str]) -> Dict[str, Any]: - def fetch_streams_by_name(stream_names: Set[str]) -> QuerySet[Stream]: - # - # This should be just - # - # Stream.objects.select_related().filter(name__iexact__in=stream_names, - # realm_id=realm_id) - # - # But chaining __in and __iexact doesn't work with Django's - # ORM, so we have the following hack to construct the relevant where clause - where_clause = ( - "upper(zerver_stream.name::text) IN (SELECT upper(name) FROM unnest(%s) AS name)" - ) - return get_active_streams(realm).extra(where=[where_clause], params=(list(stream_names),)) - - if not stream_names: - return {} - streams = list(fetch_streams_by_name(stream_names)) - return {stream.name.lower(): stream for stream in streams} - - class AbstractMessage(models.Model): sender = models.ForeignKey(UserProfile, on_delete=CASCADE) @@ -1432,91 +1180,6 @@ def get_old_unclaimed_attachments( return old_attachments, old_archived_attachments -class Subscription(models.Model): - """Keeps track of which users are part of the - audience for a given Recipient object. - - For 1:1 and group direct message Recipient objects, only the - user_profile and recipient fields have any meaning, defining the - immutable set of users who are in the audience for that Recipient. - - For Recipient objects associated with a Stream, the remaining - fields in this model describe the user's subscription to that stream. - """ - - user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) - recipient = models.ForeignKey(Recipient, on_delete=CASCADE) - - # Whether the user has since unsubscribed. We mark Subscription - # objects as inactive, rather than deleting them, when a user - # unsubscribes, so we can preserve user customizations like - # notification settings, stream color, etc., if the user later - # resubscribes. - active = models.BooleanField(default=True) - # This is a denormalization designed to improve the performance of - # bulk queries of Subscription objects, Whether the subscribed user - # is active tends to be a key condition in those queries. - # We intentionally don't specify a default value to promote thinking - # about this explicitly, as in some special cases, such as data import, - # we may be creating Subscription objects for a user that's deactivated. - is_user_active = models.BooleanField() - - # Whether this user had muted this stream. - is_muted = models.BooleanField(default=False) - - DEFAULT_STREAM_COLOR = "#c2c2c2" - color = models.CharField(max_length=10, default=DEFAULT_STREAM_COLOR) - pin_to_top = models.BooleanField(default=False) - - # These fields are stream-level overrides for the user's default - # configuration for notification, configured in UserProfile. The - # default, None, means we just inherit the user-level default. - desktop_notifications = models.BooleanField(null=True, default=None) - audible_notifications = models.BooleanField(null=True, default=None) - push_notifications = models.BooleanField(null=True, default=None) - email_notifications = models.BooleanField(null=True, default=None) - wildcard_mentions_notify = models.BooleanField(null=True, default=None) - - class Meta: - unique_together = ("user_profile", "recipient") - indexes = [ - models.Index( - fields=("recipient", "user_profile"), - name="zerver_subscription_recipient_id_user_profile_id_idx", - condition=Q(active=True, is_user_active=True), - ), - ] - - @override - def __str__(self) -> str: - return f"{self.user_profile!r} -> {self.recipient!r}" - - # Subscription fields included whenever a Subscription object is provided to - # Zulip clients via the API. A few details worth noting: - # * These fields will generally be merged with Stream.API_FIELDS - # data about the stream. - # * "user_profile" is usually implied as full API access to Subscription - # is primarily done for the current user; API access to other users' - # subscriptions is generally limited to boolean yes/no. - # * "id" and "recipient_id" are not included as they are not used - # in the Zulip API; it's an internal implementation detail. - # Subscription objects are always looked up in the API via - # (user_profile, stream) pairs. - # * "active" is often excluded in API use cases where it is implied. - # * "is_muted" often needs to be copied to not "in_home_view" for - # backwards-compatibility. - API_FIELDS = [ - "audible_notifications", - "color", - "desktop_notifications", - "email_notifications", - "is_muted", - "pin_to_top", - "push_notifications", - "wildcard_mentions_notify", - ] - - class UserActivity(models.Model): """Data table recording the last time each user hit Zulip endpoints via which Clients; unlike UserPresence, these data are not exposed @@ -1627,38 +1290,6 @@ class UserStatus(AbstractEmoji): status_text = models.CharField(max_length=255, default="") -class DefaultStream(models.Model): - realm = models.ForeignKey(Realm, on_delete=CASCADE) - stream = models.ForeignKey(Stream, on_delete=CASCADE) - - class Meta: - unique_together = ("realm", "stream") - - -class DefaultStreamGroup(models.Model): - MAX_NAME_LENGTH = 60 - - name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True) - realm = models.ForeignKey(Realm, on_delete=CASCADE) - streams = models.ManyToManyField("zerver.Stream") - description = models.CharField(max_length=1024, default="") - - class Meta: - unique_together = ("realm", "name") - - def to_dict(self) -> Dict[str, Any]: - return dict( - name=self.name, - id=self.id, - description=self.description, - streams=[stream.to_dict() for stream in self.streams.all().order_by("name")], - ) - - -def get_default_stream_groups(realm: Realm) -> QuerySet[DefaultStreamGroup]: - return DefaultStreamGroup.objects.filter(realm=realm) - - class AbstractScheduledJob(models.Model): scheduled_timestamp = models.DateTimeField(db_index=True) # JSON representation of arguments to consumer diff --git a/zerver/models/streams.py b/zerver/models/streams.py new file mode 100644 index 0000000000..edf5bc8115 --- /dev/null +++ b/zerver/models/streams.py @@ -0,0 +1,387 @@ +import secrets +from typing import Any, Dict, Set + +from django.db import models +from django.db.models import CASCADE, Q, QuerySet +from django.db.models.functions import Upper +from django.db.models.signals import post_delete, post_save +from django.utils.timezone import now as timezone_now +from django.utils.translation import gettext_lazy +from django_stubs_ext import StrPromise +from typing_extensions import override + +from zerver.lib.cache import flush_stream +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.types import DefaultStreamDict, GroupPermissionSetting +from zerver.models.groups import SystemGroups, UserGroup +from zerver.models.realms import Realm +from zerver.models.recipients import Recipient +from zerver.models.users import UserProfile + + +def generate_email_token_for_stream() -> str: + return secrets.token_hex(16) + + +class Stream(models.Model): + MAX_NAME_LENGTH = 60 + MAX_DESCRIPTION_LENGTH = 1024 + + name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True) + realm = models.ForeignKey(Realm, db_index=True, on_delete=CASCADE) + date_created = models.DateTimeField(default=timezone_now) + deactivated = models.BooleanField(default=False) + description = models.CharField(max_length=MAX_DESCRIPTION_LENGTH, default="") + rendered_description = models.TextField(default="") + + # Foreign key to the Recipient object for STREAM type messages to this stream. + recipient = models.ForeignKey(Recipient, null=True, on_delete=models.SET_NULL) + + # Various permission policy configurations + PERMISSION_POLICIES: Dict[str, Dict[str, Any]] = { + "web_public": { + "invite_only": False, + "history_public_to_subscribers": True, + "is_web_public": True, + "policy_name": gettext_lazy("Web-public"), + }, + "public": { + "invite_only": False, + "history_public_to_subscribers": True, + "is_web_public": False, + "policy_name": gettext_lazy("Public"), + }, + "private_shared_history": { + "invite_only": True, + "history_public_to_subscribers": True, + "is_web_public": False, + "policy_name": gettext_lazy("Private, shared history"), + }, + "private_protected_history": { + "invite_only": True, + "history_public_to_subscribers": False, + "is_web_public": False, + "policy_name": gettext_lazy("Private, protected history"), + }, + # Public streams with protected history are currently only + # available in Zephyr realms + "public_protected_history": { + "invite_only": False, + "history_public_to_subscribers": False, + "is_web_public": False, + "policy_name": gettext_lazy("Public, protected history"), + }, + } + invite_only = models.BooleanField(default=False) + history_public_to_subscribers = models.BooleanField(default=True) + + # Whether this stream's content should be published by the web-public archive features + is_web_public = models.BooleanField(default=False) + + STREAM_POST_POLICY_EVERYONE = 1 + STREAM_POST_POLICY_ADMINS = 2 + STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS = 3 + STREAM_POST_POLICY_MODERATORS = 4 + # TODO: Implement policy to restrict posting to a user group or admins. + + # Who in the organization has permission to send messages to this stream. + stream_post_policy = models.PositiveSmallIntegerField(default=STREAM_POST_POLICY_EVERYONE) + POST_POLICIES: Dict[int, StrPromise] = { + # These strings should match the strings in the + # stream_post_policy_values object in stream_data.js. + STREAM_POST_POLICY_EVERYONE: gettext_lazy("All stream members can post"), + STREAM_POST_POLICY_ADMINS: gettext_lazy("Only organization administrators can post"), + STREAM_POST_POLICY_MODERATORS: gettext_lazy( + "Only organization administrators and moderators can post" + ), + STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS: gettext_lazy( + "Only organization full members can post" + ), + } + STREAM_POST_POLICY_TYPES = list(POST_POLICIES.keys()) + + # The unique thing about Zephyr public streams is that we never list their + # users. We may try to generalize this concept later, but for now + # we just use a concrete field. (Zephyr public streams aren't exactly like + # invite-only streams--while both are private in terms of listing users, + # for Zephyr we don't even list users to stream members, yet membership + # is more public in the sense that you don't need a Zulip invite to join. + # This field is populated directly from UserProfile.is_zephyr_mirror_realm, + # and the reason for denormalizing field is performance. + is_in_zephyr_realm = models.BooleanField(default=False) + + # Used by the e-mail forwarder. The e-mail RFC specifies a maximum + # e-mail length of 254, and our max stream length is 30, so we + # have plenty of room for the token. + email_token = models.CharField( + max_length=32, + default=generate_email_token_for_stream, + unique=True, + ) + + # For old messages being automatically deleted. + # Value NULL means "use retention policy of the realm". + # Value -1 means "disable retention policy for this stream unconditionally". + # Non-negative values have the natural meaning of "archive messages older than days". + MESSAGE_RETENTION_SPECIAL_VALUES_MAP = { + "unlimited": -1, + "realm_default": None, + } + message_retention_days = models.IntegerField(null=True, default=None) + + # on_delete field here is set to RESTRICT because we don't want to allow + # deleting a user group in case it is referenced by this setting. + # We are not using PROTECT since we want to allow deletion of user groups + # when realm itself is deleted. + can_remove_subscribers_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT) + + # The very first message ID in the stream. Used to help clients + # determine whether they might need to display "more topics" for a + # stream based on what messages they have cached. + first_message_id = models.IntegerField(null=True, db_index=True) + + stream_permission_group_settings = { + "can_remove_subscribers_group": GroupPermissionSetting( + require_system_group=True, + allow_internet_group=False, + allow_owners_group=False, + allow_nobody_group=False, + allow_everyone_group=True, + default_group_name=SystemGroups.ADMINISTRATORS, + id_field_name="can_remove_subscribers_group_id", + ), + } + + class Meta: + indexes = [ + models.Index(Upper("name"), name="upper_stream_name_idx"), + ] + + @override + def __str__(self) -> str: + return self.name + + def is_public(self) -> bool: + # All streams are private in Zephyr mirroring realms. + return not self.invite_only and not self.is_in_zephyr_realm + + def is_history_realm_public(self) -> bool: + return self.is_public() + + def is_history_public_to_subscribers(self) -> bool: + return self.history_public_to_subscribers + + # Stream fields included whenever a Stream object is provided to + # Zulip clients via the API. A few details worth noting: + # * "id" is represented as "stream_id" in most API interfaces. + # * "email_token" is not realm-public and thus is not included here. + # * is_in_zephyr_realm is a backend-only optimization. + # * "deactivated" streams are filtered from the API entirely. + # * "realm" and "recipient" are not exposed to clients via the API. + API_FIELDS = [ + "date_created", + "description", + "first_message_id", + "history_public_to_subscribers", + "id", + "invite_only", + "is_web_public", + "message_retention_days", + "name", + "rendered_description", + "stream_post_policy", + "can_remove_subscribers_group_id", + ] + + def to_dict(self) -> DefaultStreamDict: + return DefaultStreamDict( + can_remove_subscribers_group=self.can_remove_subscribers_group_id, + date_created=datetime_to_timestamp(self.date_created), + description=self.description, + first_message_id=self.first_message_id, + history_public_to_subscribers=self.history_public_to_subscribers, + invite_only=self.invite_only, + is_web_public=self.is_web_public, + message_retention_days=self.message_retention_days, + name=self.name, + rendered_description=self.rendered_description, + stream_id=self.id, + stream_post_policy=self.stream_post_policy, + is_announcement_only=self.stream_post_policy == Stream.STREAM_POST_POLICY_ADMINS, + ) + + +post_save.connect(flush_stream, sender=Stream) +post_delete.connect(flush_stream, sender=Stream) + + +def get_realm_stream(stream_name: str, realm_id: int) -> Stream: + return Stream.objects.get(name__iexact=stream_name.strip(), realm_id=realm_id) + + +def get_active_streams(realm: Realm) -> QuerySet[Stream]: + """ + Return all streams (including invite-only streams) that have not been deactivated. + """ + return Stream.objects.filter(realm=realm, deactivated=False) + + +def get_linkable_streams(realm_id: int) -> QuerySet[Stream]: + """ + This returns the streams that we are allowed to linkify using + something like "#frontend" in our markup. For now the business + rule is that you can link any stream in the realm that hasn't + been deactivated (similar to how get_active_streams works). + """ + return Stream.objects.filter(realm_id=realm_id, deactivated=False) + + +def get_stream(stream_name: str, realm: Realm) -> Stream: + """ + Callers that don't have a Realm object already available should use + get_realm_stream directly, to avoid unnecessarily fetching the + Realm object. + """ + return get_realm_stream(stream_name, realm.id) + + +def get_stream_by_id_in_realm(stream_id: int, realm: Realm) -> Stream: + return Stream.objects.select_related("realm", "recipient").get(id=stream_id, realm=realm) + + +def bulk_get_streams(realm: Realm, stream_names: Set[str]) -> Dict[str, Any]: + def fetch_streams_by_name(stream_names: Set[str]) -> QuerySet[Stream]: + # + # This should be just + # + # Stream.objects.select_related().filter(name__iexact__in=stream_names, + # realm_id=realm_id) + # + # But chaining __in and __iexact doesn't work with Django's + # ORM, so we have the following hack to construct the relevant where clause + where_clause = ( + "upper(zerver_stream.name::text) IN (SELECT upper(name) FROM unnest(%s) AS name)" + ) + return get_active_streams(realm).extra(where=[where_clause], params=(list(stream_names),)) + + if not stream_names: + return {} + streams = list(fetch_streams_by_name(stream_names)) + return {stream.name.lower(): stream for stream in streams} + + +class Subscription(models.Model): + """Keeps track of which users are part of the + audience for a given Recipient object. + + For 1:1 and group direct message Recipient objects, only the + user_profile and recipient fields have any meaning, defining the + immutable set of users who are in the audience for that Recipient. + + For Recipient objects associated with a Stream, the remaining + fields in this model describe the user's subscription to that stream. + """ + + user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) + recipient = models.ForeignKey(Recipient, on_delete=CASCADE) + + # Whether the user has since unsubscribed. We mark Subscription + # objects as inactive, rather than deleting them, when a user + # unsubscribes, so we can preserve user customizations like + # notification settings, stream color, etc., if the user later + # resubscribes. + active = models.BooleanField(default=True) + # This is a denormalization designed to improve the performance of + # bulk queries of Subscription objects, Whether the subscribed user + # is active tends to be a key condition in those queries. + # We intentionally don't specify a default value to promote thinking + # about this explicitly, as in some special cases, such as data import, + # we may be creating Subscription objects for a user that's deactivated. + is_user_active = models.BooleanField() + + # Whether this user had muted this stream. + is_muted = models.BooleanField(default=False) + + DEFAULT_STREAM_COLOR = "#c2c2c2" + color = models.CharField(max_length=10, default=DEFAULT_STREAM_COLOR) + pin_to_top = models.BooleanField(default=False) + + # These fields are stream-level overrides for the user's default + # configuration for notification, configured in UserProfile. The + # default, None, means we just inherit the user-level default. + desktop_notifications = models.BooleanField(null=True, default=None) + audible_notifications = models.BooleanField(null=True, default=None) + push_notifications = models.BooleanField(null=True, default=None) + email_notifications = models.BooleanField(null=True, default=None) + wildcard_mentions_notify = models.BooleanField(null=True, default=None) + + class Meta: + unique_together = ("user_profile", "recipient") + indexes = [ + models.Index( + fields=("recipient", "user_profile"), + name="zerver_subscription_recipient_id_user_profile_id_idx", + condition=Q(active=True, is_user_active=True), + ), + ] + + @override + def __str__(self) -> str: + return f"{self.user_profile!r} -> {self.recipient!r}" + + # Subscription fields included whenever a Subscription object is provided to + # Zulip clients via the API. A few details worth noting: + # * These fields will generally be merged with Stream.API_FIELDS + # data about the stream. + # * "user_profile" is usually implied as full API access to Subscription + # is primarily done for the current user; API access to other users' + # subscriptions is generally limited to boolean yes/no. + # * "id" and "recipient_id" are not included as they are not used + # in the Zulip API; it's an internal implementation detail. + # Subscription objects are always looked up in the API via + # (user_profile, stream) pairs. + # * "active" is often excluded in API use cases where it is implied. + # * "is_muted" often needs to be copied to not "in_home_view" for + # backwards-compatibility. + API_FIELDS = [ + "audible_notifications", + "color", + "desktop_notifications", + "email_notifications", + "is_muted", + "pin_to_top", + "push_notifications", + "wildcard_mentions_notify", + ] + + +class DefaultStream(models.Model): + realm = models.ForeignKey(Realm, on_delete=CASCADE) + stream = models.ForeignKey(Stream, on_delete=CASCADE) + + class Meta: + unique_together = ("realm", "stream") + + +class DefaultStreamGroup(models.Model): + MAX_NAME_LENGTH = 60 + + name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True) + realm = models.ForeignKey(Realm, on_delete=CASCADE) + streams = models.ManyToManyField("zerver.Stream") + description = models.CharField(max_length=1024, default="") + + class Meta: + unique_together = ("realm", "name") + + def to_dict(self) -> Dict[str, Any]: + return dict( + name=self.name, + id=self.id, + description=self.description, + streams=[stream.to_dict() for stream in self.streams.all().order_by("name")], + ) + + +def get_default_stream_groups(realm: Realm) -> QuerySet[DefaultStreamGroup]: + return DefaultStreamGroup.objects.filter(realm=realm) diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index 6abf973362..1498636d84 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -81,13 +81,13 @@ from zerver.models import ( Subscription, UserGroup, UserProfile, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realm_emoji import EmojiInfo, get_all_custom_emoji_for_realm from zerver.models.realm_playgrounds import get_realm_playgrounds from zerver.models.realms import RealmDomainDict, get_realm, get_realm_domains +from zerver.models.streams import get_stream class TestRealmAuditLog(ZulipTestCase): diff --git a/zerver/tests/test_bots.py b/zerver/tests/test_bots.py index 4aecbf1f07..44a31f168a 100644 --- a/zerver/tests/test_bots.py +++ b/zerver/tests/test_bots.py @@ -24,9 +24,9 @@ from zerver.models import ( Subscription, UserProfile, get_bot_services, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_user, is_cross_realm_bot_email diff --git a/zerver/tests/test_digest.py b/zerver/tests/test_digest.py index a8da00bc6c..d2f4958ca3 100644 --- a/zerver/tests/test_digest.py +++ b/zerver/tests/test_digest.py @@ -26,16 +26,9 @@ from zerver.lib.digest import ( from zerver.lib.message import get_last_message_id from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - Message, - Realm, - RealmAuditLog, - Stream, - UserActivityInterval, - UserProfile, - get_stream, -) +from zerver.models import Message, Realm, RealmAuditLog, Stream, UserActivityInterval, UserProfile from zerver.models.realms import get_realm +from zerver.models.streams import get_stream class TestDigestEmailMessages(ZulipTestCase): diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py index 425c5478a6..da93be6db6 100644 --- a/zerver/tests/test_email_mirror.py +++ b/zerver/tests/test_email_mirror.py @@ -37,8 +37,9 @@ from zerver.lib.send_email import FromAddress from zerver.lib.streams import ensure_stream from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import mock_queue_publish, most_recent_message, most_recent_usermessage -from zerver.models import Attachment, Recipient, Stream, UserProfile, get_stream +from zerver.models import Attachment, Recipient, Stream, UserProfile from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot from zerver.worker.queue_processors import MirrorWorker diff --git a/zerver/tests/test_event_queue.py b/zerver/tests/test_event_queue.py index d7697bd341..59da51461a 100644 --- a/zerver/tests/test_event_queue.py +++ b/zerver/tests/test_event_queue.py @@ -15,7 +15,8 @@ from zerver.actions.user_topics import do_set_user_topic_visibility_policy from zerver.lib.cache import cache_delete, get_muting_users_cache_key from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import HostRequestMock, dummy_handler, mock_queue_publish -from zerver.models import Recipient, Subscription, UserProfile, UserTopic, get_stream +from zerver.models import Recipient, Subscription, UserProfile, UserTopic +from zerver.models.streams import get_stream from zerver.tornado.event_queue import ( ClientDescriptor, access_client_descriptor, diff --git a/zerver/tests/test_event_system.py b/zerver/tests/test_event_system.py index 0513a5105a..d1c0a3a5a8 100644 --- a/zerver/tests/test_event_system.py +++ b/zerver/tests/test_event_system.py @@ -28,15 +28,9 @@ from zerver.lib.test_helpers import ( stub_event_queue_user_events, ) from zerver.lib.users import get_api_key, get_users_for_api -from zerver.models import ( - CustomProfileField, - UserMessage, - UserPresence, - UserProfile, - get_client, - get_stream, -) +from zerver.models import CustomProfileField, UserMessage, UserPresence, UserProfile, get_client from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot from zerver.tornado.event_queue import ( allocate_client_descriptor, diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 237652f259..e561c6d24b 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -241,9 +241,9 @@ from zerver.models import ( UserStatus, UserTopic, get_client, - get_stream, ) from zerver.models.groups import SystemGroups +from zerver.models.streams import get_stream from zerver.models.users import get_user_by_delivery_email from zerver.openapi.openapi import validate_against_openapi_schema from zerver.tornado.django_api import send_event diff --git a/zerver/tests/test_example.py b/zerver/tests/test_example.py index 8ae9d79e86..27563bcf3f 100644 --- a/zerver/tests/test_example.py +++ b/zerver/tests/test_example.py @@ -11,8 +11,9 @@ from zerver.lib.streams import access_stream_for_send_message from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import most_recent_message from zerver.lib.users import is_administrator_role -from zerver.models import UserProfile, UserStatus, get_stream +from zerver.models import UserProfile, UserStatus from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_user_by_delivery_email diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index 29ff92b6b1..a682fc5909 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -24,8 +24,9 @@ from zerver.lib.soft_deactivation import do_soft_deactivate_users from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_user_messages, queries_captured from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import DefaultStream, Draft, Realm, UserActivity, UserProfile, get_stream +from zerver.models import DefaultStream, Draft, Realm, UserActivity, UserProfile from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot, get_user from zerver.worker.queue_processors import UserActivityWorker diff --git a/zerver/tests/test_i18n.py b/zerver/tests/test_i18n.py index 54dddfed62..28516ce1fd 100644 --- a/zerver/tests/test_i18n.py +++ b/zerver/tests/test_i18n.py @@ -13,7 +13,7 @@ from zerver.lib.i18n import get_browser_language_code from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import HostRequestMock from zerver.management.commands import makemessages -from zerver.models import get_realm_stream +from zerver.models.streams import get_realm_stream if TYPE_CHECKING: from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 7d3596eb4b..1a6f5a7cb0 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -83,13 +83,12 @@ from zerver.models import ( UserProfile, UserStatus, UserTopic, - get_active_streams, get_client, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.realms import get_realm from zerver.models.recipients import get_huddle_hash +from zerver.models.streams import get_active_streams, get_stream from zerver.models.users import get_system_bot, get_user_by_delivery_email diff --git a/zerver/tests/test_invite.py b/zerver/tests/test_invite.py index 2e17f75d57..9b0d25ca7d 100644 --- a/zerver/tests/test_invite.py +++ b/zerver/tests/test_invite.py @@ -63,10 +63,10 @@ from zerver.models import ( UserGroup, UserMessage, UserProfile, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_user_by_delivery_email from zerver.views.invite import INVITATION_LINK_VALIDITY_MINUTES, get_invitee_emails_set from zerver.views.registration import accounts_home diff --git a/zerver/tests/test_management_commands.py b/zerver/tests/test_management_commands.py index 975e8eb24c..4a524f5e1c 100644 --- a/zerver/tests/test_management_commands.py +++ b/zerver/tests/test_management_commands.py @@ -20,8 +20,9 @@ from zerver.actions.reactions import do_add_reaction from zerver.lib.management import ZulipBaseCommand, check_config from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import most_recent_message, stdout_suppressed -from zerver.models import Message, Reaction, Realm, Recipient, UserProfile, get_stream +from zerver.models import Message, Reaction, Realm, Recipient, UserProfile from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_user_profile_by_email diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index dd030f8946..1c6e1485b7 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -68,11 +68,11 @@ from zerver.models import ( UserMessage, UserProfile, get_client, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realms import get_realm +from zerver.models.streams import get_stream class SimulatedFencedBlockPreprocessor(FencedBlockPreprocessor): diff --git a/zerver/tests/test_message_dict.py b/zerver/tests/test_message_dict.py index 2bea535e76..83fa79c00c 100644 --- a/zerver/tests/test_message_dict.py +++ b/zerver/tests/test_message_dict.py @@ -12,17 +12,9 @@ from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import make_client from zerver.lib.topic import TOPIC_LINKS from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.models import ( - Message, - Reaction, - Realm, - RealmFilter, - Recipient, - Stream, - UserProfile, - get_stream, -) +from zerver.models import Message, Reaction, Realm, RealmFilter, Recipient, Stream, UserProfile from zerver.models.realms import get_realm +from zerver.models.streams import get_stream class MessageDictTest(ZulipTestCase): diff --git a/zerver/tests/test_message_edit.py b/zerver/tests/test_message_edit.py index 16fa688c30..4e1dddfe9f 100644 --- a/zerver/tests/test_message_edit.py +++ b/zerver/tests/test_message_edit.py @@ -30,19 +30,11 @@ from zerver.lib.user_topics import ( topic_has_visibility_policy, ) from zerver.lib.utils import assert_is_not_none -from zerver.models import ( - Message, - Realm, - Stream, - UserGroup, - UserMessage, - UserProfile, - UserTopic, - get_stream, -) +from zerver.models import Message, Realm, Stream, UserGroup, UserMessage, UserProfile, UserTopic from zerver.models.constants import MAX_TOPIC_NAME_LENGTH from zerver.models.groups import SystemGroups from zerver.models.realms import get_realm +from zerver.models.streams import get_stream if TYPE_CHECKING: from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse diff --git a/zerver/tests/test_message_edit_notifications.py b/zerver/tests/test_message_edit_notifications.py index 8f87ec692d..e857681e01 100644 --- a/zerver/tests/test_message_edit_notifications.py +++ b/zerver/tests/test_message_edit_notifications.py @@ -8,7 +8,8 @@ from zerver.actions.user_topics import do_set_user_topic_visibility_policy from zerver.lib.push_notifications import get_apns_badge_count, get_apns_badge_count_future from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import mock_queue_publish -from zerver.models import NotificationTriggers, Subscription, UserPresence, UserTopic, get_stream +from zerver.models import NotificationTriggers, Subscription, UserPresence, UserTopic +from zerver.models.streams import get_stream from zerver.tornado.event_queue import maybe_enqueue_notifications diff --git a/zerver/tests/test_message_fetch.py b/zerver/tests/test_message_fetch.py index 30e0fa4f4c..4c062ee113 100644 --- a/zerver/tests/test_message_fetch.py +++ b/zerver/tests/test_message_fetch.py @@ -58,9 +58,9 @@ from zerver.models import ( UserMessage, UserProfile, UserTopic, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.views.message_fetch import get_messages_backend if TYPE_CHECKING: diff --git a/zerver/tests/test_message_flags.py b/zerver/tests/test_message_flags.py index be1082be0a..bc906e2c07 100644 --- a/zerver/tests/test_message_flags.py +++ b/zerver/tests/test_message_flags.py @@ -33,9 +33,9 @@ from zerver.models import ( UserMessage, UserProfile, UserTopic, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream if TYPE_CHECKING: from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse diff --git a/zerver/tests/test_message_notification_emails.py b/zerver/tests/test_message_notification_emails.py index 430c5d819d..825334455b 100644 --- a/zerver/tests/test_message_notification_emails.py +++ b/zerver/tests/test_message_notification_emails.py @@ -27,9 +27,10 @@ from zerver.lib.email_notifications import ( ) from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import ZulipTestCase -from zerver.models import NotificationTriggers, UserMessage, UserProfile, UserTopic, get_stream +from zerver.models import NotificationTriggers, UserMessage, UserProfile, UserTopic from zerver.models.realm_emoji import get_name_keyed_dict_for_active_realm_emoji from zerver.models.realms import get_realm +from zerver.models.streams import get_stream class TestMessageNotificationEmails(ZulipTestCase): diff --git a/zerver/tests/test_message_send.py b/zerver/tests/test_message_send.py index fd1e604a18..bfd4ac5e82 100644 --- a/zerver/tests/test_message_send.py +++ b/zerver/tests/test_message_send.py @@ -56,12 +56,12 @@ from zerver.models import ( UserGroup, UserMessage, UserProfile, - get_stream, ) from zerver.models.constants import MAX_TOPIC_NAME_LENGTH from zerver.models.groups import SystemGroups from zerver.models.realms import get_realm from zerver.models.recipients import get_or_create_huddle +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot, get_user from zerver.views.message_send import InvalidMirrorInputError diff --git a/zerver/tests/test_message_topics.py b/zerver/tests/test_message_topics.py index a0b72b2d22..3be23cb8c0 100644 --- a/zerver/tests/test_message_topics.py +++ b/zerver/tests/test_message_topics.py @@ -6,8 +6,9 @@ from zerver.actions.streams import do_change_stream_permission from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import timeout_mock from zerver.lib.timeout import TimeoutExpiredError -from zerver.models import Message, UserMessage, get_client, get_stream +from zerver.models import Message, UserMessage, get_client from zerver.models.realms import get_realm +from zerver.models.streams import get_stream class TopicHistoryTest(ZulipTestCase): diff --git a/zerver/tests/test_outgoing_webhook_interfaces.py b/zerver/tests/test_outgoing_webhook_interfaces.py index 57ff699fef..cd3e29a4ce 100644 --- a/zerver/tests/test_outgoing_webhook_interfaces.py +++ b/zerver/tests/test_outgoing_webhook_interfaces.py @@ -12,8 +12,9 @@ from zerver.lib.outgoing_webhook import get_service_interface_class, process_suc from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.topic import TOPIC_NAME -from zerver.models import SLACK_INTERFACE, Message, NotificationTriggers, get_stream +from zerver.models import SLACK_INTERFACE, Message, NotificationTriggers from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_user from zerver.openapi.openapi import validate_against_openapi_schema diff --git a/zerver/tests/test_outgoing_webhook_system.py b/zerver/tests/test_outgoing_webhook_system.py index f6dad2df96..2a9620e4dd 100644 --- a/zerver/tests/test_outgoing_webhook_system.py +++ b/zerver/tests/test_outgoing_webhook_system.py @@ -19,8 +19,9 @@ from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic import TOPIC_NAME from zerver.lib.url_encoding import near_message_url from zerver.lib.users import add_service -from zerver.models import Recipient, Service, UserProfile, get_stream +from zerver.models import Recipient, Service, UserProfile from zerver.models.realms import get_realm +from zerver.models.streams import get_stream class ResponseMock: diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index 27a7ddbb3d..6a8261a575 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -91,9 +91,9 @@ from zerver.models import ( UserProfile, UserTopic, get_client, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zilencer.models import RemoteZulipServerAuditLog from zilencer.views import DevicesToCleanUpDict diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index 452fbca45c..f798821558 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -32,9 +32,9 @@ from zerver.models import ( UserActivity, UserProfile, get_client, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.tornado.event_queue import build_offline_notification from zerver.worker import queue_processors from zerver.worker.queue_processors import ( diff --git a/zerver/tests/test_realm.py b/zerver/tests/test_realm.py index a2fdbf446e..f93d525da6 100644 --- a/zerver/tests/test_realm.py +++ b/zerver/tests/test_realm.py @@ -53,10 +53,10 @@ from zerver.models import ( UserGroupMembership, UserMessage, UserProfile, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot, get_user_profile_by_id diff --git a/zerver/tests/test_retention.py b/zerver/tests/test_retention.py index 55d8b7dbcc..531b648c25 100644 --- a/zerver/tests/test_retention.py +++ b/zerver/tests/test_retention.py @@ -38,9 +38,9 @@ from zerver.models import ( SubMessage, UserMessage, get_client, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot # Class with helper functions useful for testing archiving of reactions: diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.py index 7210e55149..891b3167a4 100644 --- a/zerver/tests/test_signup.py +++ b/zerver/tests/test_signup.py @@ -76,9 +76,9 @@ from zerver.models import ( Subscription, UserMessage, UserProfile, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream from zerver.models.users import get_system_bot, get_user, get_user_by_delivery_email from zerver.views.auth import redirect_and_log_into_subdomain, start_two_factor_auth from zerver.views.development.registration import confirmation_key diff --git a/zerver/tests/test_soft_deactivation.py b/zerver/tests/test_soft_deactivation.py index d6c9309235..d7dea720c7 100644 --- a/zerver/tests/test_soft_deactivation.py +++ b/zerver/tests/test_soft_deactivation.py @@ -27,9 +27,9 @@ from zerver.models import ( UserActivity, UserMessage, UserProfile, - get_stream, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_stream logger_string = "zulip.soft_deactivation" diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py index a9a232cffe..c116fd7a94 100644 --- a/zerver/tests/test_subs.py +++ b/zerver/tests/test_subs.py @@ -104,12 +104,11 @@ from zerver.models import ( UserGroup, UserMessage, UserProfile, - get_default_stream_groups, - get_stream, validate_attachment_request, validate_attachment_request_for_spectator_access, ) from zerver.models.realms import get_realm +from zerver.models.streams import get_default_stream_groups, get_stream from zerver.models.users import active_non_guest_user_ids, get_user, get_user_profile_by_id_in_realm from zerver.views.streams import compose_views diff --git a/zerver/tests/test_user_topics.py b/zerver/tests/test_user_topics.py index b3f4af476b..ef48fc69ec 100644 --- a/zerver/tests/test_user_topics.py +++ b/zerver/tests/test_user_topics.py @@ -12,7 +12,8 @@ from zerver.lib.stream_topic import StreamTopicTarget from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_subscription from zerver.lib.user_topics import get_topic_mutes, topic_has_visibility_policy -from zerver.models import UserProfile, UserTopic, get_stream +from zerver.models import UserProfile, UserTopic +from zerver.models.streams import get_stream class MutedTopicsTestsDeprecated(ZulipTestCase): diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py index a66ec5b59a..d65ba87981 100644 --- a/zerver/tests/test_users.py +++ b/zerver/tests/test_users.py @@ -74,11 +74,11 @@ from zerver.models import ( UserTopic, check_valid_user_ids, get_client, - get_stream, ) from zerver.models.groups import SystemGroups from zerver.models.prereg_users import filter_to_valid_prereg_users from zerver.models.realms import InvalidFakeEmailDomainError, get_fake_email_domain, get_realm +from zerver.models.streams import get_stream from zerver.models.users import ( get_source_profile, get_system_bot, diff --git a/zerver/views/development/email_log.py b/zerver/views/development/email_log.py index 1081634321..2d570b9b2a 100644 --- a/zerver/views/development/email_log.py +++ b/zerver/views/development/email_log.py @@ -16,8 +16,9 @@ from zerver.actions.users import change_user_is_active from zerver.lib.email_notifications import enqueue_welcome_emails, send_account_registered_email from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.models import Realm, get_realm_stream +from zerver.models import Realm from zerver.models.realms import get_realm +from zerver.models.streams import get_realm_stream from zerver.models.users import get_user_by_delivery_email from zerver.views.invite import INVITATION_LINK_VALIDITY_MINUTES from zproject.email_backends import get_forward_address, set_forward_address diff --git a/zerver/views/registration.py b/zerver/views/registration.py index 6c4d3884d4..5a35f436bd 100644 --- a/zerver/views/registration.py +++ b/zerver/views/registration.py @@ -80,7 +80,6 @@ from zerver.models import ( RealmUserDefault, Stream, UserProfile, - get_default_stream_groups, ) from zerver.models.constants import MAX_LANGUAGE_ID_LENGTH from zerver.models.realms import ( @@ -91,6 +90,7 @@ from zerver.models.realms import ( get_realm, name_changes_disabled, ) +from zerver.models.streams import get_default_stream_groups from zerver.models.users import get_source_profile, get_user_by_delivery_email from zerver.views.auth import ( create_preregistration_realm, diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index eb3269b065..d45dcd1c4e 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -66,10 +66,10 @@ from zerver.models import ( UserProfile, flush_alert_word, get_client, - get_stream, ) from zerver.models.realms import get_realm from zerver.models.recipients import get_or_create_huddle +from zerver.models.streams import get_stream from zerver.models.users import get_user, get_user_by_delivery_email, get_user_profile_by_id from zilencer.models import RemoteRealm, RemoteZulipServer from zilencer.views import update_remote_realm_data_for_server