diff --git a/scripts/setup/generate_secrets.py b/scripts/setup/generate_secrets.py index c80a083f1d..e61c37106c 100755 --- a/scripts/setup/generate_secrets.py +++ b/scripts/setup/generate_secrets.py @@ -16,7 +16,6 @@ from django.utils.crypto import get_random_string import argparse import uuid import configparser -from zerver.lib.str_utils import force_str from zerver.lib.utils import generate_random_token os.chdir(os.path.join(os.path.dirname(__file__), '..', '..')) @@ -111,7 +110,7 @@ def generate_secrets(development=False): out = open(OUTPUT_SETTINGS_FILENAME, 'a') # Write a newline at the start, in case there was no newline at # the end of the file due to human editing. - out.write("\n" + force_str("".join(lines))) + out.write("\n" + "".join(lines)) out.close() print("Generated new secrets in %s." % (OUTPUT_SETTINGS_FILENAME,)) diff --git a/tools/test-backend b/tools/test-backend index 595a2a3391..482bcb2188 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -84,7 +84,6 @@ not_yet_fully_covered = { 'zerver/lib/rate_limiter.py', 'zerver/lib/sqlalchemy_utils.py', 'zerver/lib/storage.py', - 'zerver/lib/str_utils.py', 'zerver/lib/stream_recipient.py', 'zerver/lib/timeout.py', 'zerver/lib/unminify.py', diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py index 0f7fbf154b..0959cd5dcd 100644 --- a/zerver/lib/email_mirror.py +++ b/zerver/lib/email_mirror.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, cast import logging import re @@ -17,7 +17,6 @@ from zerver.lib.queue import queue_json_publish from zerver.lib.redis_utils import get_redis_client from zerver.lib.upload import upload_message_file from zerver.lib.utils import generate_random_token -from zerver.lib.str_utils import force_text from zerver.lib.send_email import FromAddress from zerver.models import Stream, Recipient, \ get_user_profile_by_id, get_display_recipient, get_personal_recipient, \ @@ -324,7 +323,7 @@ def process_message(message: message.Message, rcpt_to: Optional[str]=None, pre_c subject_header = "(no topic)" encoded_subject, encoding = decode_header(subject_header)[0] if encoding is None: - subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None + subject = cast(str, encoded_subject) # encoded_subject has type str when encoding is None else: try: subject = encoded_subject.decode(encoding) diff --git a/zerver/lib/storage.py b/zerver/lib/storage.py index acfdbf7a49..a6fd9e4c07 100644 --- a/zerver/lib/storage.py +++ b/zerver/lib/storage.py @@ -8,8 +8,6 @@ from django.conf import settings from django.contrib.staticfiles.storage import ManifestStaticFilesStorage from pipeline.storage import PipelineMixin -from zerver.lib.str_utils import force_str - class AddHeaderMixin: def post_process(self, paths: Dict[str, Tuple['ZulipStorage', str]], dry_run: bool=False, **kwargs: Any) -> List[Tuple[str, str, bool]]: @@ -38,7 +36,7 @@ class AddHeaderMixin: storage.delete(path) with storage.open(path, 'w') as new_file: - new_file.write(force_str(header + orig_contents, encoding=settings.FILE_CHARSET)) + new_file.write(header + orig_contents) ret_dict[path] = (path, path, True) diff --git a/zerver/lib/str_utils.py b/zerver/lib/str_utils.py deleted file mode 100644 index 7ca69e4048..0000000000 --- a/zerver/lib/str_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -String Utilities: - -This module helps in converting strings from one type to another. - -Currently we have strings of 3 semantic types: - -1. text strings: These strings are used to represent all textual data, - like people's names, stream names, content of messages, etc. - These strings can contain non-ASCII characters, so its type should be - typing.str (which is `str` in python 3 and `unicode` in python 2). - -2. binary strings: These strings are used to represent binary data. - This should be of type `bytes` - -3. native strings: These strings are for internal use only. Strings of - this type are not meant to be stored in database, displayed to end - users, etc. Things like exception names, parameter names, attribute - names, etc should be native strings. These strings should only - contain ASCII characters and they should have type `str`. - -There are 3 utility functions provided for converting strings from one type -to another - force_text, force_bytes, force_str - -Interconversion between text strings and binary strings can be done by -using encode and decode appropriately or by using the utility functions -force_text and force_bytes. - -It is recommended to use the utility functions for other string conversions. -""" - -from typing import Any, Dict, Mapping, Union, TypeVar - -def force_text(s: Union[str, bytes], encoding: str='utf-8') -> str: - """converts a string to a text string""" - if isinstance(s, str): - return s - elif isinstance(s, bytes): - return s.decode(encoding) - else: - raise TypeError("force_text expects a string type") - -def force_str(s: Union[str, bytes], encoding: str='utf-8') -> str: - """converts a string to a native string""" - if isinstance(s, str): - return s - elif isinstance(s, str): - return s.encode(encoding) - elif isinstance(s, bytes): - return s.decode(encoding) - else: - raise TypeError("force_str expects a string type") diff --git a/zerver/lib/zephyr.py b/zerver/lib/zephyr.py index 05beff1db2..2d11536d9a 100644 --- a/zerver/lib/zephyr.py +++ b/zerver/lib/zephyr.py @@ -2,8 +2,6 @@ import re import traceback import DNS -from zerver.lib.str_utils import force_str - def compute_mit_user_fullname(email: str) -> str: try: # Input is either e.g. username@mit.edu or user|CROSSREALM.INVALID@mit.edu @@ -12,7 +10,7 @@ def compute_mit_user_fullname(email: str) -> str: answer = DNS.dnslookup( "%s.passwd.ns.athena.mit.edu" % (match_user.group(1),), DNS.Type.TXT) - hesiod_name = force_str(answer[0][0]).split(':')[4].split(',')[0].strip() + hesiod_name = answer[0][0].split(':')[4].split(',')[0].strip() if hesiod_name != "": return hesiod_name elif match_user: diff --git a/zerver/management/commands/create_stream.py b/zerver/management/commands/create_stream.py index cdfb70d1c0..de75e6c4fe 100644 --- a/zerver/management/commands/create_stream.py +++ b/zerver/management/commands/create_stream.py @@ -5,7 +5,6 @@ from typing import Any from zerver.lib.actions import create_stream_if_needed from zerver.lib.management import ZulipBaseCommand -from zerver.lib.str_utils import force_text class Command(ZulipBaseCommand): help = """Create a stream, and subscribe all active users (excluding bots). @@ -22,6 +21,5 @@ the command.""" realm = self.get_realm(options) assert realm is not None # Should be ensured by parser - encoding = sys.getfilesystemencoding() stream_name = options['stream_name'] - create_stream_if_needed(realm, force_text(stream_name, encoding)) + create_stream_if_needed(realm, stream_name) diff --git a/zerver/management/commands/makemessages.py b/zerver/management/commands/makemessages.py index 4f881ebc31..c5ea840d79 100644 --- a/zerver/management/commands/makemessages.py +++ b/zerver/management/commands/makemessages.py @@ -41,8 +41,6 @@ from django.core.management.commands import makemessages from django.template.base import BLOCK_TAG_END, BLOCK_TAG_START from django.utils.translation import template -from zerver.lib.str_utils import force_text - strip_whitespace_right = re.compile("(%s-?\\s*(trans|pluralize).*?-%s)\\s+" % ( BLOCK_TAG_START, BLOCK_TAG_END), re.U) strip_whitespace_left = re.compile("\\s+(%s-\\s*(endtrans|pluralize).*?-?%s)" % ( @@ -249,7 +247,7 @@ class Command(makemessages.Command): old_strings = {} new_strings = { - force_text(k): v + k: v for k, v in self.get_new_strings(old_strings, translation_strings, locale).items() diff --git a/zerver/management/commands/rename_stream.py b/zerver/management/commands/rename_stream.py index 36c93ff502..37d0199875 100644 --- a/zerver/management/commands/rename_stream.py +++ b/zerver/management/commands/rename_stream.py @@ -5,7 +5,6 @@ from typing import Any from zerver.lib.actions import do_rename_stream from zerver.lib.management import ZulipBaseCommand -from zerver.lib.str_utils import force_text from zerver.models import get_stream class Command(ZulipBaseCommand): @@ -23,7 +22,6 @@ class Command(ZulipBaseCommand): assert realm is not None # Should be ensured by parser old_name = options['old_name'] new_name = options['new_name'] - encoding = sys.getfilesystemencoding() - stream = get_stream(force_text(old_name, encoding), realm) - do_rename_stream(stream, force_text(new_name, encoding)) + stream = get_stream(old_name, realm) + do_rename_stream(stream, new_name) diff --git a/zerver/views/zephyr.py b/zerver/views/zephyr.py index 94836f3d8d..c34330fee7 100644 --- a/zerver/views/zephyr.py +++ b/zerver/views/zephyr.py @@ -7,7 +7,6 @@ from zerver.decorator import authenticated_json_view from zerver.lib.ccache import make_ccache from zerver.lib.request import has_request_variables, REQ, JsonableError from zerver.lib.response import json_success, json_error -from zerver.lib.str_utils import force_str from zerver.lib.users import get_api_key from zerver.models import UserProfile @@ -48,9 +47,9 @@ def webathena_kerberos_login(request: HttpRequest, user_profile: UserProfile, api_key = get_api_key(user_profile) subprocess.check_call(["ssh", settings.PERSONAL_ZMIRROR_SERVER, "--", "/home/zulip/python-zulip-api/zulip/integrations/zephyr/process_ccache", - force_str(user), - force_str(api_key), - force_str(base64.b64encode(ccache))]) + user, + api_key, + base64.b64encode(ccache).decode("utf-8")]) except Exception: logging.exception("Error updating the user's ccache") return json_error(_("We were unable to setup mirroring for you")) diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index 947ad9efcc..987ae5e50e 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -40,7 +40,6 @@ from zerver.tornado.socket import req_redis_key, respond_send_message from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.db import reset_queries from zerver.lib.redis_utils import get_redis_client -from zerver.lib.str_utils import force_str from zerver.context_processors import common_context from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler from zerver.models import get_bot_services @@ -479,8 +478,7 @@ class DigestWorker(QueueProcessingWorker): # nocoverage @assign_queue('email_mirror') class MirrorWorker(QueueProcessingWorker): def consume(self, event: Mapping[str, Any]) -> None: - message = force_str(event["message"]) - mirror_email(email.message_from_string(message), + mirror_email(email.message_from_string(event["message"]), rcpt_to=event["rcpt_to"], pre_checked=True) @assign_queue('test', queue_type="test")