From 97e4e9886cbf34f534fa3f2a6c27e225d4ed7897 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Fri, 21 Jan 2022 22:52:54 -0800 Subject: [PATCH] python: Replace universal_newlines with text. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is supported in Python ≥ 3.7. Signed-off-by: Anders Kaseorg --- .../zulip_postgresql/check_postgresql_replication_lag | 4 +--- puppet/zulip/files/postgresql/pg_backup_and_purge | 2 +- scripts/lib/check_rabbitmq_queue.py | 4 ++-- scripts/lib/hash_reqs.py | 2 +- scripts/lib/node_cache.py | 4 +--- scripts/lib/puppet_cache.py | 2 +- scripts/lib/setup_venv.py | 6 ++---- scripts/lib/sharding.py | 2 +- scripts/lib/upgrade-zulip | 2 +- scripts/lib/upgrade-zulip-stage-2 | 2 +- scripts/lib/zulip_tools.py | 2 +- scripts/nagios/check-rabbitmq-consumers | 4 +--- scripts/setup/restore-backup | 2 +- tools/diagnose | 2 +- tools/i18n/process-mobile-i18n | 2 +- tools/lib/pretty_print.py | 2 +- tools/review | 2 +- tools/test-queue-worker-reload | 2 +- tools/update-zuliprc-api-field | 4 +--- zerver/lib/mdiff.py | 2 +- zerver/lib/test_fixtures.py | 2 +- zerver/lib/tex.py | 4 +--- zerver/logging_handlers.py | 2 +- zerver/management/commands/compilemessages.py | 2 +- zerver/openapi/javascript_examples.py | 2 +- zerver/openapi/test_curl_examples.py | 4 +--- zerver/tests/test_email_mirror.py | 4 ++-- zerver/views/report.py | 2 +- 28 files changed, 31 insertions(+), 45 deletions(-) diff --git a/puppet/zulip/files/nagios_plugins/zulip_postgresql/check_postgresql_replication_lag b/puppet/zulip/files/nagios_plugins/zulip_postgresql/check_postgresql_replication_lag index e3da831f3c..393554a9f2 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_postgresql/check_postgresql_replication_lag +++ b/puppet/zulip/files/nagios_plugins/zulip_postgresql/check_postgresql_replication_lag @@ -57,9 +57,7 @@ def run_sql_query(query: str) -> List[List[str]]: f"SELECT {query}", ] try: - output = subprocess.check_output( - command, stderr=subprocess.STDOUT, universal_newlines=True - ).strip() + output = subprocess.check_output(command, stderr=subprocess.STDOUT, text=True).strip() if not output: return [] return [x.split("\0") for x in output.split("\n")] diff --git a/puppet/zulip/files/postgresql/pg_backup_and_purge b/puppet/zulip/files/postgresql/pg_backup_and_purge index 989987a9cc..9af5b7b432 100755 --- a/puppet/zulip/files/postgresql/pg_backup_and_purge +++ b/puppet/zulip/files/postgresql/pg_backup_and_purge @@ -21,7 +21,7 @@ def run(args: List[str], dry_run: bool = False) -> str: print("Would have run: " + " ".join(map(shlex.quote, args))) return "" - return subprocess.check_output(args, stdin=subprocess.DEVNULL, universal_newlines=True) + return subprocess.check_output(args, stdin=subprocess.DEVNULL, text=True) recovery_val = run( diff --git a/scripts/lib/check_rabbitmq_queue.py b/scripts/lib/check_rabbitmq_queue.py index cae5b05f11..619ea6e4e8 100644 --- a/scripts/lib/check_rabbitmq_queue.py +++ b/scripts/lib/check_rabbitmq_queue.py @@ -134,7 +134,7 @@ def check_rabbitmq_queues() -> None: list_queues_output = subprocess.check_output( ["/usr/sbin/rabbitmqctl", "list_queues", "name", "messages", "consumers"], - universal_newlines=True, + text=True, ) queue_counts_rabbitmqctl = {} queues_with_consumers = [] @@ -151,7 +151,7 @@ def check_rabbitmq_queues() -> None: queue_stats_dir = subprocess.check_output( [os.path.join(ZULIP_PATH, "scripts/get-django-setting"), "QUEUE_STATS_DIR"], - universal_newlines=True, + text=True, ).strip() queue_stats: Dict[str, Dict[str, Any]] = {} queues_to_check = set(normal_queues).intersection(set(queues_with_consumers)) diff --git a/scripts/lib/hash_reqs.py b/scripts/lib/hash_reqs.py index 4d7c4b2641..87a3995503 100755 --- a/scripts/lib/hash_reqs.py +++ b/scripts/lib/hash_reqs.py @@ -35,7 +35,7 @@ def python_version() -> str: """ Returns the Python version as string 'Python major.minor.patchlevel' """ - return subprocess.check_output(["/usr/bin/python3", "-VV"], universal_newlines=True) + return subprocess.check_output(["/usr/bin/python3", "-VV"], text=True) def hash_deps(deps: Iterable[str]) -> str: diff --git a/scripts/lib/node_cache.py b/scripts/lib/node_cache.py index 1ae3835e93..afd6cfca7e 100644 --- a/scripts/lib/node_cache.py +++ b/scripts/lib/node_cache.py @@ -42,9 +42,7 @@ def generate_sha1sum_node_modules( data[YARN_LOCK_FILE_PATH] = f.read().strip() with open(YARN_PACKAGE_JSON) as f: data["yarn-package-version"] = json.load(f)["version"] - data["node-version"] = subprocess.check_output( - ["node", "--version"], universal_newlines=True - ).strip() + data["node-version"] = subprocess.check_output(["node", "--version"], text=True).strip() data["yarn-args"] = get_yarn_args(production=production) sha1sum = hashlib.sha1() diff --git a/scripts/lib/puppet_cache.py b/scripts/lib/puppet_cache.py index 5abe00bef0..1e85c4994b 100644 --- a/scripts/lib/puppet_cache.py +++ b/scripts/lib/puppet_cache.py @@ -23,7 +23,7 @@ def generate_sha1sum_puppet_modules() -> str: data["puppet-version"] = subprocess.check_output( # This is 10x faster than `puppet --version` ["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"], - universal_newlines=True, + text=True, ).strip() sha1sum = hashlib.sha1() diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index 4c77366045..ad5ebd66d7 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -178,9 +178,7 @@ def try_to_copy_venv(venv_path: str, new_packages: Set[str]) -> bool: venv_python3 = os.path.join(curr_venv_path, "bin", "python3") if not os.path.exists(venv_python3): continue - venv_python_version = subprocess.check_output( - [venv_python3, "-VV"], universal_newlines=True - ) + venv_python_version = subprocess.check_output([venv_python3, "-VV"], text=True) if desired_python_version != venv_python_version: continue @@ -282,7 +280,7 @@ def do_patch_activate_script(venv_path: str) -> None: def generate_hash(requirements_file: str) -> str: path = os.path.join(ZULIP_PATH, "scripts", "lib", "hash_reqs.py") - output = subprocess.check_output([path, requirements_file], universal_newlines=True) + output = subprocess.check_output([path, requirements_file], text=True) return output.split()[0] diff --git a/scripts/lib/sharding.py b/scripts/lib/sharding.py index 65b7b34fc1..127a9aa9d3 100755 --- a/scripts/lib/sharding.py +++ b/scripts/lib/sharding.py @@ -53,7 +53,7 @@ def write_updated_configs() -> None: shard_map: Dict[str, int] = {} external_host = subprocess.check_output( [os.path.join(BASE_DIR, "scripts/get-django-setting"), "EXTERNAL_HOST"], - universal_newlines=True, + text=True, ).strip() for port in config_file["tornado_sharding"]: shards = config_file["tornado_sharding"][port].strip() diff --git a/scripts/lib/upgrade-zulip b/scripts/lib/upgrade-zulip index fbf108b976..4cf8d44056 100755 --- a/scripts/lib/upgrade-zulip +++ b/scripts/lib/upgrade-zulip @@ -54,7 +54,7 @@ try: logging.info("Unpacking the tarball") unpack_zulip = os.path.realpath(os.path.join(os.path.dirname(__file__), "unpack-zulip")) deploy_path = subprocess.check_output( - [unpack_zulip, archived_tarball_path], preexec_fn=su_to_zulip, universal_newlines=True + [unpack_zulip, archived_tarball_path], preexec_fn=su_to_zulip, text=True ) # Chdir to deploy_path and then run upgrade-zulip-stage-2 from the diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index d866e80096..693900b269 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -263,7 +263,7 @@ migrations_needed = False if not args.skip_migrations: logging.info("Checking for needed migrations") migrations_output = subprocess.check_output( - ["./manage.py", "showmigrations"], preexec_fn=su_to_zulip, universal_newlines=True + ["./manage.py", "showmigrations"], preexec_fn=su_to_zulip, text=True ) need_create_large_indexes = False for ln in migrations_output.split("\n"): diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py index 2ad4cbeaa2..18274303f5 100755 --- a/scripts/lib/zulip_tools.py +++ b/scripts/lib/zulip_tools.py @@ -620,7 +620,7 @@ def has_application_server(once: bool = False) -> bool: def list_supervisor_processes(*args: str) -> List[str]: worker_status = subprocess.run( ["supervisorctl", "status", *args], - universal_newlines=True, + text=True, stdout=subprocess.PIPE, ) # `supervisorctl status` returns 3 if any are stopped, which is diff --git a/scripts/nagios/check-rabbitmq-consumers b/scripts/nagios/check-rabbitmq-consumers index b579c7a127..ed0e42b156 100755 --- a/scripts/nagios/check-rabbitmq-consumers +++ b/scripts/nagios/check-rabbitmq-consumers @@ -33,9 +33,7 @@ options = parser.parse_args() config_file = get_config_file() TORNADO_PROCESSES = len(get_tornado_ports(config_file)) -output = subprocess.check_output( - ["/usr/sbin/rabbitmqctl", "list_consumers"], universal_newlines=True -) +output = subprocess.check_output(["/usr/sbin/rabbitmqctl", "list_consumers"], text=True) consumers: Dict[str, int] = defaultdict(int) diff --git a/scripts/setup/restore-backup b/scripts/setup/restore-backup index 642b4840db..c9d8a1bdae 100755 --- a/scripts/setup/restore-backup +++ b/scripts/setup/restore-backup @@ -124,7 +124,7 @@ def restore_backup(tarball_file: IO[bytes]) -> None: os.path.join(settings.DEPLOY_ROOT, "scripts", "get-django-setting"), "RABBITMQ_HOST", ], - universal_newlines=True, + text=True, ).strip() if rabbitmq_host in ["127.0.0.1", "::1", "localhost", "localhost6"]: run([os.path.join(settings.DEPLOY_ROOT, "scripts", "setup", "configure-rabbitmq")]) diff --git a/tools/diagnose b/tools/diagnose index ae125dc970..e992db50a2 100755 --- a/tools/diagnose +++ b/tools/diagnose @@ -114,7 +114,7 @@ def check_venv() -> bool: cache_dir = "/srv/zulip-venv-cache/" for fn in ["dev.txt"]: requirements_file = os.path.join(ROOT_DIR, "requirements", fn) - output = subprocess.check_output([path, requirements_file], universal_newlines=True) + output = subprocess.check_output([path, requirements_file], text=True) sha1sum = output.split()[0] print(fn, "venv sha: ", sha1sum) if not os.path.exists(os.path.join(cache_dir, sha1sum)): diff --git a/tools/i18n/process-mobile-i18n b/tools/i18n/process-mobile-i18n index d593ccc7d5..64b43a5167 100755 --- a/tools/i18n/process-mobile-i18n +++ b/tools/i18n/process-mobile-i18n @@ -11,7 +11,7 @@ def get_json_filename(locale: str) -> str: def get_locales() -> List[str]: - output = check_output(["git", "ls-files", "locale"], universal_newlines=True) + output = check_output(["git", "ls-files", "locale"], text=True) tracked_files = output.split() regex = re.compile(r"locale/(\w+)/LC_MESSAGES/django.po") locales = ["en"] diff --git a/tools/lib/pretty_print.py b/tools/lib/pretty_print.py index daa564d43a..1691248cb6 100644 --- a/tools/lib/pretty_print.py +++ b/tools/lib/pretty_print.py @@ -177,7 +177,7 @@ Proposed {BOLDRED}diff{ENDC} for {CYAN}{fn}{ENDC}: """, flush=True, ) - subprocess.run(["diff", fn, "-"], input=phtml, universal_newlines=True) + subprocess.run(["diff", fn, "-"], input=phtml, text=True) print( f""" --- diff --git a/tools/review b/tools/review index 98b5540b60..820c0e37d6 100755 --- a/tools/review +++ b/tools/review @@ -17,7 +17,7 @@ def run(command: List[str]) -> None: def check_output(command: List[str]) -> str: - return subprocess.check_output(command, universal_newlines=True) + return subprocess.check_output(command, text=True) def get_git_branch() -> str: diff --git a/tools/test-queue-worker-reload b/tools/test-queue-worker-reload index 479f8bef82..4ee137cd6e 100755 --- a/tools/test-queue-worker-reload +++ b/tools/test-queue-worker-reload @@ -66,7 +66,7 @@ if __name__ == "__main__": bufsize=1, # line buffered stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, ) failed = check_worker_launch(run_dev) diff --git a/tools/update-zuliprc-api-field b/tools/update-zuliprc-api-field index 290cd6e606..c9fde4f53d 100755 --- a/tools/update-zuliprc-api-field +++ b/tools/update-zuliprc-api-field @@ -62,9 +62,7 @@ for zuliprc_path in zuliprc_paths_list: arguments = ["./manage.py", "print_initial_password", email] # We redirect 'stderr' to 'stdout' to avoid 'Connection to 127.0.0.1 closed' # appearing after this script finishes. - output = subprocess.check_output( - arguments, stderr=subprocess.STDOUT, universal_newlines=True - ) + output = subprocess.check_output(arguments, stderr=subprocess.STDOUT, text=True) new_key = output.split()[6] if new_key != key: diff --git a/zerver/lib/mdiff.py b/zerver/lib/mdiff.py index bca684a823..ccd58c521b 100755 --- a/zerver/lib/mdiff.py +++ b/zerver/lib/mdiff.py @@ -16,5 +16,5 @@ def diff_strings(output: str, expected_output: str) -> str: raise DiffException(msg) command = ["node", mdiff_path, output, expected_output] - diff = subprocess.check_output(command, universal_newlines=True) + diff = subprocess.check_output(command, text=True) return diff diff --git a/zerver/lib/test_fixtures.py b/zerver/lib/test_fixtures.py index 3db3a1d2a0..7fa9a59395 100644 --- a/zerver/lib/test_fixtures.py +++ b/zerver/lib/test_fixtures.py @@ -370,7 +370,7 @@ def destroy_leaked_test_databases(expiry_time: int = 60 * 60) -> int: ["psql", "-q", "-v", "ON_ERROR_STOP=1", "-h", "localhost", "postgres", "zulip_test"], input=commands, check=True, - universal_newlines=True, + text=True, ) return len(databases_to_drop) diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py index 441df8313c..d3bfd37c0b 100644 --- a/zerver/lib/tex.py +++ b/zerver/lib/tex.py @@ -34,9 +34,7 @@ def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: if not is_inline: command.extend(["--display-mode"]) try: - stdout = subprocess.check_output( - command, input=tex, stderr=subprocess.DEVNULL, universal_newlines=True - ) + stdout = subprocess.check_output(command, input=tex, stderr=subprocess.DEVNULL, text=True) # stdout contains a newline at the end return stdout.strip() except subprocess.CalledProcessError: diff --git a/zerver/logging_handlers.py b/zerver/logging_handlers.py index 5b5e3e8684..e38e8fe24d 100644 --- a/zerver/logging_handlers.py +++ b/zerver/logging_handlers.py @@ -25,7 +25,7 @@ def try_git_describe() -> Optional[str]: ["git", "describe", "--tags", "--match=[0-9]*", "--always", "--dirty", "--long"], stderr=subprocess.PIPE, cwd=os.path.join(os.path.dirname(__file__), ".."), - universal_newlines=True, + text=True, ).strip() except (FileNotFoundError, subprocess.CalledProcessError): # nocoverage return None diff --git a/zerver/management/commands/compilemessages.py b/zerver/management/commands/compilemessages.py index 1b0d89ed6f..322575c925 100644 --- a/zerver/management/commands/compilemessages.py +++ b/zerver/management/commands/compilemessages.py @@ -70,7 +70,7 @@ class Command(compilemessages.Command): raise Exception(f"Unknown language {locale}") def get_locales(self) -> List[str]: - output = check_output(["git", "ls-files", "locale"], universal_newlines=True) + output = check_output(["git", "ls-files", "locale"], text=True) tracked_files = output.split() regex = re.compile(r"locale/(\w+)/LC_MESSAGES/django.po") locales = ["en"] diff --git a/zerver/openapi/javascript_examples.py b/zerver/openapi/javascript_examples.py index 56dc09cd28..5305163f65 100644 --- a/zerver/openapi/javascript_examples.py +++ b/zerver/openapi/javascript_examples.py @@ -20,7 +20,7 @@ def test_js_bindings(client: Client) -> None: output = subprocess.check_output( args=["node", "--unhandled-rejections=strict", "zerver/openapi/javascript_examples.js"], - universal_newlines=True, + text=True, ) endpoint_responses = json.loads(output) diff --git a/zerver/openapi/test_curl_examples.py b/zerver/openapi/test_curl_examples.py index 65eba9bc28..8184bf67e2 100644 --- a/zerver/openapi/test_curl_examples.py +++ b/zerver/openapi/test_curl_examples.py @@ -98,9 +98,7 @@ def test_generated_curl_examples_for_success(client: Client) -> None: try: # We split this across two lines so if curl fails and # returns non-JSON output, we'll still print it. - response_json = subprocess.check_output( - generated_curl_command, universal_newlines=True - ) + response_json = subprocess.check_output(generated_curl_command, text=True) response = json.loads(response_json) assert response["result"] == "success" except (AssertionError, Exception): diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py index 8a53aad063..002c478a46 100644 --- a/zerver/tests/test_email_mirror.py +++ b/zerver/tests/test_email_mirror.py @@ -1283,7 +1283,7 @@ class TestScriptMTA(ZulipTestCase): [script, "-r", stream_to_address, "-s", settings.SHARED_SECRET, "-t"], input=mail, check=True, - universal_newlines=True, + text=True, ) def test_error_no_recipient(self) -> None: @@ -1298,7 +1298,7 @@ class TestScriptMTA(ZulipTestCase): [script, "-s", settings.SHARED_SECRET, "-t"], input=mail, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, ) self.assertEqual( p.stdout, diff --git a/zerver/views/report.py b/zerver/views/report.py index 29ae2ef4e3..d4d4a51fef 100644 --- a/zerver/views/report.py +++ b/zerver/views/report.py @@ -134,7 +134,7 @@ def report_error( try: version: Optional[str] = subprocess.check_output( ["git", "show", "-s", "--oneline"], - universal_newlines=True, + text=True, ) except (FileNotFoundError, subprocess.CalledProcessError): version = None