From 5901e7ba7e9868f636e210572abfa66a7d569615 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Sat, 18 Apr 2020 18:48:37 -0700 Subject: [PATCH] python: Convert function type annotations to Python 3 style. Generated by com2ann (slightly patched to avoid also converting assignment type annotations, which require Python 3.6), followed by some manual whitespace adjustment, and six fixes for runtime issues: - def __init__(self, token: Token, parent: Optional[Node]) -> None: + def __init__(self, token: Token, parent: "Optional[Node]") -> None: -def main(options: argparse.Namespace) -> NoReturn: +def main(options: argparse.Namespace) -> "NoReturn": -def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]: +def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]": -def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None: +def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None: -def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool: +def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool: - method_kwarg_pairs: List[FuncKwargPair], + method_kwarg_pairs: "List[FuncKwargPair]", Signed-off-by: Anders Kaseorg --- .../check_send_receive_time | 9 +- .../check_postgres_replication_lag | 9 +- .../zulip_postgres_appdb/check_fts_update_log | 3 +- .../check_postgres_backup | 3 +- .../files/postgresql/pg_backup_and_purge | 3 +- .../files/postgresql/process_fts_updates | 6 +- .../check_personal_zephyr_mirrors | 3 +- .../check_user_zephyr_mirror_liveness | 3 +- .../zulip_zephyr_mirror/check_zephyr_mirror | 3 +- scripts/lib/clean-unused-caches | 3 +- scripts/lib/clean_emoji_cache.py | 3 +- scripts/lib/clean_node_cache.py | 3 +- scripts/lib/clean_venv_cache.py | 3 +- scripts/lib/email-mirror-postfix | 8 +- scripts/lib/hash_reqs.py | 12 +- scripts/lib/node_cache.py | 26 ++- scripts/lib/setup_venv.py | 49 ++-- scripts/lib/zulip_tools.py | 83 +++---- scripts/nagios/check-rabbitmq-consumers | 3 +- scripts/nagios/cron_file_helper.py | 3 +- scripts/purge-old-deployments | 9 +- scripts/setup/generate_secrets.py | 15 +- scripts/setup/restore-backup | 3 +- tools/check-frontend-i18n | 3 +- tools/check-issue-labels | 15 +- tools/check-provision | 3 +- tools/check-templates | 9 +- tools/create-test-api-docs | 15 +- tools/diagnose | 33 +-- tools/droplets/create.py | 24 +- tools/get-handlebar-vars | 12 +- tools/html-grep | 3 +- tools/js-dep-visualizer.py | 21 +- tools/lib/capitalization.py | 15 +- tools/lib/gitlint-rules.py | 9 +- tools/lib/graph.py | 39 ++-- tools/lib/html_branches.py | 37 ++- tools/lib/html_grep.py | 12 +- tools/lib/pretty_print.py | 6 +- tools/lib/provision.py | 12 +- tools/lib/provision_inner.py | 6 +- tools/lib/sanity_check.py | 3 +- tools/lib/template_parser.py | 99 +++----- tools/lib/test_script.py | 3 +- tools/lib/test_server.py | 9 +- tools/lint | 18 +- tools/linter_lib/pep8.py | 3 +- tools/linter_lib/pyflakes.py | 3 +- tools/pretty-print-html | 3 +- tools/renumber-migrations | 9 +- tools/replacer | 3 +- tools/review | 24 +- tools/run-dev.py | 50 ++-- .../setup/generate_zulip_bots_static_files.py | 3 +- tools/test-emoji-name-scripts | 3 +- tools/test-js-with-node | 3 +- tools/test-locked-requirements | 21 +- tools/test-tools | 3 +- tools/webpack | 12 +- zerver/openapi/python_examples.py | 219 ++++++------------ zerver/tests/test_auth_backends.py | 6 +- zerver/tests/test_bots.py | 3 +- zerver/tests/test_events.py | 3 +- zerver/tests/test_messages.py | 13 +- zerver/tests/test_push_notifications.py | 9 +- zerver/tests/test_service_bot_system.py | 3 +- zerver/tests/test_signup.py | 6 +- zerver/views/streams.py | 7 +- 68 files changed, 389 insertions(+), 691 deletions(-) diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time index ca7fed8e62..89f4c8a403 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time @@ -85,8 +85,7 @@ states = { "UNKNOWN": 3 } -def report(state, timestamp=None, msg=None): - # type: (str, Any, Optional[str]) -> None +def report(state: str, timestamp: Any = None, msg: Optional[str] = None) -> None: now = int(time.time()) if msg is None: msg = "send time was %s" % (timestamp,) @@ -97,14 +96,12 @@ def report(state, timestamp=None, msg=None): print("%s: %s" % (state, msg)) exit(states[state]) -def send_zulip(sender, message): - # type: (zulip.Client, Dict[str, Any]) -> None +def send_zulip(sender: zulip.Client, message: Dict[str, Any]) -> None: result = sender.send_message(message) if result["result"] != "success" and options.nagios: report("CRITICAL", msg="Error sending Zulip, args were: %s, %s" % (message, result)) -def get_zulips(): - # type: () -> List[Dict[str, Any]] +def get_zulips() -> List[Dict[str, Any]]: global queue_id, last_event_id res = zulip_recipient.get_events(queue_id=queue_id, last_event_id=last_event_id) if 'error' in res.get('result', {}): diff --git a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag index e1c500b037..52314a5523 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag +++ b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag @@ -20,13 +20,11 @@ states = { "UNKNOWN": 3 } -def report(state, msg): - # type: (str, str) -> NoReturn +def report(state: str, msg: str) -> NoReturn: print("%s: %s" % (state, msg)) exit(states[state]) -def get_loc_over_ssh(host, func): - # type: (str, str) -> str +def get_loc_over_ssh(host: str, func: str) -> str: try: return subprocess.check_output(['ssh', host, 'psql -v ON_ERROR_STOP=1 zulip -t -c "SELECT %s()"' % (func,)], @@ -35,8 +33,7 @@ def get_loc_over_ssh(host, func): except subprocess.CalledProcessError as e: report('CRITICAL', 'ssh failed: %s: %s' % (str(e), e.output)) -def loc_to_abs_offset(loc_str): - # type: (str) -> int +def loc_to_abs_offset(loc_str: str) -> int: m = re.match(r'^\s*([0-9a-fA-F]+)/([0-9a-fA-F]+)\s*$', loc_str) if not m: raise ValueError("Unknown xlog location format: " + loc_str) diff --git a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log index 8dafa5a1bb..0571287bdc 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log +++ b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log @@ -21,8 +21,7 @@ states = { "UNKNOWN": 3 } -def report(state, num): - # type: (str, str) -> None +def report(state: str, num: str) -> None: print("%s: %s rows in fts_update_log table" % (state, num)) exit(states[state]) diff --git a/puppet/zulip/files/nagios_plugins/zulip_postgres_common/check_postgres_backup b/puppet/zulip/files/nagios_plugins/zulip_postgres_common/check_postgres_backup index 8724caea7f..ae05c16510 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_postgres_common/check_postgres_backup +++ b/puppet/zulip/files/nagios_plugins/zulip_postgres_common/check_postgres_backup @@ -13,8 +13,7 @@ states = { "UNKNOWN": 3 } -def report(state, msg): - # type: (str, str) -> None +def report(state: str, msg: str) -> None: print("%s: %s" % (state, msg)) exit(states[state]) diff --git a/puppet/zulip/files/postgresql/pg_backup_and_purge b/puppet/zulip/files/postgresql/pg_backup_and_purge index 583e1266ad..7e15cbc3cb 100755 --- a/puppet/zulip/files/postgresql/pg_backup_and_purge +++ b/puppet/zulip/files/postgresql/pg_backup_and_purge @@ -16,8 +16,7 @@ logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s") logger = logging.getLogger(__name__) -def run(args, dry_run=False): - # type: (List[str], bool) -> str +def run(args: List[str], dry_run: bool = False) -> str: if dry_run: print("Would have run: " + " ".join(map(shlex.quote, args))) return "" diff --git a/puppet/zulip/files/postgresql/process_fts_updates b/puppet/zulip/files/postgresql/process_fts_updates index 9518737748..8411ddf9f7 100755 --- a/puppet/zulip/files/postgresql/process_fts_updates +++ b/puppet/zulip/files/postgresql/process_fts_updates @@ -33,8 +33,7 @@ import os BATCH_SIZE = 1000 -def update_fts_columns(cursor): - # type: (psycopg2.extensions.cursor) -> int +def update_fts_columns(cursor: psycopg2.extensions.cursor) -> int: cursor.execute("SELECT id, message_id FROM fts_update_log LIMIT %s;" % ( BATCH_SIZE,)) ids = [] @@ -52,8 +51,7 @@ def update_fts_columns(cursor): cursor.execute("DELETE FROM fts_update_log WHERE id = ANY(%s)", (ids,)) return len(ids) -def am_master(cursor): - # type: (psycopg2.extensions.cursor) -> bool +def am_master(cursor: psycopg2.extensions.cursor) -> bool: cursor.execute("SELECT pg_is_in_recovery()") return not cursor.fetchall()[0][0] diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors index 1c2b82da26..f9ea4383df 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors @@ -21,8 +21,7 @@ states = { "UNKNOWN": 3 } # type: Dict[str, int] -def report(state, output): - # type: (str, str) -> None +def report(state: str, output: str) -> None: print("%s\n%s" % (state, output)) exit(states[state]) diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness index 54902bd5f0..3b16960ddf 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness @@ -35,8 +35,7 @@ states = { "UNKNOWN": 3 } # type: Dict[str, int] -def report(state, short_msg, too_old=None): - # type: (str, str, Optional[Set[Any]]) -> None +def report(state: str, short_msg: str, too_old: Optional[Set[Any]] = None) -> None: too_old_data = "" if too_old: too_old_data = "\nLast call to get_message for recently out of date mirrors:\n" + "\n".join( diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror index e55d83c5e5..b608bab6d8 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror @@ -22,8 +22,7 @@ states = { "UNKNOWN": 3 } # type: Dict[str, int] -def report(state, data, last_check): - # type: (str, str, float) -> None +def report(state: str, data: str, last_check: float) -> None: print("%s: Last test run completed at %s\n%s" % ( state, time.strftime("%Y-%m-%d %H:%M %Z", time.gmtime(last_check)), data)) diff --git a/scripts/lib/clean-unused-caches b/scripts/lib/clean-unused-caches index ec4f55592e..480c6a2e03 100755 --- a/scripts/lib/clean-unused-caches +++ b/scripts/lib/clean-unused-caches @@ -7,8 +7,7 @@ sys.path.append(ZULIP_PATH) from scripts.lib.zulip_tools import parse_cache_script_args from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache -def main(): - # type: () -> None +def main() -> None: args = parse_cache_script_args("This script cleans unused zulip caches.") os.chdir(ZULIP_PATH) clean_venv_cache.main(args) diff --git a/scripts/lib/clean_emoji_cache.py b/scripts/lib/clean_emoji_cache.py index 80402e5594..ec96d7f67b 100755 --- a/scripts/lib/clean_emoji_cache.py +++ b/scripts/lib/clean_emoji_cache.py @@ -16,8 +16,7 @@ EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache" if ENV == "travis": EMOJI_CACHE_PATH = os.path.join(os.environ["HOME"], "zulip-emoji-cache") -def get_caches_in_use(threshold_days): - # type: (int) -> Set[str] +def get_caches_in_use(threshold_days: int) -> Set[str]: setups_to_check = {ZULIP_PATH} caches_in_use = set() diff --git a/scripts/lib/clean_node_cache.py b/scripts/lib/clean_node_cache.py index 208caef7d5..a3e1e95c20 100755 --- a/scripts/lib/clean_node_cache.py +++ b/scripts/lib/clean_node_cache.py @@ -23,8 +23,7 @@ if ENV == "travis": 'hence yarn is not installed. Exiting without cleaning npm cache.') sys.exit(0) -def get_caches_in_use(threshold_days): - # type: (int) -> Set[str] +def get_caches_in_use(threshold_days: int) -> Set[str]: setups_to_check = {ZULIP_PATH} caches_in_use = set() diff --git a/scripts/lib/clean_venv_cache.py b/scripts/lib/clean_venv_cache.py index 3238bc0615..37a9a063e8 100755 --- a/scripts/lib/clean_venv_cache.py +++ b/scripts/lib/clean_venv_cache.py @@ -18,8 +18,7 @@ VENV_CACHE_DIR = '/srv/zulip-venv-cache' if ENV == "travis": VENV_CACHE_DIR = os.path.join(os.environ["HOME"], "zulip-venv-cache") -def get_caches_in_use(threshold_days): - # type: (int) -> Set[str] +def get_caches_in_use(threshold_days: int) -> Set[str]: setups_to_check = {ZULIP_PATH} caches_in_use = set() diff --git a/scripts/lib/email-mirror-postfix b/scripts/lib/email-mirror-postfix index 25b56bce74..c8e404e118 100755 --- a/scripts/lib/email-mirror-postfix +++ b/scripts/lib/email-mirror-postfix @@ -82,8 +82,7 @@ options = parser.parse_args() MAX_ALLOWED_PAYLOAD = 25 * 1024 * 1024 -def process_response_error(e): - # type: (HTTPError) -> None +def process_response_error(e: HTTPError) -> None: if e.code == 400: response_content = e.read() response_data = json.loads(response_content.decode('utf8')) @@ -94,8 +93,9 @@ def process_response_error(e): exit(1) -def send_email_mirror(rcpt_to, shared_secret, host, url, test, verify_ssl): - # type: (str, str, str, str, bool, bool) -> None +def send_email_mirror( + rcpt_to: str, shared_secret: str, host: str, url: str, test: bool, verify_ssl: bool +) -> None: if not rcpt_to: print("5.1.1 Bad destination mailbox address: No missed message email address.") exit(posix.EX_NOUSER) diff --git a/scripts/lib/hash_reqs.py b/scripts/lib/hash_reqs.py index e7de7b4cb5..6115b5ef12 100755 --- a/scripts/lib/hash_reqs.py +++ b/scripts/lib/hash_reqs.py @@ -5,8 +5,7 @@ import argparse import hashlib from typing import Iterable, List, MutableSet -def expand_reqs_helper(fpath, visited): - # type: (str, MutableSet[str]) -> List[str] +def expand_reqs_helper(fpath: str, visited: MutableSet[str]) -> List[str]: if fpath in visited: return [] else: @@ -27,8 +26,7 @@ def expand_reqs_helper(fpath, visited): result.append(dep) return result -def expand_reqs(fpath): - # type: (str) -> List[str] +def expand_reqs(fpath: str) -> List[str]: """ Returns a sorted list of unique dependencies specified by the requirements file `fpath`. Removes comments from the output and recursively visits files specified inside `fpath`. @@ -38,13 +36,11 @@ def expand_reqs(fpath): output = expand_reqs_helper(absfpath, set()) return sorted(set(output)) -def hash_deps(deps): - # type: (Iterable[str]) -> str +def hash_deps(deps: Iterable[str]) -> str: deps_str = "\n".join(deps) + "\n" return hashlib.sha1(deps_str.encode('utf-8')).hexdigest() -def main(): - # type: () -> int +def main() -> int: description = ("Finds the SHA1 hash of list of dependencies in a requirements file" " after recursively visiting all files specified in it.") parser = argparse.ArgumentParser(description=description) diff --git a/scripts/lib/node_cache.py b/scripts/lib/node_cache.py index 74db364dcf..aa673a073b 100644 --- a/scripts/lib/node_cache.py +++ b/scripts/lib/node_cache.py @@ -20,16 +20,16 @@ YARN_PACKAGE_JSON = os.path.join(ZULIP_SRV_PATH, 'zulip-yarn/package.json') DEFAULT_PRODUCTION = False -def get_yarn_args(production): - # type: (bool) -> List[str] +def get_yarn_args(production: bool) -> List[str]: if production: yarn_args = ["--prod"] else: yarn_args = [] return yarn_args -def generate_sha1sum_node_modules(setup_dir=None, production=DEFAULT_PRODUCTION): - # type: (Optional[str], bool) -> str +def generate_sha1sum_node_modules( + setup_dir: Optional[str] = None, production: bool = DEFAULT_PRODUCTION +) -> str: if setup_dir is None: setup_dir = os.path.realpath(os.getcwd()) PACKAGE_JSON_FILE_PATH = os.path.join(setup_dir, 'package.json') @@ -47,9 +47,12 @@ def generate_sha1sum_node_modules(setup_dir=None, production=DEFAULT_PRODUCTION) sha1sum.update(''.join(sorted(yarn_args)).encode('utf8')) return sha1sum.hexdigest() -def setup_node_modules(production=DEFAULT_PRODUCTION, stdout=None, stderr=None, - prefer_offline=False): - # type: (bool, Optional[IO[Any]], Optional[IO[Any]], bool) -> None +def setup_node_modules( + production: bool = DEFAULT_PRODUCTION, + stdout: Optional[IO[Any]] = None, + stderr: Optional[IO[Any]] = None, + prefer_offline: bool = False, +) -> None: yarn_args = get_yarn_args(production=production) if prefer_offline: yarn_args.append("--prefer-offline") @@ -72,8 +75,13 @@ def setup_node_modules(production=DEFAULT_PRODUCTION, stdout=None, stderr=None, shutil.rmtree('node_modules') os.symlink(cached_node_modules, 'node_modules') -def do_yarn_install(target_path, yarn_args, success_stamp, stdout=None, stderr=None): - # type: (str, List[str], str, Optional[IO[Any]], Optional[IO[Any]]) -> None +def do_yarn_install( + target_path: str, + yarn_args: List[str], + success_stamp: str, + stdout: Optional[IO[Any]] = None, + stderr: Optional[IO[Any]] = None, +) -> None: os.makedirs(target_path, exist_ok=True) shutil.copy('package.json', target_path) shutil.copy("yarn.lock", target_path) diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index 235ba75d55..02d4836548 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -102,8 +102,7 @@ YUM_THUMBOR_VENV_DEPENDENCIES = [ "gifsicle", ] -def get_venv_dependencies(vendor, os_version): - # type: (str, str) -> List[str] +def get_venv_dependencies(vendor: str, os_version: str) -> List[str]: if vendor == 'ubuntu' and os_version == '20.04': return VENV_DEPENDENCIES + [PYTHON_DEV_DEPENDENCY.format("2"), ] elif "debian" in os_families(): @@ -115,18 +114,15 @@ def get_venv_dependencies(vendor, os_version): else: raise AssertionError("Invalid vendor") -def install_venv_deps(pip, requirements_file, python2): - # type: (str, str, bool) -> None +def install_venv_deps(pip: str, requirements_file: str, python2: bool) -> None: pip_requirements = os.path.join(ZULIP_PATH, "requirements", "pip2.txt" if python2 else "pip.txt") run([pip, "install", "--force-reinstall", "--require-hashes", "--requirement", pip_requirements]) run([pip, "install", "--no-deps", "--require-hashes", "--requirement", requirements_file]) -def get_index_filename(venv_path): - # type: (str) -> str +def get_index_filename(venv_path: str) -> str: return os.path.join(venv_path, 'package_index') -def get_package_names(requirements_file): - # type: (str) -> List[str] +def get_package_names(requirements_file: str) -> List[str]: packages = expand_reqs(requirements_file) cleaned = [] operators = ['~=', '==', '!=', '<', '>'] @@ -148,8 +144,7 @@ def get_package_names(requirements_file): return sorted(cleaned) -def create_requirements_index_file(venv_path, requirements_file): - # type: (str, str) -> str +def create_requirements_index_file(venv_path: str, requirements_file: str) -> str: """ Creates a file, called package_index, in the virtual environment directory that contains all the PIP packages installed in the @@ -164,8 +159,7 @@ def create_requirements_index_file(venv_path, requirements_file): return index_filename -def get_venv_packages(venv_path): - # type: (str) -> Set[str] +def get_venv_packages(venv_path: str) -> Set[str]: """ Returns the packages installed in the virtual environment using the package index file. @@ -173,8 +167,7 @@ def get_venv_packages(venv_path): with open(get_index_filename(venv_path)) as reader: return {p.strip() for p in reader.read().split('\n') if p.strip()} -def try_to_copy_venv(venv_path, new_packages): - # type: (str, Set[str]) -> bool +def try_to_copy_venv(venv_path: str, new_packages: Set[str]) -> bool: """ Tries to copy packages from an old virtual environment in the cache to the new virtual environment. The algorithm works as follows: @@ -247,12 +240,12 @@ def try_to_copy_venv(venv_path, new_packages): return False -def get_logfile_name(venv_path): - # type: (str) -> str +def get_logfile_name(venv_path: str) -> str: return "{}/setup-venv.log".format(venv_path) -def create_log_entry(target_log, parent, copied_packages, new_packages): - # type: (str, str, Set[str], Set[str]) -> None +def create_log_entry( + target_log: str, parent: str, copied_packages: Set[str], new_packages: Set[str] +) -> None: venv_path = os.path.dirname(target_log) with open(target_log, 'a') as writer: @@ -267,13 +260,11 @@ def create_log_entry(target_log, parent, copied_packages, new_packages): writer.write("\n".join('- {}'.format(p) for p in sorted(new_packages))) writer.write("\n\n") -def copy_parent_log(source_log, target_log): - # type: (str, str) -> None +def copy_parent_log(source_log: str, target_log: str) -> None: if os.path.exists(source_log): shutil.copyfile(source_log, target_log) -def do_patch_activate_script(venv_path): - # type: (str) -> None +def do_patch_activate_script(venv_path: str) -> None: """ Patches the bin/activate script so that the value of the environment variable VIRTUAL_ENV is set to venv_path during the script's execution whenever it is sourced. @@ -290,8 +281,12 @@ def do_patch_activate_script(venv_path): with open(script_path, 'w') as f: f.write("".join(lines)) -def setup_virtualenv(target_venv_path, requirements_file, python2=False, patch_activate_script=False): - # type: (Optional[str], str, bool, bool) -> str +def setup_virtualenv( + target_venv_path: Optional[str], + requirements_file: str, + python2: bool = False, + patch_activate_script: bool = False, +) -> str: # Check if a cached version already exists path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py') @@ -314,15 +309,13 @@ def setup_virtualenv(target_venv_path, requirements_file, python2=False, patch_a do_patch_activate_script(target_venv_path) return cached_venv_path -def add_cert_to_pipconf(): - # type: () -> None +def add_cert_to_pipconf() -> None: conffile = os.path.expanduser("~/.pip/pip.conf") confdir = os.path.expanduser("~/.pip/") os.makedirs(confdir, exist_ok=True) run(["crudini", "--set", conffile, "global", "cert", os.environ["CUSTOM_CA_CERTIFICATES"]]) -def do_setup_virtualenv(venv_path, requirements_file, python2): - # type: (str, str, bool) -> None +def do_setup_virtualenv(venv_path: str, requirements_file: str, python2: bool) -> None: # Setup Python virtualenv new_packages = set(get_package_names(requirements_file)) diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py index 07a7fd94d8..297e2db90a 100755 --- a/scripts/lib/zulip_tools.py +++ b/scripts/lib/zulip_tools.py @@ -39,8 +39,7 @@ BLUE = '\x1b[34m' MAGENTA = '\x1b[35m' CYAN = '\x1b[36m' -def overwrite_symlink(src, dst): - # type: (str, str) -> None +def overwrite_symlink(src: str, dst: str) -> None: while True: tmp = tempfile.mktemp( prefix='.' + os.path.basename(dst) + '.', @@ -56,8 +55,7 @@ def overwrite_symlink(src, dst): os.remove(tmp) raise -def parse_cache_script_args(description): - # type: (str) -> argparse.Namespace +def parse_cache_script_args(description: str) -> argparse.Namespace: parser = argparse.ArgumentParser(description=description) parser.add_argument( @@ -88,8 +86,7 @@ def get_deploy_root() -> str: os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) ) -def get_deployment_version(extract_path): - # type: (str) -> str +def get_deployment_version(extract_path: str) -> str: version = '0.0.0' for item in os.listdir(extract_path): item_path = os.path.join(extract_path, item) @@ -101,14 +98,12 @@ def get_deployment_version(extract_path): break return version -def is_invalid_upgrade(current_version, new_version): - # type: (str, str) -> bool +def is_invalid_upgrade(current_version: str, new_version: str) -> bool: if new_version > '1.4.3' and current_version <= '1.3.10': return True return False -def subprocess_text_output(args): - # type: (Sequence[str]) -> str +def subprocess_text_output(args: Sequence[str]) -> str: return subprocess.check_output(args, universal_newlines=True).strip() def get_zulip_pwent() -> pwd.struct_passwd: @@ -121,8 +116,7 @@ def get_zulip_pwent() -> pwd.struct_passwd: # `zulip` user as that's the correct value in production. return pwd.getpwnam("zulip") -def su_to_zulip(save_suid=False): - # type: (bool) -> None +def su_to_zulip(save_suid: bool = False) -> None: """Warning: su_to_zulip assumes that the zulip checkout is owned by the zulip user (or whatever normal user is running the Zulip installation). It should never be run from the installer or other @@ -136,14 +130,12 @@ def su_to_zulip(save_suid=False): os.setuid(pwent.pw_uid) os.environ['HOME'] = pwent.pw_dir -def make_deploy_path(): - # type: () -> str +def make_deploy_path() -> str: timestamp = datetime.datetime.now().strftime(TIMESTAMP_FORMAT) return os.path.join(DEPLOYMENTS_DIR, timestamp) TEMPLATE_DATABASE_DIR = "test-backend/databases" -def get_dev_uuid_var_path(create_if_missing=False): - # type: (bool) -> str +def get_dev_uuid_var_path(create_if_missing: bool = False) -> str: zulip_path = get_deploy_root() uuid_path = os.path.join(os.path.realpath(os.path.dirname(zulip_path)), ".zulip-dev-uuid") if os.path.exists(uuid_path): @@ -163,8 +155,7 @@ def get_dev_uuid_var_path(create_if_missing=False): os.makedirs(result_path, exist_ok=True) return result_path -def get_deployment_lock(error_rerun_script): - # type: (str) -> None +def get_deployment_lock(error_rerun_script: str) -> None: start_time = time.time() got_lock = False while time.time() - start_time < 300: @@ -187,12 +178,10 @@ def get_deployment_lock(error_rerun_script): ENDC) sys.exit(1) -def release_deployment_lock(): - # type: () -> None +def release_deployment_lock() -> None: shutil.rmtree(LOCK_DIR) -def run(args, **kwargs): - # type: (Sequence[str], **Any) -> None +def run(args: Sequence[str], **kwargs: Any) -> None: # Output what we're doing in the `set -x` style print("+ %s" % (" ".join(map(shlex.quote, args)),)) @@ -208,8 +197,7 @@ def run(args, **kwargs): print() raise -def log_management_command(cmd, log_path): - # type: (str, str) -> None +def log_management_command(cmd: str, log_path: str) -> None: log_dir = os.path.dirname(log_path) if not os.path.exists(log_dir): os.makedirs(log_dir) @@ -223,16 +211,14 @@ def log_management_command(cmd, log_path): logger.info("Ran '%s'" % (cmd,)) -def get_environment(): - # type: () -> str +def get_environment() -> str: if os.path.exists(DEPLOYMENTS_DIR): return "prod" if os.environ.get("TRAVIS"): return "travis" return "dev" -def get_recent_deployments(threshold_days): - # type: (int) -> Set[str] +def get_recent_deployments(threshold_days: int) -> Set[str]: # Returns a list of deployments not older than threshold days # including `/root/zulip` directory if it exists. recent = set() @@ -259,16 +245,14 @@ def get_recent_deployments(threshold_days): recent.add("/root/zulip") return recent -def get_threshold_timestamp(threshold_days): - # type: (int) -> int +def get_threshold_timestamp(threshold_days: int) -> int: # Given number of days, this function returns timestamp corresponding # to the time prior to given number of days. threshold = datetime.datetime.now() - datetime.timedelta(days=threshold_days) threshold_timestamp = int(time.mktime(threshold.utctimetuple())) return threshold_timestamp -def get_caches_to_be_purged(caches_dir, caches_in_use, threshold_days): - # type: (str, Set[str], int) -> Set[str] +def get_caches_to_be_purged(caches_dir: str, caches_in_use: Set[str], threshold_days: int) -> Set[str]: # Given a directory containing caches, a list of caches in use # and threshold days, this function return a list of caches # which can be purged. Remove the cache only if it is: @@ -287,8 +271,9 @@ def get_caches_to_be_purged(caches_dir, caches_in_use, threshold_days): caches_to_purge.add(cache_dir) return caches_to_purge -def purge_unused_caches(caches_dir, caches_in_use, cache_type, args): - # type: (str, Set[str], str, argparse.Namespace) -> None +def purge_unused_caches( + caches_dir: str, caches_in_use: Set[str], cache_type: str, args: argparse.Namespace +) -> None: all_caches = {os.path.join(caches_dir, cache) for cache in os.listdir(caches_dir)} caches_to_purge = get_caches_to_be_purged(caches_dir, caches_in_use, args.threshold_days) caches_to_keep = all_caches - caches_to_purge @@ -298,8 +283,7 @@ def purge_unused_caches(caches_dir, caches_in_use, cache_type, args): if args.verbose: print("Done!") -def generate_sha1sum_emoji(zulip_path): - # type: (str) -> str +def generate_sha1sum_emoji(zulip_path: str) -> str: ZULIP_EMOJI_DIR = os.path.join(zulip_path, 'tools', 'setup', 'emoji') sha = hashlib.sha1() @@ -332,8 +316,14 @@ def generate_sha1sum_emoji(zulip_path): return sha.hexdigest() -def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbose, no_headings): - # type: (Set[str], Set[str], str, bool, bool, bool) -> None +def may_be_perform_purging( + dirs_to_purge: Set[str], + dirs_to_keep: Set[str], + dir_type: str, + dry_run: bool, + verbose: bool, + no_headings: bool, +) -> None: if dry_run: print("Performing a dry run...") if not no_headings: @@ -350,8 +340,7 @@ def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbo print("Keeping used %s: %s" % (dir_type, directory)) @functools.lru_cache(None) -def parse_os_release(): - # type: () -> Dict[str, str] +def parse_os_release() -> Dict[str, str]: """ Example of the useful subset of the data: { @@ -423,8 +412,7 @@ def is_root() -> bool: return True return False -def run_as_root(args, **kwargs): - # type: (List[str], **Any) -> None +def run_as_root(args: List[str], **kwargs: Any) -> None: sudo_args = kwargs.pop('sudo_args', []) if not is_root(): args = ['sudo'] + sudo_args + ['--'] + args @@ -454,8 +442,12 @@ def assert_running_as_root(strip_lib_from_paths: bool=False) -> None: print("{} must be run as root.".format(script_name)) sys.exit(1) -def get_config(config_file, section, key, default_value=""): - # type: (configparser.RawConfigParser, str, str, str) -> str +def get_config( + config_file: configparser.RawConfigParser, + section: str, + key: str, + default_value: str = "", +) -> str: if config_file.has_option(section, key): return config_file.get(section, key) return default_value @@ -465,8 +457,7 @@ def get_config_file() -> configparser.RawConfigParser: config_file.read("/etc/zulip/zulip.conf") return config_file -def get_deploy_options(config_file): - # type: (configparser.RawConfigParser) -> List[str] +def get_deploy_options(config_file: configparser.RawConfigParser) -> List[str]: return get_config(config_file, 'deployment', 'deploy_options', "").strip().split() def get_or_create_dev_uuid_var_path(path: str) -> str: diff --git a/scripts/nagios/check-rabbitmq-consumers b/scripts/nagios/check-rabbitmq-consumers index 6e89315f6f..4a4c4e208f 100755 --- a/scripts/nagios/check-rabbitmq-consumers +++ b/scripts/nagios/check-rabbitmq-consumers @@ -37,8 +37,7 @@ options = parser.parse_args() config_file = configparser.RawConfigParser() config_file.read("/etc/zulip/zulip.conf") -def get_config(section, key, default_value): - # type: (str, str, str) -> str +def get_config(section: str, key: str, default_value: str) -> str: if config_file.has_option(section, key): return config_file.get(section, key) return default_value diff --git a/scripts/nagios/cron_file_helper.py b/scripts/nagios/cron_file_helper.py index 540267e568..e71e8218f6 100644 --- a/scripts/nagios/cron_file_helper.py +++ b/scripts/nagios/cron_file_helper.py @@ -2,8 +2,7 @@ import time from typing import Tuple -def nagios_from_file(results_file): - # type: (str) -> Tuple[int, str] +def nagios_from_file(results_file: str) -> Tuple[int, str]: """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/scripts/purge-old-deployments b/scripts/purge-old-deployments index b003eaa8d4..f82759c94b 100755 --- a/scripts/purge-old-deployments +++ b/scripts/purge-old-deployments @@ -11,8 +11,7 @@ sys.path.append(ZULIP_PATH) from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, \ may_be_perform_purging -def parse_args(): - # type: () -> argparse.Namespace +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="This script can be used for cleaning old unused deployments.", epilog="Orphaned/unused caches older than threshold days will be automatically " @@ -34,8 +33,7 @@ def parse_args(): args.verbose |= args.dry_run # Always print a detailed report in case of dry run. return args -def get_deployments_to_be_purged(recent_deployments): - # type: (Set[str]) -> Set[str] +def get_deployments_to_be_purged(recent_deployments: Set[str]) -> Set[str]: all_deployments = {os.path.join(DEPLOYMENTS_DIR, deployment) for deployment in os.listdir(DEPLOYMENTS_DIR)} deployments_to_purge = set() @@ -52,8 +50,7 @@ def get_deployments_to_be_purged(recent_deployments): deployments_to_purge.add(deployment) return deployments_to_purge -def main(): - # type: () -> None +def main() -> None: args = parse_args() deployments_to_keep = get_recent_deployments(args.threshold_days) deployments_to_purge = get_deployments_to_be_purged(deployments_to_keep) diff --git a/scripts/setup/generate_secrets.py b/scripts/setup/generate_secrets.py index bd64d11f85..68bd0e52b8 100755 --- a/scripts/setup/generate_secrets.py +++ b/scripts/setup/generate_secrets.py @@ -31,14 +31,12 @@ AUTOGENERATED_SETTINGS = [ 'thumbor_key', ] -def generate_django_secretkey(): - # type: () -> str +def generate_django_secretkey() -> str: """Secret key generation taken from Django's startproject.py""" chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' return get_random_string(50, chars) -def get_old_conf(output_filename): - # type: (str) -> Dict[str, str] +def get_old_conf(output_filename: str) -> Dict[str, str]: if not os.path.exists(output_filename) or os.path.getsize(output_filename) == 0: return {} @@ -47,8 +45,7 @@ def get_old_conf(output_filename): return dict(secrets_file.items("secrets")) -def generate_secrets(development=False): - # type: (bool) -> None +def generate_secrets(development: bool = False) -> None: if development: OUTPUT_SETTINGS_FILENAME = "zproject/dev-secrets.conf" else: @@ -59,12 +56,10 @@ def generate_secrets(development=False): if len(current_conf) == 0: lines = ['[secrets]\n'] - def need_secret(name): - # type: (str) -> bool + def need_secret(name: str) -> bool: return name not in current_conf - def add_secret(name, value): - # type: (str, str) -> None + def add_secret(name: str, value: str) -> None: lines.append("%s = %s\n" % (name, value)) current_conf[name] = value diff --git a/scripts/setup/restore-backup b/scripts/setup/restore-backup index 028a9ffc5f..f8eae9e31f 100755 --- a/scripts/setup/restore-backup +++ b/scripts/setup/restore-backup @@ -19,8 +19,7 @@ parser = argparse.ArgumentParser() parser.add_argument("tarball", help="Filename of input tarball") -def restore_backup(tarball_file): - # type: (IO[bytes]) -> None +def restore_backup(tarball_file: IO[bytes]) -> None: su_to_zulip(save_suid=True) diff --git a/tools/check-frontend-i18n b/tools/check-frontend-i18n index 3201cc2581..24c40a70da 100755 --- a/tools/check-frontend-i18n +++ b/tools/check-frontend-i18n @@ -14,8 +14,7 @@ import subprocess sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from scripts.lib.zulip_tools import WARNING, FAIL, ENDC -def find_handlebars(translatable_strings): - # type: (List[str]) -> List[str] +def find_handlebars(translatable_strings: List[str]) -> List[str]: errored = [] for string in translatable_strings: if '{{' in string: diff --git a/tools/check-issue-labels b/tools/check-issue-labels index c40465dbf9..8d0c5fb4e6 100755 --- a/tools/check-issue-labels +++ b/tools/check-issue-labels @@ -21,34 +21,29 @@ from typing import Any, Dict, Optional # usage: python check-issue-labels # Pass --force as an argument to run without a token. -def get_config(): - # type: () -> ConfigParser +def get_config() -> ConfigParser: config = ConfigParser() config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf.ini')) return config -def area_labeled(issue): - # type: (Dict[str, Any]) -> bool +def area_labeled(issue: Dict[str, Any]) -> bool: for label in issue["labels"]: label_name = str(label["name"]) if "area:" in label_name: return True return False -def is_issue(item): - # type: (Dict[str, Any]) -> bool +def is_issue(item: Dict[str, Any]) -> bool: return "issues" in item["html_url"] -def get_next_page_url(link_header): - # type: (str) -> Optional[str] +def get_next_page_url(link_header: str) -> Optional[str]: matches = re.findall(r'\<(\S+)\>; rel=\"next\"', link_header) try: return matches[0] except IndexError: return None -def check_issue_labels(): - # type: () -> None +def check_issue_labels() -> None: parser = argparse.ArgumentParser() parser.add_argument('--force', action="store_true", dest="force", default=False) args = parser.parse_args() diff --git a/tools/check-provision b/tools/check-provision index 72da67e780..41d5a5bcbb 100755 --- a/tools/check-provision +++ b/tools/check-provision @@ -12,8 +12,7 @@ from tools.lib.test_script import ( assert_provisioning_status_ok, ) -def run(): - # type: () -> None +def run() -> None: parser = argparse.ArgumentParser() parser.add_argument('--force', default=False, action="store_true", diff --git a/tools/check-templates b/tools/check-templates index cc157f41f5..9d768a4986 100755 --- a/tools/check-templates +++ b/tools/check-templates @@ -22,8 +22,7 @@ EXCLUDED_FILES = [ 'static/assets/icons/template.hbs', ] -def check_our_files(modified_only, all_dups, fix, targets): - # type: (bool, bool, bool, List[str]) -> None +def check_our_files(modified_only: bool, all_dups: bool, fix: bool, targets: List[str]) -> None: by_lang = lister.list_files( targets=targets, modified_only=args.modified, @@ -34,8 +33,7 @@ def check_our_files(modified_only, all_dups, fix, targets): check_handlebar_templates(by_lang['hbs'], fix) check_html_templates(by_lang['html'], all_dups, fix) -def check_html_templates(templates, all_dups, fix): - # type: (Iterable[str], bool, bool) -> None +def check_html_templates(templates: Iterable[str], all_dups: bool, fix: bool) -> None: # Our files with .html extensions are usually for Django, but we also # have a few static .html files. # @@ -143,8 +141,7 @@ def check_html_templates(templates, all_dups, fix): if not validate_indent_html(fn, fix): sys.exit(1) -def check_handlebar_templates(templates, fix): - # type: (Iterable[str], bool) -> None +def check_handlebar_templates(templates: Iterable[str], fix: bool) -> None: # Check all our handlebars templates. templates = [fn for fn in templates if fn.endswith('.hbs')] diff --git a/tools/create-test-api-docs b/tools/create-test-api-docs index 47405bb7ff..937be8a58a 100755 --- a/tools/create-test-api-docs +++ b/tools/create-test-api-docs @@ -14,8 +14,7 @@ import ujson Call = Dict[str, Any] -def clean_up_pattern(s): - # type: (str) -> str +def clean_up_pattern(s: str) -> str: paren_level = 0 in_braces = False result = '' @@ -35,8 +34,7 @@ def clean_up_pattern(s): prior_char = c return result -def encode_info(info): - # type: (Any) -> str +def encode_info(info: Any) -> str: try: result = '' try: @@ -54,12 +52,10 @@ def encode_info(info): pass return 'NOT ENCODABLE' -def fix_test_name(s): - # type: (str) -> str +def fix_test_name(s: str) -> str: return s.replace('zerver.tests.', '') -def create_single_page(pattern, out_dir, href, calls): - # type: (str, str, str, List[Call]) -> None +def create_single_page(pattern: str, out_dir: str, href: str, calls: List[Call]) -> None: fn = out_dir + '/' + href with open(fn, 'w') as f: f.write(''' @@ -85,8 +81,7 @@ def create_single_page(pattern, out_dir, href, calls): f.write('
') f.write('') -def create_user_docs(): - # type: () -> None +def create_user_docs() -> None: fn = 'var/url_coverage.txt' # TODO: make path more robust, maybe use json suffix out_dir = 'var/api_docs' diff --git a/tools/diagnose b/tools/diagnose index 37c2aa75d3..657ab87520 100755 --- a/tools/diagnose +++ b/tools/diagnose @@ -14,8 +14,7 @@ sys.path.insert(0, ROOT_DIR) from scripts.lib.zulip_tools import get_dev_uuid_var_path UUID_VAR_PATH = get_dev_uuid_var_path() -def run(check_func): - # type: (Callable[[], bool]) -> None +def run(check_func: Callable[[], bool]) -> None: ''' This decorator simply runs functions. It makes it more convenient to add new checks without a big main() function. @@ -24,32 +23,27 @@ def run(check_func): if not rc: sys.exit(1) -def run_command(args): - # type: (List[str]) -> None +def run_command(args: List[str]) -> None: print(' '.join(map(shlex.quote, args))) subprocess.check_call(args) @run -def check_python_version(): - # type: () -> bool +def check_python_version() -> bool: subprocess.check_call(['/usr/bin/env', 'python', '-V']) return True @run -def pwd(): - # type: () -> bool +def pwd() -> bool: print(os.getcwd()) return True @run -def host_info(): - # type: () -> bool +def host_info() -> bool: print(platform.platform()) return True @run -def check_django(): - # type: () -> bool +def check_django() -> bool: try: import django print('Django version:', django.get_version()) @@ -70,8 +64,7 @@ def check_django(): return False @run -def provision_version(): - # type: () -> bool +def provision_version() -> bool: fn = os.path.join(UUID_VAR_PATH, 'provision_version') with open(fn) as f: version = f.read().strip() @@ -84,15 +77,13 @@ def provision_version(): return True @run -def node_stuff(): - # type: () -> bool +def node_stuff() -> bool: print('node version:') subprocess.check_call(['node', '--version']) return True @run -def test_models(): - # type: () -> bool +def test_models() -> bool: settings_module = "zproject.settings" os.environ['DJANGO_SETTINGS_MODULE'] = settings_module import django @@ -103,8 +94,7 @@ def test_models(): return True @run -def check_venv(): - # type: () -> bool +def check_venv() -> bool: path = os.path.join(ROOT_DIR, 'scripts', 'lib', 'hash_reqs.py') cache_dir = '/srv/zulip-venv-cache/' for fn in ['dev.txt']: @@ -118,8 +108,7 @@ def check_venv(): return True @run -def check_migrations(): - # type: () -> bool +def check_migrations() -> bool: print() rc = subprocess.check_call('./tools/test-migrations') return (rc == 0) diff --git a/tools/droplets/create.py b/tools/droplets/create.py index 3473b5b37b..8fc89fbaef 100644 --- a/tools/droplets/create.py +++ b/tools/droplets/create.py @@ -33,14 +33,12 @@ parser.add_argument("username", help="Github username for whom you want to creat parser.add_argument('--tags', nargs='+', default=[]) parser.add_argument('-f', '--recreate', dest='recreate', action="store_true", default=False) -def get_config(): - # type: () -> configparser.ConfigParser +def get_config() -> configparser.ConfigParser: config = configparser.ConfigParser() config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf.ini')) return config -def user_exists(username): - # type: (str) -> bool +def user_exists(username: str) -> bool: print("Checking to see if GitHub user {} exists...".format(username)) user_api_url = "https://api.github.com/users/{}".format(username) try: @@ -53,8 +51,7 @@ def user_exists(username): print("Does the github user {} exist?".format(username)) sys.exit(1) -def get_keys(username): - # type: (str) -> List[Dict[str, Any]] +def get_keys(username: str) -> List[Dict[str, Any]]: print("Checking to see that GitHub user has available public keys...") apiurl_keys = "https://api.github.com/users/{}/keys".format(username) try: @@ -70,8 +67,7 @@ def get_keys(username): print("Has user {} added ssh keys to their github account?".format(username)) sys.exit(1) -def fork_exists(username): - # type: (str) -> bool +def fork_exists(username: str) -> bool: print("Checking to see GitHub user has forked zulip/zulip...") apiurl_fork = "https://api.github.com/repos/{}/zulip".format(username) try: @@ -100,8 +96,7 @@ def exit_if_droplet_exists(my_token: str, username: str, recreate: bool) -> None return print("...No droplet found...proceeding.") -def set_user_data(username, userkeys): - # type: (str, List[Dict[str, Any]]) -> str +def set_user_data(username: str, userkeys: List[Dict[str, Any]]) -> str: print("Setting cloud-config data, populated with GitHub user's public keys...") ssh_authorized_keys = "" @@ -137,8 +132,7 @@ cd /home/zulipdev/{1} && git remote add origin https://github.com/{0}/{1}.git && print("...returning cloud-config data.") return cloudconf -def create_droplet(my_token, template_id, username, tags, user_data): - # type: (str, str, str, List[str], str) -> str +def create_droplet(my_token: str, template_id: str, username: str, tags: List[str], user_data: str) -> str: droplet = digitalocean.Droplet( token=my_token, name='{}.zulipdev.org'.format(username), @@ -177,8 +171,7 @@ def delete_existing_records(records: List[digitalocean.Record], record_name: str if count: print("Deleted {} existing A records for {}.zulipdev.org.".format(count, record_name)) -def create_dns_record(my_token, username, ip_address): - # type: (str, str, str) -> None +def create_dns_record(my_token: str, username: str, ip_address: str) -> None: domain = digitalocean.Domain(token=my_token, name='zulipdev.org') domain.load() records = domain.get_records() @@ -192,8 +185,7 @@ def create_dns_record(my_token, username, ip_address): print("Creating new A record for *.{}.zulipdev.org that points to {}.".format(username, ip_address)) domain.create_new_domain_record(type='A', name=wildcard_name, data=ip_address) -def print_completion(username): - # type: (str) -> None +def print_completion(username: str) -> None: print(""" COMPLETE! Droplet for GitHub user {0} is available at {0}.zulipdev.org. diff --git a/tools/get-handlebar-vars b/tools/get-handlebar-vars index 8d711a8c27..b6c87264fd 100755 --- a/tools/get-handlebar-vars +++ b/tools/get-handlebar-vars @@ -9,12 +9,10 @@ sanity_check.check_venv(__file__) from typing import Any, Dict, List -def debug(obj): - # type: (Any) -> None +def debug(obj: Any) -> None: print(json.dumps(obj, indent=4)) -def parse_file(fn): - # type: (str) -> Dict[str, Any] +def parse_file(fn: str) -> Dict[str, Any]: with open(fn) as f: text = f.read() tags = re.findall(r'{+\s*(.*?)\s*}+', text) @@ -22,8 +20,7 @@ def parse_file(fn): context = root stack = [] # type: List[Dict[str, Any]] - def set_var(var, val): - # type: (str, Any) -> None + def set_var(var: str, val: Any) -> None: num_levels_up = len(re.findall(r'\.\.', var)) if num_levels_up: var = var.split('/')[-1] @@ -95,8 +92,7 @@ def parse_file(fn): set_var(tag, '') - def clean_this(obj): - # type: (Any) -> Any + def clean_this(obj: Any) -> Any: if isinstance(obj, list): return [clean_this(item) for item in obj] if isinstance(obj, dict): diff --git a/tools/html-grep b/tools/html-grep index 92adb24ff6..69040d8967 100755 --- a/tools/html-grep +++ b/tools/html-grep @@ -46,8 +46,7 @@ USAGE = ''' TODO: allow specific files to be searched.''' -def check_our_files(): - # type: () -> None +def check_our_files() -> None: parser = argparse.ArgumentParser(description=USAGE, formatter_class=argparse.RawTextHelpFormatter) diff --git a/tools/js-dep-visualizer.py b/tools/js-dep-visualizer.py index be44ecf6ec..1122f7f60f 100755 --- a/tools/js-dep-visualizer.py +++ b/tools/js-dep-visualizer.py @@ -31,8 +31,7 @@ JS_FILES_DIR = os.path.join(ROOT_DIR, 'static/js') OUTPUT_FILE_PATH = os.path.relpath(os.path.join(ROOT_DIR, 'var/zulip-deps.dot')) PNG_FILE_PATH = os.path.relpath(os.path.join(ROOT_DIR, 'var/zulip-deps.png')) -def get_js_edges(): - # type: () -> Tuple[EdgeSet, MethodDict] +def get_js_edges() -> Tuple[EdgeSet, MethodDict]: names = set() modules = [] # type: List[Dict[str, Any]] for js_file in os.listdir(JS_FILES_DIR): @@ -75,8 +74,7 @@ def get_js_edges(): methods[tup].append(method) return edges, methods -def find_edges_to_remove(graph, methods): - # type: (Graph, MethodDict) -> Tuple[Graph, List[Edge]] +def find_edges_to_remove(graph: Graph, methods: MethodDict) -> Tuple[Graph, List[Edge]]: EXEMPT_EDGES = [ # These are sensible dependencies, so don't cut them. ('rows', 'message_store'), @@ -144,8 +142,7 @@ def find_edges_to_remove(graph, methods): ('message_edit', 'resize'), ] # type: List[Edge] - def is_exempt(edge): - # type: (Tuple[str, str]) -> bool + def is_exempt(edge: Tuple[str, str]) -> bool: parent, child = edge if edge == ('server_events', 'reload'): return False @@ -223,8 +220,7 @@ def find_edges_to_remove(graph, methods): ('emoji_picker', 'reactions'), ] - def cut_is_legal(edge): - # type: (Edge) -> bool + def cut_is_legal(edge: Edge) -> bool: parent, child = edge if child in ['reload', 'popovers', 'overlays', 'notifications', 'server_events', 'compose_actions']: @@ -255,8 +251,7 @@ def find_edges_to_remove(graph, methods): return graph, removed_edges -def report_roadmap(edges, methods): - # type: (List[Edge], MethodDict) -> None +def report_roadmap(edges: List[Edge], methods: MethodDict) -> None: child_modules = {child for parent, child in edges} module_methods = defaultdict(set) # type: DefaultDict[str, Set[str]] callers = defaultdict(set) # type: DefaultDict[Tuple[str, str], Set[str]] @@ -277,8 +272,7 @@ def report_roadmap(edges, methods): print() print() -def produce_partial_output(graph): - # type: (Graph) -> None +def produce_partial_output(graph: Graph) -> None: print(graph.num_edges()) buffer = make_dot_file(graph) @@ -290,8 +284,7 @@ def produce_partial_output(graph): print('See dot file here: {}'.format(OUTPUT_FILE_PATH)) print('See output png file: {}'.format(PNG_FILE_PATH)) -def run(): - # type: () -> None +def run() -> None: edges, methods = get_js_edges() graph = Graph(edges) graph, removed_edges = find_edges_to_remove(graph, methods) diff --git a/tools/lib/capitalization.py b/tools/lib/capitalization.py index fd71d81a6a..320e0ed83c 100644 --- a/tools/lib/capitalization.py +++ b/tools/lib/capitalization.py @@ -180,8 +180,7 @@ BANNED_WORDS = { 'Use organization instead.'), } -def get_safe_phrase(phrase): - # type: (str) -> str +def get_safe_phrase(phrase: str) -> str: """ Safe phrase is in lower case and doesn't contain characters which can conflict with split boundaries. All conflicting characters are replaced @@ -190,8 +189,7 @@ def get_safe_phrase(phrase): phrase = SPLIT_BOUNDARY_REGEX.sub('_', phrase) return phrase.lower() -def replace_with_safe_phrase(matchobj): - # type: (Match[str]) -> str +def replace_with_safe_phrase(matchobj: Match[str]) -> str: """ The idea is to convert IGNORED_PHRASES into safe phrases, see `get_safe_phrase()` function. The only exception is when the @@ -215,8 +213,7 @@ def replace_with_safe_phrase(matchobj): return safe_string -def get_safe_text(text): - # type: (str) -> str +def get_safe_text(text: str) -> str: """ This returns text which is rendered by BeautifulSoup and is in the form that can be split easily and has all IGNORED_PHRASES processed. @@ -228,8 +225,7 @@ def get_safe_text(text): return text -def is_capitalized(safe_text): - # type: (str) -> bool +def is_capitalized(safe_text: str) -> bool: sentences = SPLIT_BOUNDARY_REGEX.split(safe_text) sentences = [sentence.strip() for sentence in sentences if sentence.strip()] @@ -259,8 +255,7 @@ def check_banned_words(text: str) -> List[str]: return errors -def check_capitalization(strings): - # type: (List[str]) -> Tuple[List[str], List[str], List[str]] +def check_capitalization(strings: List[str]) -> Tuple[List[str], List[str], List[str]]: errors = [] ignored = [] banned_word_errors = [] diff --git a/tools/lib/gitlint-rules.py b/tools/lib/gitlint-rules.py index f7d36e05da..c9225ab6f0 100644 --- a/tools/lib/gitlint-rules.py +++ b/tools/lib/gitlint-rules.py @@ -84,8 +84,7 @@ imperative_forms = sorted([ ]) -def head_binary_search(key, words): - # type: (Text, List[str]) -> str +def head_binary_search(key: Text, words: List[str]) -> str: """ Find the imperative mood version of `word` by looking at the first 3 characters. """ @@ -124,8 +123,7 @@ class ImperativeMood(LineRule): error_msg = ('The first word in commit title should be in imperative mood ' '("{word}" -> "{imperative}"): "{title}"') - def validate(self, line, commit): - # type: (Text, gitlint.commit) -> List[RuleViolation] + def validate(self, line: Text, commit: gitlint.commit) -> List[RuleViolation]: violations = [] # Ignore the section tag (ie `
: .`) @@ -153,8 +151,7 @@ class TitleMatchRegexAllowException(LineRule): target = CommitMessageTitle options_spec = [StrOption('regex', ".*", "Regex the title should match")] - def validate(self, title, commit): - # type: (Text, gitlint.commit) -> List[RuleViolation] + def validate(self, title: Text, commit: gitlint.commit) -> List[RuleViolation]: regex = self.options['regex'].value pattern = re.compile(regex, re.UNICODE) diff --git a/tools/lib/graph.py b/tools/lib/graph.py index 762dbbdf67..e682256bc4 100644 --- a/tools/lib/graph.py +++ b/tools/lib/graph.py @@ -6,8 +6,7 @@ Edge = Tuple[str, str] EdgeSet = Set[Edge] class Graph: - def __init__(self, tuples): - # type: (EdgeSet) -> None + def __init__(self, tuples: EdgeSet) -> None: self.children = defaultdict(list) # type: DefaultDict[str, List[str]] self.parents = defaultdict(list) # type: DefaultDict[str, List[str]] self.nodes = set() # type: Set[str] @@ -18,30 +17,25 @@ class Graph: self.nodes.add(parent) self.nodes.add(child) - def copy(self): - # type: () -> 'Graph' + def copy(self) -> 'Graph': return Graph(self.edges()) - def num_edges(self): - # type: () -> int + def num_edges(self) -> int: return len(self.edges()) - def minus_edge(self, edge): - # type: (Edge) -> 'Graph' + def minus_edge(self, edge: Edge) -> 'Graph': edges = self.edges().copy() edges.remove(edge) return Graph(edges) - def edges(self): - # type: () -> EdgeSet + def edges(self) -> EdgeSet: s = set() for parent in self.nodes: for child in self.children[parent]: s.add((parent, child)) return s - def remove_exterior_nodes(self): - # type: () -> None + def remove_exterior_nodes(self) -> None: still_work_to_do = True while still_work_to_do: still_work_to_do = False # for now @@ -51,8 +45,7 @@ class Graph: still_work_to_do = True break - def is_exterior_node(self, node): - # type: (str) -> bool + def is_exterior_node(self, node: str) -> bool: parents = self.parents[node] children = self.children[node] if not parents: @@ -66,16 +59,14 @@ class Graph: # effectively be collapsed into the parent, so don't add clutter. return parents[0] == children[0] - def remove(self, node): - # type: (str) -> None + def remove(self, node: str) -> None: for parent in self.parents[node]: self.children[parent].remove(node) for child in self.children[node]: self.parents[child].remove(node) self.nodes.remove(node) - def report(self): - # type: () -> None + def report(self) -> None: print('parents/children/module') tups = sorted([ (len(self.parents[node]), len(self.children[node]), node) @@ -83,14 +74,12 @@ class Graph: for tup in tups: print(tup) -def best_edge_to_remove(orig_graph, is_exempt): - # type: (Graph, Callable[[Edge], bool]) -> Optional[Edge] +def best_edge_to_remove(orig_graph: Graph, is_exempt: Callable[[Edge], bool]) -> Optional[Edge]: # expects an already reduced graph as input orig_edges = orig_graph.edges() - def get_choices(): - # type: () -> Iterator[Tuple[int, Edge]] + def get_choices() -> Iterator[Tuple[int, Edge]]: for edge in orig_edges: if is_exempt(edge): continue @@ -107,8 +96,7 @@ def best_edge_to_remove(orig_graph, is_exempt): raise Exception('no edges work here') return best_edge -def make_dot_file(graph): - # type: (Graph) -> str +def make_dot_file(graph: Graph) -> str: buffer = 'digraph G {\n' for node in graph.nodes: buffer += node + ';\n' @@ -117,8 +105,7 @@ def make_dot_file(graph): buffer += '}' return buffer -def test(): - # type: () -> None +def test() -> None: graph = Graph({ ('x', 'a'), ('a', 'b'), diff --git a/tools/lib/html_branches.py b/tools/lib/html_branches.py index 02a98a593e..fe6f616b40 100644 --- a/tools/lib/html_branches.py +++ b/tools/lib/html_branches.py @@ -21,8 +21,7 @@ class HtmlTreeBranch: conceptually be something like "p div(#yo) span(.bar)". """ - def __init__(self, tags, fn): - # type: (List['TagInfo'], Optional[str]) -> None + def __init__(self, tags: List['TagInfo'], fn: Optional[str]) -> None: self.tags = tags self.fn = fn self.line = tags[-1].token.line @@ -32,8 +31,7 @@ class HtmlTreeBranch: for word in tag.words: self.words.add(word) - def staircase_text(self): - # type: () -> str + def staircase_text(self) -> str: """ produces representation of a node in staircase-like format: @@ -49,8 +47,7 @@ class HtmlTreeBranch: indent += ' ' * 4 return res - def text(self): - # type: () -> str + def text(self) -> str: """ produces one-line representation of branch: @@ -60,16 +57,15 @@ class HtmlTreeBranch: class Node: - def __init__(self, token, parent): # FIXME parent parameter is not used! - # type: (Token, Optional[Node]) -> None + def __init__(self, token: Token, parent: "Optional[Node]") -> None: + # FIXME parent parameter is not used! self.token = token self.children = [] # type: List[Node] self.parent = None # type: Optional[Node] class TagInfo: - def __init__(self, tag, classes, ids, token): - # type: (str, List[str], List[str], Token) -> None + def __init__(self, tag: str, classes: List[str], ids: List[str], token: Token) -> None: self.tag = tag self.classes = classes self.ids = ids @@ -79,8 +75,7 @@ class TagInfo: ['.' + s for s in classes] + \ ['#' + s for s in ids] - def text(self): - # type: () -> str + def text(self) -> str: s = self.tag if self.classes: s += '.' + '.'.join(self.classes) @@ -89,8 +84,7 @@ class TagInfo: return s -def get_tag_info(token): - # type: (Token) -> TagInfo +def get_tag_info(token: Token) -> TagInfo: s = token.s tag = token.tag classes = [] # type: List[str] @@ -112,8 +106,7 @@ def get_tag_info(token): return TagInfo(tag=tag, classes=classes, ids=ids, token=token) -def split_for_id_and_class(element): - # type: (str) -> List[str] +def split_for_id_and_class(element: str) -> List[str]: # Here we split a given string which is expected to contain id or class # attributes from HTML tags. This also takes care of template variables # in string during splitting process. For eg. 'red black {{ a|b|c }}' @@ -139,13 +132,11 @@ def split_for_id_and_class(element): return lst -def html_branches(text, fn=None): - # type: (str, Optional[str]) -> List[HtmlTreeBranch] +def html_branches(text: str, fn: Optional[str] = None) -> List[HtmlTreeBranch]: tree = html_tag_tree(text) branches = [] # type: List[HtmlTreeBranch] - def walk(node, tag_info_list=None): - # type: (Node, Optional[List[TagInfo]]) -> None + def walk(node: Node, tag_info_list: Optional[List[TagInfo]] = None) -> None: info = get_tag_info(node.token) if tag_info_list is None: tag_info_list = [info] @@ -165,8 +156,7 @@ def html_branches(text, fn=None): return branches -def html_tag_tree(text): - # type: (str) -> Node +def html_tag_tree(text: str) -> Node: tokens = tokenize(text) top_level = Node(token=None, parent=None) stack = [top_level] @@ -188,8 +178,7 @@ def html_tag_tree(text): return top_level -def build_id_dict(templates): - # type: (List[str]) -> (Dict[str, List[str]]) +def build_id_dict(templates: List[str]) -> (Dict[str, List[str]]): template_id_dict = defaultdict(list) # type: (Dict[str, List[str]]) for fn in templates: diff --git a/tools/lib/html_grep.py b/tools/lib/html_grep.py index 6cc3eef604..2857e9bf04 100644 --- a/tools/lib/html_grep.py +++ b/tools/lib/html_grep.py @@ -3,8 +3,7 @@ from typing import Dict, List, Set from .html_branches import html_branches, HtmlTreeBranch -def show_all_branches(fns): - # type: (List[str]) -> None +def show_all_branches(fns: List[str]) -> None: for fn in fns: print(fn) with open(fn) as f: @@ -21,8 +20,7 @@ class Grepper: HtmlTreeBranch objects. ''' - def __init__(self, fns): - # type: (List[str]) -> None + def __init__(self, fns: List[str]) -> None: all_branches = [] # type: List[HtmlTreeBranch] for fn in fns: @@ -38,8 +36,7 @@ class Grepper: self.all_branches = set(all_branches) - def grep(self, word_set): - # type: (Set[str]) -> None + def grep(self, word_set: Set[str]) -> None: words = list(word_set) # type: List[str] @@ -57,7 +54,6 @@ class Grepper: print(branch.staircase_text()) print('') -def grep(fns, words): - # type: (List[str], Set[str]) -> None +def grep(fns: List[str], words: Set[str]) -> None: grepper = Grepper(fns) grepper.grep(words) diff --git a/tools/lib/pretty_print.py b/tools/lib/pretty_print.py index fa08722651..8b6787a239 100644 --- a/tools/lib/pretty_print.py +++ b/tools/lib/pretty_print.py @@ -9,8 +9,7 @@ from zulint.printer import GREEN, ENDC import subprocess -def pretty_print_html(html, num_spaces=4): - # type: (str, int) -> str +def pretty_print_html(html: str, num_spaces: int = 4) -> str: # We use 1-based indexing for both rows and columns. tokens = tokenize(html) lines = html.split('\n') @@ -191,8 +190,7 @@ def pretty_print_html(html, num_spaces=4): return '\n'.join(formatted_lines) -def validate_indent_html(fn, fix): - # type: (str, bool) -> int +def validate_indent_html(fn: str, fix: bool) -> int: with open(fn) as f: html = f.read() phtml = pretty_print_html(html) diff --git a/tools/lib/provision.py b/tools/lib/provision.py index deb1af8e01..692ec01a7e 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -217,8 +217,7 @@ REPO_STOPWORDS_PATH = os.path.join( "zulip_english.stop", ) -def install_system_deps(): - # type: () -> None +def install_system_deps() -> None: # By doing list -> set -> list conversion, we remove duplicates. deps_to_install = sorted(set(SYSTEM_DEPENDENCIES)) @@ -235,8 +234,7 @@ def install_system_deps(): if BUILD_PGROONGA_FROM_SOURCE: run_as_root(["./scripts/lib/build-pgroonga"]) -def install_apt_deps(deps_to_install): - # type: (List[str]) -> None +def install_apt_deps(deps_to_install: List[str]) -> None: # setup-apt-repo does an `apt-get update` if the sources.list files changed. run_as_root(["./scripts/lib/setup-apt-repo"]) @@ -253,8 +251,7 @@ def install_apt_deps(deps_to_install): + deps_to_install ) -def install_yum_deps(deps_to_install): - # type: (List[str]) -> None +def install_yum_deps(deps_to_install: List[str]) -> None: print(WARNING + "RedHat support is still experimental.") run_as_root(["./scripts/lib/setup-yum-repo"]) @@ -314,8 +311,7 @@ def install_yum_deps(deps_to_install): overwrite_symlink("/usr/share/myspell/en_US.aff", "/usr/pgsql-%s/share/tsearch_data/en_us.affix" % (POSTGRES_VERSION,)) -def main(options): - # type: (argparse.Namespace) -> NoReturn +def main(options: argparse.Namespace) -> "NoReturn": # yarn and management commands expect to be run from the root of the # project. diff --git a/tools/lib/provision_inner.py b/tools/lib/provision_inner.py index 2543898086..0975a53169 100755 --- a/tools/lib/provision_inner.py +++ b/tools/lib/provision_inner.py @@ -33,12 +33,10 @@ def create_var_directories() -> None: path = os.path.join(var_dir, sub_dir) os.makedirs(path, exist_ok=True) -def setup_shell_profile(shell_profile): - # type: (str) -> None +def setup_shell_profile(shell_profile: str) -> None: shell_profile_path = os.path.expanduser(shell_profile) - def write_command(command): - # type: (str) -> None + def write_command(command: str) -> None: if os.path.exists(shell_profile_path): with open(shell_profile_path) as shell_profile_file: lines = [line.strip() for line in shell_profile_file.readlines()] diff --git a/tools/lib/sanity_check.py b/tools/lib/sanity_check.py index 911731adde..c5bfc9edf7 100644 --- a/tools/lib/sanity_check.py +++ b/tools/lib/sanity_check.py @@ -2,8 +2,7 @@ import os import pwd import sys -def check_venv(filename): - # type: (str) -> None +def check_venv(filename: str) -> None: try: import django import ujson diff --git a/tools/lib/template_parser.py b/tools/lib/template_parser.py index 8a98c99af5..c9084def2d 100644 --- a/tools/lib/template_parser.py +++ b/tools/lib/template_parser.py @@ -1,30 +1,25 @@ from typing import Callable, List, Optional, Text class TemplateParserException(Exception): - def __init__(self, message): - # type: (str) -> None + def __init__(self, message: str) -> None: self.message = message - def __str__(self): - # type: () -> str + def __str__(self) -> str: return self.message class TokenizationException(Exception): - def __init__(self, message, line_content=None): - # type: (str, Optional[str]) -> None + def __init__(self, message: str, line_content: Optional[str] = None) -> None: self.message = message self.line_content = line_content class TokenizerState: - def __init__(self): - # type: () -> None + def __init__(self) -> None: self.i = 0 self.line = 1 self.col = 1 class Token: - def __init__(self, kind, s, tag, line, col, line_span): - # type: (str, str, str, int, int, int) -> None + def __init__(self, kind: str, s: str, tag: str, line: int, col: int, line_span: int) -> None: self.kind = kind self.s = s self.tag = tag @@ -32,10 +27,8 @@ class Token: self.col = col self.line_span = line_span -def tokenize(text): - # type: (str) -> List[Token] - def advance(n): - # type: (int) -> None +def tokenize(text: str) -> List[Token]: + def advance(n: int) -> None: for _ in range(n): state.i += 1 if state.i >= 0 and text[state.i - 1] == '\n': @@ -44,55 +37,43 @@ def tokenize(text): else: state.col += 1 - def looking_at(s): - # type: (str) -> bool + def looking_at(s: str) -> bool: return text[state.i:state.i+len(s)] == s - def looking_at_htmlcomment(): - # type: () -> bool + def looking_at_htmlcomment() -> bool: return looking_at("