tools: Use root-based absolute import for tools.lib, etc.

Mypy can’t follow absolute imports based on directories other than the
root.  This was hiding some type errors due to ignore_missing_imports.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

scripts/zulip-puppet-apply

@@ -9,11 +9,13 @@ import tempfile
 
 import yaml
 
-from lib.puppet_cache import setup_puppet_modules
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-from lib.zulip_tools import assert_running_as_root, parse_os_release
+sys.path.insert(0, BASE_DIR)
+
+from scripts.lib.puppet_cache import setup_puppet_modules
+from scripts.lib.zulip_tools import assert_running_as_root, parse_os_release
 
 assert_running_as_root()
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 parser = argparse.ArgumentParser(description="Run Puppet")
 parser.add_argument(

tools/build-docs

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 import argparse
 import os
 import sys
 
 ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(ZULIP_PATH)
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 from scripts.lib.zulip_tools import ENDC, WARNING, run
 
 

tools/check-capitalization

@@ -1,10 +1,5 @@
 #!/usr/bin/env python3
 
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 import argparse
 import json
 import os
@@ -13,6 +8,12 @@ import subprocess
 import sys
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 from scripts.lib.zulip_tools import ENDC, FAIL, WARNING
 from tools.lib.capitalization import check_capitalization
 

tools/check-frontend-i18n

@@ -1,18 +1,18 @@
 #!/usr/bin/env python3
-from typing import List
-
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 import argparse
 import json
 import os
 import subprocess
 import sys
+from typing import List
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 from scripts.lib.zulip_tools import ENDC, FAIL, WARNING
 
 

tools/check-templates

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 import argparse
 import logging
+import os
 import sys
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
 # check for the venv
-from lib import sanity_check
+from tools.lib import sanity_check
-from lib.html_branches import build_id_dict
-from lib.pretty_print import validate_indent_html
-from lib.template_parser import validate
 
 sanity_check.check_venv(__file__)
 
@@ -15,6 +15,10 @@ from typing import Dict, Iterable, List
 
 from zulint import lister
 
+from tools.lib.html_branches import build_id_dict
+from tools.lib.pretty_print import validate_indent_html
+from tools.lib.template_parser import validate
+
 EXCLUDED_FILES = [
     ## Test data Files for testing modules in tests
     "tools/tests/test_template_data",

tools/create-test-api-docs

@@ -1,16 +1,19 @@
 #!/usr/bin/env python3
 
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 import html
 import os
 import pprint
+import sys
 from collections import defaultdict
 from typing import Any, Dict, List, Set
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 import orjson
 
 Call = Dict[str, Any]

tools/generate-integration-docs-screenshot

@@ -1,17 +1,23 @@
 #!/usr/bin/env python3
 """Create or update a webhook integration screenshot using a test fixture."""
 
-# check for the venv
+import argparse
-from lib import sanity_check
+import base64
-
-sanity_check.check_venv(__file__)
 import os
+import subprocess
 import sys
+from typing import Any, Dict, Optional, Tuple
+from urllib.parse import parse_qsl, urlencode
 
 TOOLS_DIR = os.path.abspath(os.path.dirname(__file__))
 ROOT_DIR = os.path.dirname(TOOLS_DIR)
-
 sys.path.insert(0, ROOT_DIR)
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 from scripts.lib.setup_path import setup_path
 
 setup_path()
@@ -21,12 +27,6 @@ import django
 
 django.setup()
 
-import argparse
-import base64
-import subprocess
-from typing import Any, Dict, Optional, Tuple
-from urllib.parse import parse_qsl, urlencode
-
 import orjson
 import requests
 import zulip

tools/lint

@@ -5,21 +5,21 @@ import random
 import re
 import sys
 
+tools_dir = os.path.dirname(os.path.abspath(__file__))
+root_dir = os.path.join(tools_dir, "..")
+sys.path.insert(0, root_dir)
+
 # check for the venv
-from lib import sanity_check
+from tools.lib import sanity_check
 
 sanity_check.check_venv(__file__)
 
 from zulint.command import LinterConfig, add_default_linter_arguments
 
-from linter_lib.custom_check import non_py_rules, python_rules
+from tools.linter_lib.custom_check import non_py_rules, python_rules
 
 
 def run() -> None:
-    tools_dir = os.path.dirname(os.path.abspath(__file__))
-    root_dir = os.path.dirname(tools_dir)
-    sys.path.insert(0, root_dir)
-
     from tools.lib.test_script import (
         add_provision_check_override_param,
         assert_provisioning_status_ok,

tools/pretty-print-html

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
+import os
 import sys
 from typing import List
 
-from lib.pretty_print import pretty_print_html
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from tools.lib.pretty_print import pretty_print_html
 
 
 def clean_html(filenames: List[str]) -> None:

tools/run-dev.py

@@ -8,17 +8,17 @@ import sys
 from typing import Any, Callable, Generator, List, Sequence
 from urllib.parse import urlunparse
 
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
 # check for the venv
-from lib import sanity_check
+from tools.lib import sanity_check
 
 sanity_check.check_venv(__file__)
 
 from tornado import gen, httpclient, httputil, web
 from tornado.ioloop import IOLoop
 
-TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.dirname(TOOLS_DIR))
-
 from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok
 
 if "posix" in os.name and os.geteuid() == 0:
@@ -90,8 +90,6 @@ else:
 manage_args = [f"--settings={settings_module}"]
 os.environ["DJANGO_SETTINGS_MODULE"] = settings_module
 
-sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
-
 from scripts.lib.zulip_tools import CYAN, ENDC, FAIL
 
 proxy_port = base_port

tools/test-api

@@ -5,15 +5,15 @@ import sys
 
 os.environ["RUNNING_OPENAPI_CURL_TEST"] = "1"
 
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ZULIP_PATH)
 os.chdir(ZULIP_PATH)
 
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 from zulip import Client
 
 from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok

tools/test-backend

@@ -1,10 +1,5 @@
 #!/usr/bin/env python3
 
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 import argparse
 import contextlib
 import glob
@@ -16,6 +11,15 @@ import tempfile
 from typing import Iterator, List
 from unittest import mock
 
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+os.chdir(os.path.dirname(TOOLS_DIR))
+sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 import django
 import orjson
 import responses
@@ -185,10 +189,6 @@ class ZulipInternetBlockedError(Exception):
 
 
 def main() -> None:
-    TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
-    os.chdir(os.path.dirname(TOOLS_DIR))
-    sys.path.insert(0, os.path.dirname(TOOLS_DIR))
-
     default_parallel = os.cpu_count()
 
     # Remove proxy settings for running backend tests

tools/test-help-documentation

@@ -7,16 +7,16 @@ import sys
 from typing import Iterator
 
 ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-# check for the venv
-from lib import sanity_check
-
-sanity_check.check_venv(__file__)
-
 os.chdir(ZULIP_PATH)
 sys.path.insert(0, ZULIP_PATH)
 
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 from tools.lib.test_script import add_provision_check_override_param
+from tools.lib.test_server import test_server_running
 
 parser = argparse.ArgumentParser()
 add_provision_check_override_param(parser)
@@ -28,8 +28,6 @@ parser.add_argument(
 )
 options = parser.parse_args()
 
-from tools.lib.test_server import test_server_running
-
 os.makedirs("var/help-documentation", exist_ok=True)
 
 LOG_FILE = "var/help-documentation/server.log"

tools/test-queue-worker-reload

@@ -6,8 +6,10 @@ import sys
 import time
 import types
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
 # check for the venv
-from lib import sanity_check
+from tools.lib import sanity_check
 
 sanity_check.check_venv(__file__)
 

tools/test-run-dev

@@ -6,7 +6,9 @@ import sys
 import time
 from typing import Tuple
 
-from lib import sanity_check
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from tools.lib import sanity_check
 
 sanity_check.check_venv(__file__)
 

tools/test-tools

@@ -4,8 +4,14 @@ import os
 import sys
 import unittest
 
+tools_dir = os.path.dirname(os.path.abspath(__file__))
+root_dir = os.path.abspath(os.path.join(tools_dir, ".."))
+tools_test_dir = os.path.join(tools_dir, "tests")
+
+sys.path.insert(0, root_dir)
+
 # check for the venv
-from lib import sanity_check
+from tools.lib import sanity_check
 
 sanity_check.check_venv(__file__)
 
@@ -14,22 +20,13 @@ if __name__ == "__main__":
     parser.add_argument("--coverage", action="store_true", help="compute test coverage")
     args = parser.parse_args()
 
-    def dir_join(dir1: str, dir2: str) -> str:
-        return os.path.abspath(os.path.join(dir1, dir2))
-
-    tools_dir = os.path.dirname(os.path.abspath(__file__))
-    root_dir = dir_join(tools_dir, "..")
-    tools_test_dir = dir_join(tools_dir, "tests")
-
-    sys.path.insert(0, root_dir)
-
     loader = unittest.TestLoader()
 
     if args.coverage:
         import coverage
 
         cov = coverage.Coverage(
-            branch=True, omit=["*/zulip-venv-cache/*", dir_join(tools_test_dir, "*")]
+            branch=True, omit=["*/zulip-venv-cache/*", os.path.join(tools_test_dir, "*")]
         )
         cov.start()
 

tools/tests/test_zulint_custom_rules.py

@@ -5,7 +5,7 @@ from unittest.mock import patch
 
 from zulint.custom_rules import RuleList
 
-from linter_lib.custom_check import non_py_rules, python_rules
+from tools.linter_lib.custom_check import non_py_rules, python_rules
 
 ROOT_DIR = os.path.abspath(os.path.join(__file__, "..", "..", ".."))
 CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`."
@@ -13,9 +13,10 @@ CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py
 
 class TestRuleList(TestCase):
     def setUp(self) -> None:
-        self.all_rules = python_rules.rules
+        all_rules = list(python_rules.rules)
         for rule in non_py_rules:
-            self.all_rules.extend(rule.rules)
+            all_rules.extend(rule.rules)
+        self.all_rules = all_rules
 
     def test_paths_in_rules(self) -> None:
         """Verifies that the paths mentioned in linter rules actually exist"""

tools/update-prod-static

@@ -10,17 +10,19 @@ from scripts.lib.setup_path import setup_path
 
 setup_path()
 
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
 os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
 from django.conf import settings
 
-# check for the venv
-from lib import sanity_check
 from scripts.lib.node_cache import setup_node_modules
 from scripts.lib.zulip_tools import assert_not_running_as_root, run
 
 assert_not_running_as_root()
 
-sanity_check.check_venv(__file__)
 
 os.chdir(settings.DEPLOY_ROOT)