lint: Replace pycodestyle and pyflakes with ruff.

https://github.com/charliermarsh/ruff

Signed-off-by: Anders Kaseorg <anders@zulip.com>

.gitignore

@@ -33,6 +33,7 @@ package-lock.json
 !/var/puppeteer/test_credentials.d.ts
 
 /.dmypy.json
+/.ruff_cache
 
 # Generated i18n data
 /locale/en

docs/contributing/code-style.md

@@ -52,7 +52,7 @@ The Vagrant setup process runs this for you.
 - JavaScript ([ESLint](https://eslint.org/),
   [Prettier](https://prettier.io/))
 - Python ([mypy](http://mypy-lang.org/),
-  [Pyflakes](https://pypi.python.org/pypi/pyflakes),
+  [ruff](https://github.com/charliermarsh/ruff),
   [Black](https://github.com/psf/black),
   [isort](https://pycqa.github.io/isort/))
 - templates

docs/testing/linters.md

@@ -30,7 +30,7 @@ below will direct you to the official documentation for these projects.
 - [Prettier](https://prettier.io/)
 - [Puppet](https://puppet.com/) (puppet provides its own mechanism for
   validating manifests)
-- [pyflakes](https://pypi.python.org/pypi/pyflakes)
+- [ruff](https://github.com/charliermarsh/ruff)
 - [stylelint](https://github.com/stylelint/stylelint)
 
 Zulip also uses some home-grown code to perform tasks like validating
@@ -109,7 +109,7 @@ describes our test system in detail.
 Most of our lint checks get performed by `./tools/lint`. These include the
 following checks:
 
-- Check Python code with pyflakes.
+- Check Python code with ruff.
 - Check Python formatting with Black and isort.
 - Check JavaScript and TypeScript code with ESLint.
 - Check CSS, JavaScript, TypeScript, and YAML formatting with Prettier.
@@ -131,7 +131,7 @@ The rest of this document pertains to the checks that occur in `./tools/lint`.
 
 Zulip has a script called `lint` that lives in our "tools" directory.
 It is the workhorse of our linting system, although in some cases it
-dispatches the heavy lifting to other components such as pyflakes,
+dispatches the heavy lifting to other components such as ruff,
 eslint, and other home grown tools.
 
 You can find the source code [here](https://github.com/zulip/zulip/blob/main/tools/lint).
@@ -139,8 +139,7 @@ You can find the source code [here](https://github.com/zulip/zulip/blob/main/too
 In order for our entire lint suite to run in a timely fashion, the `lint`
 script performs several lint checks in parallel by forking out subprocesses.
 
-Note that our project does custom regex-based checks on the code, and we
+Note that our project does custom regex-based checks on the code. The code for these
-also customize how we call pyflakes and pycodestyle (pep8). The code for these
 types of checks mostly lives [here](https://github.com/zulip/zulip/tree/main/tools/linter_lib).
 
 ### Special options
@@ -182,10 +181,8 @@ Our Python code is formatted using Black (using the options in the
 options in `.isort.cfg`). The `lint` script enforces this by running
 Black and isort in check mode, or in write mode with `--fix`.
 
-The bulk of our Python linting gets outsourced to the "pyflakes" tool. We
+The bulk of our Python linting gets outsourced to the "ruff" tool,
-call "pyflakes" in a fairly vanilla fashion, and then we post-process its
+which is configured in the `tool.ruff` section of `pyproject.toml`.
-output to exclude certain specific errors that Zulip is comfortable
-ignoring.
 
 Zulip also has custom regex-based rules that it applies to Python code.
 Look for `python_rules` in the source code for `lint`. Note that we

pyproject.toml

@@ -96,3 +96,12 @@ ignore_missing_imports = true
 
 [tool.django-stubs]
 django_settings_module = "zproject.settings"
+
+[tool.ruff]
+# See https://github.com/charliermarsh/ruff#rules for error code definitions.
+ignore = [
+    "E402", # Module level import not at top of file
+    "E501", # Line too long
+    "E731", # Do not assign a lambda expression, use a def
+]
+line-length = 100

requirements/dev.in

@@ -29,14 +29,11 @@ isort
 # For doing highly usable Python profiling
 line-profiler
 
-# for pep8 linter
-pycodestyle
-
 # Python reformatter
 black
 
-# Needed to run pyflakes linter
+# Python linter
-pyflakes
+ruff
 
 # Needed for watching file changes
 pyinotify

requirements/dev.txt

@@ -1423,9 +1423,7 @@ pyasn1-modules==0.2.8 \
 pycodestyle==2.9.1 \
     --hash=sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785 \
     --hash=sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b
-    # via
+    # via zulint
-    #   -r requirements/dev.in
-    #   zulint
 pycparser==2.21 \
     --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
     --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
@@ -1436,9 +1434,7 @@ pydispatcher==2.0.6 \
 pyflakes==2.5.0 \
     --hash=sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2 \
     --hash=sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3
-    # via
+    # via zulint
-    #   -r requirements/dev.in
-    #   zulint
 pygments==2.13.0 \
     --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \
     --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42
@@ -1869,6 +1865,24 @@ ruamel.yaml.clib==0.2.6 \
     --hash=sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed \
     --hash=sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c
     # via ruamel.yaml
+ruff==0.0.99 \
+    --hash=sha256:0dfe9104092f4e2a9ac6af8328702bba7c725f037ed3ff83a3bab37de1edce1b \
+    --hash=sha256:1047ee9b65b5ec700bf7a803e4cfc8b22755e69de73058f65fedf4859fecf52a \
+    --hash=sha256:11992bc96095bfa9af49ad33b47c4612cc4e27e84052f707a8cd4cf39f0f71fd \
+    --hash=sha256:21ea89b56d3042c12d727d83d17b62dfb55acfe3a1f515b87b805554e7a3bb88 \
+    --hash=sha256:331ad9d419faaa47c13a540b704d8c7dc84883c6e682c167a86726c4d1a9fcab \
+    --hash=sha256:347d46a6e793bf7ca28c04f043054efdb27945ee4fc76561da19387b70e0646c \
+    --hash=sha256:3e2986309ccfddf43d0517ef5d6da6e3b50a32eac227f8307acf7d90f581373b \
+    --hash=sha256:44a6967bce6696e602132ea2a1f5e14ae7d072ddb7bbb8c453d6f552794cc21b \
+    --hash=sha256:60695d481e0091cb4fe67a1a9f0aeaca5a27a3cda45cc46b5bcfd0a4e17bc8dc \
+    --hash=sha256:74ceb85fb2466646a8e87eb4eb66201569ad773cc4ec66384af6728eb2d21416 \
+    --hash=sha256:897b4336118c3c980951381bdfcbf21c0eab941c3ce2eee1082fe480c64483a2 \
+    --hash=sha256:8d4f215a1d337601f7f30facc51ee0db9eb97b9c8d21071d9910e42583c65df7 \
+    --hash=sha256:9ceb8819b4f36fbcfc50810371951950189883f4c510ce0c7603eee7999f37a8 \
+    --hash=sha256:a83fce30d1030b0621a03e5a94316ef909510ac5c56eba8bda664d16ad67cf79 \
+    --hash=sha256:b0ae12b517aee0827548092d31222e1fae4b149514414c76e4d8697cf984d80f \
+    --hash=sha256:c82bb07d05018a7c7d58175331b2d73d45570b73c74fa16dc0b6b8828d243c5d
+    # via -r requirements/dev.in
 s3transfer==0.6.0 \
     --hash=sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd \
     --hash=sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947

setup.cfg

@@ -1,29 +0,0 @@
-[pycodestyle]
-ignore =
-    # These rules are ignored for the reasons explained in the comments.
-    # See https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes for
-    # error code definitions.
-
-    # All formatting is handled by Black.
-    E1,
-    E2,
-    E3,
-    E401,
-    E5,
-    E701,
-    E702,
-    E703,
-    E704,
-    W1,
-    W2,
-    W3,
-    W5,
-
-    # "module level import not at top of file"
-    # Most of these are there for valid reasons, though there might be a
-    # few that could be eliminated.
-    E402,
-
-    # "do not assign a lambda expression, use a def"
-    # Fixing these would probably reduce readability in most cases.
-    E731,

tools/lint

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 import argparse
 import os
-import random
 import re
 import sys
 
@@ -25,8 +24,6 @@ def run() -> None:
         assert_provisioning_status_ok,
     )
     from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
-    from tools.linter_lib.pep8 import check_pep8
-    from tools.linter_lib.pyflakes import check_pyflakes
 
     parser = argparse.ArgumentParser()
     add_provision_check_override_param(parser)
@@ -146,6 +143,13 @@ def run() -> None:
         if args.use_mypy_daemon
         else lambda _: False,
     )
+    linter_config.external_linter(
+        "ruff",
+        ["ruff", "--quiet"],
+        ["py", "pyi"],
+        fix_arg="--fix",
+        description="Python linter",
+    )
     linter_config.external_linter(
         "tsc",
         ["tools/run-tsc"],
@@ -224,27 +228,6 @@ def run() -> None:
             failed = failed or rule.check(by_lang, verbose=args.verbose)
         return 1 if failed else 0
 
-    @linter_config.lint
-    def pyflakes() -> int:
-        """Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)"""
-        failed = check_pyflakes(by_lang["py"], args)
-        return 1 if failed else 0
-
-    python_part1 = {x for x in by_lang["py"] + by_lang["pyi"] if random.randint(0, 1) == 0}
-    python_part2 = {y for y in by_lang["py"] + by_lang["pyi"] if y not in python_part1}
-
-    @linter_config.lint
-    def pep8_1of2() -> int:
-        """Standard Python style linter on 50% of files (config: setup.cfg)"""
-        failed = check_pep8(list(python_part1))
-        return 1 if failed else 0
-
-    @linter_config.lint
-    def pep8_2of2() -> int:
-        """Standard Python style linter on other 50% of files (config: setup.cfg)"""
-        failed = check_pep8(list(python_part2))
-        return 1 if failed else 0
-
     linter_config.do_lint()
 
 

tools/linter_lib/pep8.py

@@ -1,10 +0,0 @@
-from typing import List
-
-from zulint.linters import run_command
-from zulint.printer import colors
-
-
-def check_pep8(files: List[str]) -> bool:
-    if not files:
-        return False
-    return run_command("pep8", next(colors), ["pycodestyle", "--", *files]) != 0

tools/linter_lib/pyflakes.py

@@ -1,34 +0,0 @@
-import argparse
-from typing import List
-
-from zulint.linters import run_pyflakes
-
-
-def check_pyflakes(files: List[str], options: argparse.Namespace) -> bool:
-    suppress_patterns = [
-        ("scripts/lib/pythonrc.py", "imported but unused"),
-        # LDAP imports are necessary for docker-zulip.
-        ("zproject/prod_settings_template.py", "imported but unused"),
-        # Our ipython startup pythonrc file intentionally imports *
-        ("scripts/lib/pythonrc.py", " import *' used; unable to detect undefined names"),
-        (
-            "zerver/views/realm.py",
-            "local variable 'message_retention_days' is assigned to but never used",
-        ),
-        (
-            "zerver/views/realm.py",
-            "local variable 'message_content_delete_limit_seconds' is assigned to but never used",
-        ),
-        ("settings.py", "settings import *' used; unable to detect undefined names"),
-        (
-            "settings.py",
-            "'from .prod_settings_template import *' used; unable to detect undefined names",
-        ),
-        ("settings.py", "settings.*' imported but unused"),
-        ("settings.py", "'.prod_settings_template.*' imported but unused"),
-        # Sphinx adds `tags` specially to the environment when running conf.py.
-        ("docs/conf.py", "undefined name 'tags'"),
-    ]
-    if options.full:
-        suppress_patterns = []
-    return run_pyflakes(files, options, suppress_patterns)

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 154
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (208, 0)
+PROVISION_VERSION = (208, 1)

zerver/views/realm.py

@@ -175,6 +175,7 @@ def update_realm(
         message_retention_days = parse_message_retention_days(
             message_retention_days_raw, Realm.MESSAGE_RETENTION_SPECIAL_VALUES_MAP
         )
+    message_retention_days  # used by locals() below
 
     if (
         invite_to_realm_policy is not None or invite_required is not None

zproject/prod_settings.pyi

@@ -1 +1 @@
-from .prod_settings_template import *
+from .prod_settings_template import *  # noqa: F403

zproject/prod_settings_template.py

@@ -158,7 +158,7 @@ AUTHENTICATION_BACKENDS: Tuple[str, ...] = (
 ## optionally using LDAP as an authentication mechanism.
 
 import ldap
-from django_auth_ldap.config import GroupOfNamesType, LDAPGroupQuery, LDAPSearch
+from django_auth_ldap.config import GroupOfNamesType, LDAPGroupQuery, LDAPSearch  # noqa: F401
 
 ## Connecting to the LDAP server.
 ##