From adffad384cbd748fc7839f238a118f2c9cf64f86 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Sat, 29 Oct 2022 18:07:03 -0400 Subject: [PATCH] lint: Replace pycodestyle and pyflakes with ruff. https://github.com/charliermarsh/ruff Signed-off-by: Anders Kaseorg --- .gitignore | 1 + docs/contributing/code-style.md | 2 +- docs/testing/linters.md | 15 ++++++------- pyproject.toml | 9 ++++++++ requirements/dev.in | 7 ++---- requirements/dev.txt | 26 +++++++++++++++++------ setup.cfg | 29 ------------------------- tools/lint | 31 ++++++--------------------- tools/linter_lib/pep8.py | 10 --------- tools/linter_lib/pyflakes.py | 34 ------------------------------ version.py | 2 +- zerver/views/realm.py | 1 + zproject/prod_settings.pyi | 2 +- zproject/prod_settings_template.py | 2 +- 14 files changed, 50 insertions(+), 121 deletions(-) delete mode 100644 setup.cfg delete mode 100644 tools/linter_lib/pep8.py delete mode 100644 tools/linter_lib/pyflakes.py diff --git a/.gitignore b/.gitignore index 8d6fae4d72..9028702760 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ package-lock.json !/var/puppeteer/test_credentials.d.ts /.dmypy.json +/.ruff_cache # Generated i18n data /locale/en diff --git a/docs/contributing/code-style.md b/docs/contributing/code-style.md index 16670ec348..1a34938197 100644 --- a/docs/contributing/code-style.md +++ b/docs/contributing/code-style.md @@ -52,7 +52,7 @@ The Vagrant setup process runs this for you. - JavaScript ([ESLint](https://eslint.org/), [Prettier](https://prettier.io/)) - Python ([mypy](http://mypy-lang.org/), - [Pyflakes](https://pypi.python.org/pypi/pyflakes), + [ruff](https://github.com/charliermarsh/ruff), [Black](https://github.com/psf/black), [isort](https://pycqa.github.io/isort/)) - templates diff --git a/docs/testing/linters.md b/docs/testing/linters.md index dcc628aa82..e449d1259e 100644 --- a/docs/testing/linters.md +++ b/docs/testing/linters.md @@ -30,7 +30,7 @@ below will direct you to the official documentation for these projects. - [Prettier](https://prettier.io/) - [Puppet](https://puppet.com/) (puppet provides its own mechanism for validating manifests) -- [pyflakes](https://pypi.python.org/pypi/pyflakes) +- [ruff](https://github.com/charliermarsh/ruff) - [stylelint](https://github.com/stylelint/stylelint) Zulip also uses some home-grown code to perform tasks like validating @@ -109,7 +109,7 @@ describes our test system in detail. Most of our lint checks get performed by `./tools/lint`. These include the following checks: -- Check Python code with pyflakes. +- Check Python code with ruff. - Check Python formatting with Black and isort. - Check JavaScript and TypeScript code with ESLint. - Check CSS, JavaScript, TypeScript, and YAML formatting with Prettier. @@ -131,7 +131,7 @@ The rest of this document pertains to the checks that occur in `./tools/lint`. Zulip has a script called `lint` that lives in our "tools" directory. It is the workhorse of our linting system, although in some cases it -dispatches the heavy lifting to other components such as pyflakes, +dispatches the heavy lifting to other components such as ruff, eslint, and other home grown tools. You can find the source code [here](https://github.com/zulip/zulip/blob/main/tools/lint). @@ -139,8 +139,7 @@ You can find the source code [here](https://github.com/zulip/zulip/blob/main/too In order for our entire lint suite to run in a timely fashion, the `lint` script performs several lint checks in parallel by forking out subprocesses. -Note that our project does custom regex-based checks on the code, and we -also customize how we call pyflakes and pycodestyle (pep8). The code for these +Note that our project does custom regex-based checks on the code. The code for these types of checks mostly lives [here](https://github.com/zulip/zulip/tree/main/tools/linter_lib). ### Special options @@ -182,10 +181,8 @@ Our Python code is formatted using Black (using the options in the options in `.isort.cfg`). The `lint` script enforces this by running Black and isort in check mode, or in write mode with `--fix`. -The bulk of our Python linting gets outsourced to the "pyflakes" tool. We -call "pyflakes" in a fairly vanilla fashion, and then we post-process its -output to exclude certain specific errors that Zulip is comfortable -ignoring. +The bulk of our Python linting gets outsourced to the "ruff" tool, +which is configured in the `tool.ruff` section of `pyproject.toml`. Zulip also has custom regex-based rules that it applies to Python code. Look for `python_rules` in the source code for `lint`. Note that we diff --git a/pyproject.toml b/pyproject.toml index cf16ab0bb7..f16ff73768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,3 +96,12 @@ ignore_missing_imports = true [tool.django-stubs] django_settings_module = "zproject.settings" + +[tool.ruff] +# See https://github.com/charliermarsh/ruff#rules for error code definitions. +ignore = [ + "E402", # Module level import not at top of file + "E501", # Line too long + "E731", # Do not assign a lambda expression, use a def +] +line-length = 100 diff --git a/requirements/dev.in b/requirements/dev.in index d8f4b87ee1..3d6448f1b4 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -29,14 +29,11 @@ isort # For doing highly usable Python profiling line-profiler -# for pep8 linter -pycodestyle - # Python reformatter black -# Needed to run pyflakes linter -pyflakes +# Python linter +ruff # Needed for watching file changes pyinotify diff --git a/requirements/dev.txt b/requirements/dev.txt index a0d7c8b277..68ffbd94bf 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1423,9 +1423,7 @@ pyasn1-modules==0.2.8 \ pycodestyle==2.9.1 \ --hash=sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785 \ --hash=sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b - # via - # -r requirements/dev.in - # zulint + # via zulint pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 @@ -1436,9 +1434,7 @@ pydispatcher==2.0.6 \ pyflakes==2.5.0 \ --hash=sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2 \ --hash=sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3 - # via - # -r requirements/dev.in - # zulint + # via zulint pygments==2.13.0 \ --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \ --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42 @@ -1869,6 +1865,24 @@ ruamel.yaml.clib==0.2.6 \ --hash=sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed \ --hash=sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c # via ruamel.yaml +ruff==0.0.99 \ + --hash=sha256:0dfe9104092f4e2a9ac6af8328702bba7c725f037ed3ff83a3bab37de1edce1b \ + --hash=sha256:1047ee9b65b5ec700bf7a803e4cfc8b22755e69de73058f65fedf4859fecf52a \ + --hash=sha256:11992bc96095bfa9af49ad33b47c4612cc4e27e84052f707a8cd4cf39f0f71fd \ + --hash=sha256:21ea89b56d3042c12d727d83d17b62dfb55acfe3a1f515b87b805554e7a3bb88 \ + --hash=sha256:331ad9d419faaa47c13a540b704d8c7dc84883c6e682c167a86726c4d1a9fcab \ + --hash=sha256:347d46a6e793bf7ca28c04f043054efdb27945ee4fc76561da19387b70e0646c \ + --hash=sha256:3e2986309ccfddf43d0517ef5d6da6e3b50a32eac227f8307acf7d90f581373b \ + --hash=sha256:44a6967bce6696e602132ea2a1f5e14ae7d072ddb7bbb8c453d6f552794cc21b \ + --hash=sha256:60695d481e0091cb4fe67a1a9f0aeaca5a27a3cda45cc46b5bcfd0a4e17bc8dc \ + --hash=sha256:74ceb85fb2466646a8e87eb4eb66201569ad773cc4ec66384af6728eb2d21416 \ + --hash=sha256:897b4336118c3c980951381bdfcbf21c0eab941c3ce2eee1082fe480c64483a2 \ + --hash=sha256:8d4f215a1d337601f7f30facc51ee0db9eb97b9c8d21071d9910e42583c65df7 \ + --hash=sha256:9ceb8819b4f36fbcfc50810371951950189883f4c510ce0c7603eee7999f37a8 \ + --hash=sha256:a83fce30d1030b0621a03e5a94316ef909510ac5c56eba8bda664d16ad67cf79 \ + --hash=sha256:b0ae12b517aee0827548092d31222e1fae4b149514414c76e4d8697cf984d80f \ + --hash=sha256:c82bb07d05018a7c7d58175331b2d73d45570b73c74fa16dc0b6b8828d243c5d + # via -r requirements/dev.in s3transfer==0.6.0 \ --hash=sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd \ --hash=sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ae06b75a03..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,29 +0,0 @@ -[pycodestyle] -ignore = - # These rules are ignored for the reasons explained in the comments. - # See https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes for - # error code definitions. - - # All formatting is handled by Black. - E1, - E2, - E3, - E401, - E5, - E701, - E702, - E703, - E704, - W1, - W2, - W3, - W5, - - # "module level import not at top of file" - # Most of these are there for valid reasons, though there might be a - # few that could be eliminated. - E402, - - # "do not assign a lambda expression, use a def" - # Fixing these would probably reduce readability in most cases. - E731, diff --git a/tools/lint b/tools/lint index 5c08fa772e..519e03cc31 100755 --- a/tools/lint +++ b/tools/lint @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse import os -import random import re import sys @@ -25,8 +24,6 @@ def run() -> None: assert_provisioning_status_ok, ) from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE - from tools.linter_lib.pep8 import check_pep8 - from tools.linter_lib.pyflakes import check_pyflakes parser = argparse.ArgumentParser() add_provision_check_override_param(parser) @@ -146,6 +143,13 @@ def run() -> None: if args.use_mypy_daemon else lambda _: False, ) + linter_config.external_linter( + "ruff", + ["ruff", "--quiet"], + ["py", "pyi"], + fix_arg="--fix", + description="Python linter", + ) linter_config.external_linter( "tsc", ["tools/run-tsc"], @@ -224,27 +228,6 @@ def run() -> None: failed = failed or rule.check(by_lang, verbose=args.verbose) return 1 if failed else 0 - @linter_config.lint - def pyflakes() -> int: - """Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)""" - failed = check_pyflakes(by_lang["py"], args) - return 1 if failed else 0 - - python_part1 = {x for x in by_lang["py"] + by_lang["pyi"] if random.randint(0, 1) == 0} - python_part2 = {y for y in by_lang["py"] + by_lang["pyi"] if y not in python_part1} - - @linter_config.lint - def pep8_1of2() -> int: - """Standard Python style linter on 50% of files (config: setup.cfg)""" - failed = check_pep8(list(python_part1)) - return 1 if failed else 0 - - @linter_config.lint - def pep8_2of2() -> int: - """Standard Python style linter on other 50% of files (config: setup.cfg)""" - failed = check_pep8(list(python_part2)) - return 1 if failed else 0 - linter_config.do_lint() diff --git a/tools/linter_lib/pep8.py b/tools/linter_lib/pep8.py deleted file mode 100644 index b8f3098d64..0000000000 --- a/tools/linter_lib/pep8.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import List - -from zulint.linters import run_command -from zulint.printer import colors - - -def check_pep8(files: List[str]) -> bool: - if not files: - return False - return run_command("pep8", next(colors), ["pycodestyle", "--", *files]) != 0 diff --git a/tools/linter_lib/pyflakes.py b/tools/linter_lib/pyflakes.py deleted file mode 100644 index e9090d3c66..0000000000 --- a/tools/linter_lib/pyflakes.py +++ /dev/null @@ -1,34 +0,0 @@ -import argparse -from typing import List - -from zulint.linters import run_pyflakes - - -def check_pyflakes(files: List[str], options: argparse.Namespace) -> bool: - suppress_patterns = [ - ("scripts/lib/pythonrc.py", "imported but unused"), - # LDAP imports are necessary for docker-zulip. - ("zproject/prod_settings_template.py", "imported but unused"), - # Our ipython startup pythonrc file intentionally imports * - ("scripts/lib/pythonrc.py", " import *' used; unable to detect undefined names"), - ( - "zerver/views/realm.py", - "local variable 'message_retention_days' is assigned to but never used", - ), - ( - "zerver/views/realm.py", - "local variable 'message_content_delete_limit_seconds' is assigned to but never used", - ), - ("settings.py", "settings import *' used; unable to detect undefined names"), - ( - "settings.py", - "'from .prod_settings_template import *' used; unable to detect undefined names", - ), - ("settings.py", "settings.*' imported but unused"), - ("settings.py", "'.prod_settings_template.*' imported but unused"), - # Sphinx adds `tags` specially to the environment when running conf.py. - ("docs/conf.py", "undefined name 'tags'"), - ] - if options.full: - suppress_patterns = [] - return run_pyflakes(files, options, suppress_patterns) diff --git a/version.py b/version.py index 0a7143b097..48a54ba1be 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 154 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = (208, 0) +PROVISION_VERSION = (208, 1) diff --git a/zerver/views/realm.py b/zerver/views/realm.py index 4debe20414..f3046aa002 100644 --- a/zerver/views/realm.py +++ b/zerver/views/realm.py @@ -175,6 +175,7 @@ def update_realm( message_retention_days = parse_message_retention_days( message_retention_days_raw, Realm.MESSAGE_RETENTION_SPECIAL_VALUES_MAP ) + message_retention_days # used by locals() below if ( invite_to_realm_policy is not None or invite_required is not None diff --git a/zproject/prod_settings.pyi b/zproject/prod_settings.pyi index b63f09596c..22df20a42d 100644 --- a/zproject/prod_settings.pyi +++ b/zproject/prod_settings.pyi @@ -1 +1 @@ -from .prod_settings_template import * +from .prod_settings_template import * # noqa: F403 diff --git a/zproject/prod_settings_template.py b/zproject/prod_settings_template.py index 49fbbf3ae6..f68c94ab69 100644 --- a/zproject/prod_settings_template.py +++ b/zproject/prod_settings_template.py @@ -158,7 +158,7 @@ AUTHENTICATION_BACKENDS: Tuple[str, ...] = ( ## optionally using LDAP as an authentication mechanism. import ldap -from django_auth_ldap.config import GroupOfNamesType, LDAPGroupQuery, LDAPSearch +from django_auth_ldap.config import GroupOfNamesType, LDAPGroupQuery, LDAPSearch # noqa: F401 ## Connecting to the LDAP server. ##