zulip/tools/lint

#!/usr/bin/env python3
import argparse
import os
import random
import re
import sys

# check for the venv
from lib import sanity_check

sanity_check.check_venv(__file__)

from zulint.command import LinterConfig, add_default_linter_arguments

from linter_lib.custom_check import non_py_rules, python_rules


def run() -> None:
    tools_dir = os.path.dirname(os.path.abspath(__file__))
    root_dir = os.path.dirname(tools_dir)
    sys.path.insert(0, root_dir)

    from tools.lib.test_script import (
        add_provision_check_override_param,
        assert_provisioning_status_ok,
    )
    from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
    from tools.linter_lib.pep8 import check_pep8
    from tools.linter_lib.pyflakes import check_pyflakes

    parser = argparse.ArgumentParser()
    add_provision_check_override_param(parser)
    parser.add_argument("--full", action="store_true", help="Check some things we typically ignore")
    add_default_linter_arguments(parser)
    args = parser.parse_args()

    os.chdir(root_dir)

    assert_provisioning_status_ok(args.skip_provision_check)

    # Invoke the appropriate lint checker for each language,
    # and also check files for extra whitespace.

    linter_config = LinterConfig(args)

    by_lang = linter_config.list_files(
        groups={
            "backend": ["py", "sh", "pp", "json", "md", "txt", "text", "yaml", "rst", "yml"],
            "frontend": ["js", "ts", "flow", "css", "hbs", "html", "lock"],
        },
        exclude=EXCLUDED_FILES,
    )

    linter_config.external_linter(
        "css",
        ["node", "node_modules/.bin/stylelint"],
        ["css"],
        fix_arg="--fix",
        description="Standard CSS style and formatting linter (config: stylelint.config.js)",
    )
    linter_config.external_linter(
        "eslint",
        ["node", "node_modules/.bin/eslint", "--max-warnings=0", "--cache", "--ext", ".js,.ts"],
        ["js", "ts"],
        fix_arg="--fix",
        description="Standard JavaScript style and formatting linter (config: .eslintrc).",
    )
    linter_config.external_linter(
        "puppet",
        ["puppet", "parser", "validate"],
        ["pp"],
        description="Runs the puppet parser validator, checking for syntax errors.",
    )
    linter_config.external_linter(
        "puppet-lint",
        ["puppet-lint", "--fail-on-warnings", *PUPPET_CHECK_RULES_TO_EXCLUDE],
        ["pp"],
        fix_arg="--fix",
        description="Standard puppet linter (config: tools/linter_lib/exclude.py)",
    )
    linter_config.external_linter(
        "templates",
        ["tools/check-templates"],
        ["hbs", "html"],
        description="Custom linter checks whitespace formatting of HTML templates",
        fix_arg="--fix",
    )
    linter_config.external_linter(
        "openapi",
        ["node", "tools/check-openapi"],
        ["yaml"],
        description="Validates our OpenAPI/Swagger API documentation "
        "(zerver/openapi/zulip.yaml) ",
    )
    linter_config.external_linter(
        "shellcheck",
        ["shellcheck", "-x", "-P", "SCRIPTDIR"],
        ["sh"],
        description="Standard shell script linter",
    )
    linter_config.external_linter(
        "shfmt",
        ["shfmt"],
        ["sh"],
        check_arg="-d",
        fix_arg="-w",
        description="Formats shell scripts",
    )
    command = ["tools/run-mypy", "--quiet"]
    if args.skip_provision_check:
        command.append("--skip-provision-check")
    linter_config.external_linter(
        "mypy",
        command,
        ["py"],
        pass_targets=False,
        description="Static type checker for Python (config: mypy.ini)",
    )
    linter_config.external_linter(
        "tsc",
        ["tools/run-tsc"],
        ["ts"],
        pass_targets=False,
        description="TypeScript compiler (config: tsconfig.json)",
    )
    linter_config.external_linter(
        "yarn-deduplicate",
        ["tools/run-yarn-deduplicate"],
        ["lock"],
        pass_targets=False,
        description="Shares duplicate packages in yarn.lock",
    )
    linter_config.external_linter(
        "gitlint",
        ["tools/commit-message-lint"],
        description="Checks commit messages for common formatting errors (config: .gitlint)",
    )
    linter_config.external_linter(
        "isort",
        ["isort"],
        ["py"],
        description="Sorts Python import statements",
        check_arg=["--check-only", "--diff"],
    )
    linter_config.external_linter(
        "prettier",
        ["node_modules/.bin/prettier", "--check", "--loglevel=warn"],
        ["css", "flow", "js", "json", "ts", "yaml", "yml"],
        fix_arg=["--write"],
        description="Formats CSS, JavaScript, YAML",
    )
    linter_config.external_linter(
        "black",
        ["black"],
        ["py"],
        description="Reformats Python code",
        check_arg=["--check"],
        suppress_line=lambda line: line == "All done! ✨ 🍰 ✨\n"
        or re.fullmatch(r"\d+ files? would be left unchanged\.\n", line) is not None,
    )

    semgrep_command = [
        "semgrep",
        "--config=./tools/semgrep.yml",
        "--error",
        "--disable-version-check",
        "--quiet",
        # This option is dangerous in the context of running
        # semgrep-as-a-service on untrusted user code, since it
        # causes Python code in the rules configuration to be
        # executed.  From our standpoint, it is required for
        # `pattern-where-python` rules, and there's no real
        # security impact, since if you can put arbitrary code
        # into zulip.git, you can run arbitrary code in a Zulip
        # development environment anyway.
        "--dangerously-allow-arbitrary-code-execution-from-rules",
    ]
    linter_config.external_linter(
        "semgrep-py",
        [*semgrep_command, "--lang=python"],
        ["py"],
        fix_arg="--autofix",
        description="Syntactic Grep (semgrep) Code Search Tool (config: ./tools/semgrep.yml)",
    )

    linter_config.external_linter(
        "thirdparty",
        ["tools/check-thirdparty"],
        description="Check docs/THIRDPARTY copyright file syntax",
    )

    @linter_config.lint
    def custom_py() -> int:
        """Runs custom checks for python files (config: tools/linter_lib/custom_check.py)"""
        failed = python_rules.check(by_lang, verbose=args.verbose)
        return 1 if failed else 0

    @linter_config.lint
    def custom_nonpy() -> int:
        """Runs custom checks for non-python files (config: tools/linter_lib/custom_check.py)"""
        failed = False
        for rule in non_py_rules:
            failed = failed or rule.check(by_lang, verbose=args.verbose)
        return 1 if failed else 0

    @linter_config.lint
    def pyflakes() -> int:
        """Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)"""
        failed = check_pyflakes(by_lang["py"], args)
        return 1 if failed else 0

    python_part1 = {x for x in by_lang["py"] if random.randint(0, 1) == 0}
    python_part2 = {y for y in by_lang["py"] if y not in python_part1}

    @linter_config.lint
    def pep8_1of2() -> int:
        """Standard Python style linter on 50% of files (config: setup.cfg)"""
        failed = check_pep8(list(python_part1))
        return 1 if failed else 0

    @linter_config.lint
    def pep8_2of2() -> int:
        """Standard Python style linter on other 50% of files (config: setup.cfg)"""
        failed = check_pep8(list(python_part2))
        return 1 if failed else 0

    linter_config.do_lint()


if __name__ == "__main__":
    run()