zulip/tools/lint

#!/usr/bin/env python3
import argparse
import os
import random
import sys

# check for the venv
from lib import sanity_check

sanity_check.check_venv(__file__)

from zulint.command import LinterConfig, add_default_linter_arguments

from linter_lib.custom_check import non_py_rules, python_rules


def run() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument('--force', action="store_true", help='Run tests despite possible problems.')
    parser.add_argument('--full', action='store_true', help='Check some things we typically ignore')
    add_default_linter_arguments(parser)
    args = parser.parse_args()

    tools_dir = os.path.dirname(os.path.abspath(__file__))
    root_dir = os.path.dirname(tools_dir)
    sys.path.insert(0, root_dir)

    from tools.lib.test_script import assert_provisioning_status_ok
    from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
    from tools.linter_lib.pep8 import check_pep8
    from tools.linter_lib.pyflakes import check_pyflakes

    os.chdir(root_dir)

    assert_provisioning_status_ok(args.force)

    # Invoke the appropriate lint checker for each language,
    # and also check files for extra whitespace.

    linter_config = LinterConfig(args)

    by_lang = linter_config.list_files(
        groups={
            'backend': ['py', 'sh', 'pp', 'json', 'md', 'txt', 'text', 'yaml', 'rst', 'yml'],
            'frontend': ['js', 'ts', 'css', 'hbs', 'html', 'lock'],
        },
        exclude=EXCLUDED_FILES,
    )

    linter_config.external_linter(
        'css',
        ['node', 'node_modules/.bin/stylelint'],
        ['css'],
        fix_arg='--fix',
        description="Standard CSS style and formatting linter (config: stylelint.config.js)",
    )
    linter_config.external_linter(
        'eslint',
        ['node', 'node_modules/.bin/eslint', '--max-warnings=0', '--cache', '--ext', '.js,.ts'],
        ['js', 'ts'],
        fix_arg='--fix',
        description="Standard JavaScript style and formatting linter (config: .eslintrc).",
    )
    linter_config.external_linter(
        'puppet',
        ['puppet', 'parser', 'validate'],
        ['pp'],
        description="Runs the puppet parser validator, checking for syntax errors.",
    )
    linter_config.external_linter(
        'puppet-lint',
        ['puppet-lint', '--fail-on-warnings', *PUPPET_CHECK_RULES_TO_EXCLUDE],
        ['pp'],
        fix_arg='--fix',
        description="Standard puppet linter (config: tools/linter_lib/exclude.py)",
    )
    linter_config.external_linter(
        'templates',
        ['tools/check-templates'],
        ['hbs', 'html'],
        description="Custom linter checks whitespace formatting of HTML templates",
        fix_arg='--fix',
    )
    linter_config.external_linter(
        'openapi',
        ['node', 'tools/check-openapi'],
        ['yaml'],
        description="Validates our OpenAPI/Swagger API documentation "
        "(zerver/openapi/zulip.yaml) ",
    )
    linter_config.external_linter(
        'shellcheck',
        ['shellcheck', '-x', '-P', 'SCRIPTDIR'],
        ['sh'],
        description="Standard shell script linter",
    )
    linter_config.external_linter(
        'shfmt',
        ['shfmt'],
        ['sh'],
        check_arg='-d',
        fix_arg='-w',
        description="Formats shell scripts",
    )
    command = ['tools/run-mypy', '--quiet']
    if args.force:
        command.append('--force')
    linter_config.external_linter(
        'mypy',
        command,
        ['py'],
        pass_targets=False,
        description="Static type checker for Python (config: mypy.ini)",
    )
    linter_config.external_linter(
        'tsc',
        ['tools/run-tsc'],
        ['ts'],
        pass_targets=False,
        description="TypeScript compiler (config: tsconfig.json)",
    )
    linter_config.external_linter(
        'yarn-deduplicate',
        ['tools/run-yarn-deduplicate'],
        ['lock'],
        pass_targets=False,
        description="Shares duplicate packages in yarn.lock",
    )
    linter_config.external_linter(
        'gitlint',
        ['tools/commit-message-lint'],
        description="Checks commit messages for common formatting errors (config: .gitlint)",
    )
    linter_config.external_linter(
        'isort',
        ['isort'],
        ['py'],
        description="Sorts Python import statements",
        check_arg=['--check-only', '--diff'],
    )
    linter_config.external_linter(
        'prettier',
        ['node_modules/.bin/prettier', '--check', '--loglevel=warn'],
        ['css', 'js', 'json', 'ts', 'yaml', 'yml'],
        fix_arg=['--write'],
        description="Formats CSS, JavaScript, YAML",
    )

    semgrep_command = [
        "semgrep",
        "--config=./tools/semgrep.yml",
        "--error",
        "--disable-version-check",
        "--quiet",
        # This option is dangerous in the context of running
        # semgrep-as-a-service on untrusted user code, since it
        # causes Python code in the rules configuration to be
        # executed.  From our standpoint, it is required for
        # `pattern-where-python` rules, and there's no real
        # security impact, since if you can put arbitrary code
        # into zulip.git, you can run arbitrary code in a Zulip
        # development environment anyway.
        "--dangerously-allow-arbitrary-code-execution-from-rules",
    ]
    linter_config.external_linter(
        'semgrep-py',
        [*semgrep_command, "--lang=python"],
        ['py'],
        fix_arg='--autofix',
        description="Syntactic Grep (semgrep) Code Search Tool (config: ./tools/semgrep.yml)",
    )

    linter_config.external_linter(
        'thirdparty',
        ['tools/check-thirdparty'],
        description="Check docs/THIRDPARTY copyright file syntax",
    )

    @linter_config.lint
    def custom_py() -> int:
        """Runs custom checks for python files (config: tools/linter_lib/custom_check.py)"""
        failed = python_rules.check(by_lang, verbose=args.verbose)
        return 1 if failed else 0

    @linter_config.lint
    def custom_nonpy() -> int:
        """Runs custom checks for non-python files (config: tools/linter_lib/custom_check.py)"""
        failed = False
        for rule in non_py_rules:
            failed = failed or rule.check(by_lang, verbose=args.verbose)
        return 1 if failed else 0

    @linter_config.lint
    def pyflakes() -> int:
        """Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)"""
        failed = check_pyflakes(by_lang['py'], args)
        return 1 if failed else 0

    python_part1 = {x for x in by_lang['py'] if random.randint(0, 1) == 0}
    python_part2 = {y for y in by_lang['py'] if y not in python_part1}

    @linter_config.lint
    def pep8_1of2() -> int:
        """Standard Python style linter on 50% of files (config: setup.cfg)"""
        failed = check_pep8(list(python_part1))
        return 1 if failed else 0

    @linter_config.lint
    def pep8_2of2() -> int:
        """Standard Python style linter on other 50% of files (config: setup.cfg)"""
        failed = check_pep8(list(python_part2))
        return 1 if failed else 0

    linter_config.do_lint()


if __name__ == '__main__':
    run()