zulip/tools/lint

#!/usr/bin/env python3
import argparse
import os
import re
import sys

# check for the venv
from lib import sanity_check

sanity_check.check_venv(__file__)

import random

from zulint.command import LinterConfig, add_default_linter_arguments

from linter_lib.custom_check import non_py_rules, python_rules


def run() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument('--force', default=False,
                        action="store_true",
                        help='Run tests despite possible problems.')
    parser.add_argument('--full',
                        action='store_true',
                        help='Check some things we typically ignore')
    add_default_linter_arguments(parser)
    args = parser.parse_args()

    tools_dir = os.path.dirname(os.path.abspath(__file__))
    root_dir = os.path.dirname(tools_dir)
    sys.path.insert(0, root_dir)

    from tools.lib.test_script import assert_provisioning_status_ok
    from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
    from tools.linter_lib.pep8 import check_pep8
    from tools.linter_lib.pyflakes import check_pyflakes

    os.chdir(root_dir)

    assert_provisioning_status_ok(args.force)

    # Invoke the appropriate lint checker for each language,
    # and also check files for extra whitespace.

    linter_config = LinterConfig(args)

    by_lang = linter_config.list_files(groups={
        'backend': ['py', 'sh', 'pp', 'json', 'md', 'txt', 'text', 'yaml', 'rst', 'yml'],
        'frontend': ['js', 'ts', 'css', 'scss', 'hbs', 'html', 'lock'],
    }, exclude=EXCLUDED_FILES)

    linter_config.external_linter('css', ['node', 'node_modules/.bin/stylelint'], ['css', 'scss'],
                                  fix_arg='--fix',
                                  description="Standard CSS style and formatting linter "
                                  "(config: .stylelintrc)")
    linter_config.external_linter('eslint', ['node', 'node_modules/.bin/eslint',
                                             '--quiet', '--cache', '--ext', '.js,.ts'], ['js', 'ts'],
                                  fix_arg='--fix',
                                  description="Standard JavaScript style and formatting linter"
                                  "(config: .eslintrc).")
    linter_config.external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'],
                                  description="Runs the puppet parser validator, "
                                  "checking for syntax errors.")
    linter_config.external_linter('puppet-lint',
                                  ['puppet-lint', '--fail-on-warnings'] + PUPPET_CHECK_RULES_TO_EXCLUDE,
                                  ['pp'],
                                  fix_arg='--fix',
                                  description="Standard puppet linter"
                                  "(config: tools/linter_lib/exclude.py)")
    linter_config.external_linter('templates', ['tools/check-templates'], ['hbs', 'html'],
                                  description="Custom linter checks whitespace formatting"
                                  "of HTML templates.",
                                  fix_arg='--fix')
    linter_config.external_linter('openapi', ['node', 'tools/check-openapi'], ['yaml'],
                                  description="Validates our OpenAPI/Swagger API documentation"
                                  "(zerver/openapi/zulip.yaml) ")
    linter_config.external_linter('shellcheck', ['shellcheck', '-x', '-P', 'SCRIPTDIR'], ['sh'],
                                  description="Standard shell script linter.")
    command = ['tools/run-mypy', '--quiet']
    if args.force:
        command.append('--force')
    linter_config.external_linter('mypy', command, ['py'], pass_targets=False,
                                  description="Static type checker for Python (config: mypy.ini)")
    linter_config.external_linter('tsc', ['tools/run-tsc'], ['ts'], pass_targets=False,
                                  description="TypeScript compiler (config: tsconfig.json)")
    linter_config.external_linter('yarn-deduplicate', ['tools/run-yarn-deduplicate'], ['lock'],
                                  pass_targets=False,
                                  description="Shares duplicate packages in yarn.lock")
    linter_config.external_linter('gitlint', ['tools/commit-message-lint'],
                                  description="Checks commit messages for common formatting errors."
                                  "(config: .gitlint)")
    linter_config.external_linter('isort', ['isort'], ['py'],
                                  description="Sorts Python import statements",
                                  check_arg=['--check-only', '--diff'])
    linter_config.external_linter('prettier', ['node_modules/.bin/prettier', '--check'],
                                  ['js', 'ts', 'yaml', 'yml'],
                                  fix_arg=['--write'],
                                  description="Formats JavaScript and YAML",
                                  # https://github.com/prettier/prettier/pull/8703
                                  suppress_line=lambda line: line in ["Checking formatting...\n", "All matched files use Prettier code style!\n"])

    semgrep_command = ["semgrep", "--config=./tools/semgrep.yml", "--error",
                       # This option is dangerous in the context of running
                       # semgrep-as-a-service on untrusted user code, since it
                       # causes Python code in the rules configuration to be
                       # executed.  From our standpoint, it is required for
                       # `pattern-where-python` rules, and there's no real
                       # security impact, since if you can put arbitrary code
                       # into zulip.git, you can run arbitrary code in a Zulip
                       # development environment anyway.
                       "--dangerously-allow-arbitrary-code-execution-from-rules"]
    linter_config.external_linter('semgrep-py', [*semgrep_command, "--lang=python"], ['py'],
                                  fix_arg='--autofix',
                                  description="Syntactic Grep (semgrep) Code Search Tool "
                                  "(config: ./tools/semgrep.yml)",
                                  # https://github.com/returntocorp/semgrep/issues/1228
                                  suppress_line=lambda line: bool(re.match(r"running \d+ rules\.\.\.$", line)))

    linter_config.external_linter('thirdparty', ['tools/check-thirdparty'],
                                  description="Check docs/THIRDPARTY copyright file syntax")

    @linter_config.lint
    def custom_py() -> int:
        """Runs custom checks for python files (config: tools/linter_lib/custom_check.py)"""
        failed = python_rules.check(by_lang, verbose=args.verbose)
        return 1 if failed else 0

    @linter_config.lint
    def custom_nonpy() -> int:
        """Runs custom checks for non-python files (config: tools/linter_lib/custom_check.py)"""
        failed = False
        for rule in non_py_rules:
            failed = failed or rule.check(by_lang, verbose=args.verbose)
        return 1 if failed else 0

    @linter_config.lint
    def pyflakes() -> int:
        """Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)"""
        failed = check_pyflakes(by_lang['py'], args)
        return 1 if failed else 0

    python_part1 = {x for x in by_lang['py'] if random.randint(0, 1) == 0}
    python_part2 = {y for y in by_lang['py'] if y not in python_part1}

    @linter_config.lint
    def pep8_1of2() -> int:
        """Standard Python style linter on 50% of files (config: setup.cfg)"""
        failed = check_pep8(list(python_part1))
        return 1 if failed else 0

    @linter_config.lint
    def pep8_2of2() -> int:
        """Standard Python style linter on other 50% of files (config: setup.cfg)"""
        failed = check_pep8(list(python_part2))
        return 1 if failed else 0

    linter_config.do_lint()

if __name__ == '__main__':
    run()