#!/usr/bin/env python3 import argparse import os import re import sys # check for the venv from lib import sanity_check sanity_check.check_venv(__file__) import random from zulint.command import LinterConfig, add_default_linter_arguments from linter_lib.custom_check import non_py_rules, python_rules def run() -> None: parser = argparse.ArgumentParser() parser.add_argument('--force', default=False, action="store_true", help='Run tests despite possible problems.') parser.add_argument('--full', action='store_true', help='Check some things we typically ignore') add_default_linter_arguments(parser) args = parser.parse_args() tools_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) from tools.lib.test_script import assert_provisioning_status_ok from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE from tools.linter_lib.pep8 import check_pep8 from tools.linter_lib.pyflakes import check_pyflakes os.chdir(root_dir) assert_provisioning_status_ok(args.force) # Invoke the appropriate lint checker for each language, # and also check files for extra whitespace. linter_config = LinterConfig(args) by_lang = linter_config.list_files(groups={ 'backend': ['py', 'sh', 'pp', 'json', 'md', 'txt', 'text', 'yaml', 'rst', 'yml'], 'frontend': ['js', 'ts', 'css', 'scss', 'hbs', 'html', 'lock'], }, exclude=EXCLUDED_FILES) linter_config.external_linter('css', ['node', 'node_modules/.bin/stylelint'], ['css', 'scss'], fix_arg='--fix', description="Standard CSS style and formatting linter " "(config: .stylelintrc)") linter_config.external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet', '--cache', '--ext', '.js,.ts'], ['js', 'ts'], fix_arg='--fix', description="Standard JavaScript style and formatting linter" "(config: .eslintrc).") linter_config.external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'], description="Runs the puppet parser validator, " "checking for syntax errors.") linter_config.external_linter('puppet-lint', ['puppet-lint', '--fail-on-warnings'] + PUPPET_CHECK_RULES_TO_EXCLUDE, ['pp'], fix_arg='--fix', description="Standard puppet linter" "(config: tools/linter_lib/exclude.py)") linter_config.external_linter('templates', ['tools/check-templates'], ['hbs', 'html'], description="Custom linter checks whitespace formatting" "of HTML templates.", fix_arg='--fix') linter_config.external_linter('openapi', ['node', 'tools/check-openapi'], ['yaml'], description="Validates our OpenAPI/Swagger API documentation" "(zerver/openapi/zulip.yaml) ") linter_config.external_linter('shellcheck', ['shellcheck', '-x', '-P', 'SCRIPTDIR'], ['sh'], description="Standard shell script linter.") command = ['tools/run-mypy', '--quiet'] if args.force: command.append('--force') linter_config.external_linter('mypy', command, ['py'], pass_targets=False, description="Static type checker for Python (config: mypy.ini)") linter_config.external_linter('tsc', ['tools/run-tsc'], ['ts'], pass_targets=False, description="TypeScript compiler (config: tsconfig.json)") linter_config.external_linter('yarn-deduplicate', ['tools/run-yarn-deduplicate'], ['lock'], pass_targets=False, description="Shares duplicate packages in yarn.lock") linter_config.external_linter('gitlint', ['tools/commit-message-lint'], description="Checks commit messages for common formatting errors." "(config: .gitlint)") linter_config.external_linter('isort', ['isort'], ['py'], description="Sorts Python import statements", check_arg=['--check-only', '--diff']) linter_config.external_linter('prettier', ['node_modules/.bin/prettier', '--check'], ['js', 'ts', 'yaml', 'yml'], fix_arg=['--write'], description="Formats JavaScript and YAML", # https://github.com/prettier/prettier/pull/8703 suppress_line=lambda line: line in ["Checking formatting...\n", "All matched files use Prettier code style!\n"]) semgrep_command = ["semgrep", "--config=./tools/semgrep.yml", "--error", # This option is dangerous in the context of running # semgrep-as-a-service on untrusted user code, since it # causes Python code in the rules configuration to be # executed. From our standpoint, it is required for # `pattern-where-python` rules, and there's no real # security impact, since if you can put arbitrary code # into zulip.git, you can run arbitrary code in a Zulip # development environment anyway. "--dangerously-allow-arbitrary-code-execution-from-rules"] linter_config.external_linter('semgrep-py', [*semgrep_command, "--lang=python"], ['py'], fix_arg='--autofix', description="Syntactic Grep (semgrep) Code Search Tool " "(config: ./tools/semgrep.yml)", # https://github.com/returntocorp/semgrep/issues/1228 suppress_line=lambda line: bool(re.match(r"running \d+ rules\.\.\.$", line))) linter_config.external_linter('thirdparty', ['tools/check-thirdparty'], description="Check docs/THIRDPARTY copyright file syntax") @linter_config.lint def custom_py() -> int: """Runs custom checks for python files (config: tools/linter_lib/custom_check.py)""" failed = python_rules.check(by_lang, verbose=args.verbose) return 1 if failed else 0 @linter_config.lint def custom_nonpy() -> int: """Runs custom checks for non-python files (config: tools/linter_lib/custom_check.py)""" failed = False for rule in non_py_rules: failed = failed or rule.check(by_lang, verbose=args.verbose) return 1 if failed else 0 @linter_config.lint def pyflakes() -> int: """Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)""" failed = check_pyflakes(by_lang['py'], args) return 1 if failed else 0 python_part1 = {x for x in by_lang['py'] if random.randint(0, 1) == 0} python_part2 = {y for y in by_lang['py'] if y not in python_part1} @linter_config.lint def pep8_1of2() -> int: """Standard Python style linter on 50% of files (config: setup.cfg)""" failed = check_pep8(list(python_part1)) return 1 if failed else 0 @linter_config.lint def pep8_2of2() -> int: """Standard Python style linter on other 50% of files (config: setup.cfg)""" failed = check_pep8(list(python_part2)) return 1 if failed else 0 linter_config.do_lint() if __name__ == '__main__': run()