#!/usr/bin/env python3 from __future__ import print_function from __future__ import absolute_import import logging import os import sys import argparse import subprocess from linter_lib.printer import print_err, colors # check for the venv from lib import sanity_check sanity_check.check_venv(__file__) import lister from typing import cast, Callable, Dict, Iterator, List def run_parallel(lint_functions): # type: (Dict[str, Callable[[], int]]) -> bool pids = [] for name, func in lint_functions.items(): pid = os.fork() if pid == 0: logging.info("start " + name) result = func() logging.info("finish " + name) sys.stdout.flush() sys.stderr.flush() os._exit(result) pids.append(pid) failed = False for pid in pids: (_, status) = os.waitpid(pid, 0) if status != 0: failed = True return failed def run(): # type: () -> None parser = argparse.ArgumentParser() parser.add_argument('--force', default=False, action="store_true", help='Run tests despite possible problems.') parser.add_argument('--full', action='store_true', help='Check some things we typically ignore') parser.add_argument('--no-gitlint', action='store_true', help='Disable gitlint') parser.add_argument('--modified', '-m', action='store_true', help='Only check modified files') parser.add_argument('--verbose', '-v', action='store_true', help='Print verbose timing output') parser.add_argument('targets', nargs='*', help='Specify directories to check') limited_tests_group = parser.add_mutually_exclusive_group() limited_tests_group.add_argument('--frontend', action='store_true', help='Only check files relevant to frontend') limited_tests_group.add_argument('--backend', action='store_true', help='Only check files relevant to backend') args = parser.parse_args() tools_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) from tools.linter_lib.custom_check import build_custom_checkers from tools.linter_lib.exclude import EXCLUDED_FILES from tools.linter_lib.pyflakes import check_pyflakes from tools.linter_lib.pep8 import check_pep8 from tools.lib.test_script import ( get_provisioning_status, ) os.chdir(root_dir) if not args.force: ok, msg = get_provisioning_status() if not ok: print(msg) print('If you really know what you are doing, use --force to run anyway.') sys.exit(1) backend_file_types = ['py', 'sh', 'pp', 'json', 'md', 'txt', 'text', 'yaml', 'rst'] frontend_file_types = ['js', 'css', 'handlebars', 'html'] file_types = backend_file_types + frontend_file_types if args.backend: file_types = backend_file_types if args.frontend: file_types = frontend_file_types by_lang = cast(Dict[str, List[str]], lister.list_files(args.targets, modified_only=args.modified, ftypes=file_types, use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES)) # Invoke the appropriate lint checker for each language, # and also check files for extra whitespace. logging.basicConfig(format="%(asctime)s %(message)s") logger = logging.getLogger() if args.verbose: logger.setLevel(logging.INFO) else: logger.setLevel(logging.WARNING) check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang) lint_functions = {} # type: Dict[str, Callable[[], int]] def lint(func): # type: (Callable[[], int]) -> Callable[[], int] lint_functions[func.__name__] = func return func def external_linter(name, command, target_langs=[]): # type: (str, List[str], List[str]) -> None """Registers an external linter program to be run as part of the linter. This program will be passed the subset of files being linted that have extensions in target_langs. If there are no such files, exits without doing anything. If target_langs is empty, just runs the linter unconditionally. """ color = next(colors) def run_linter(): # type: () -> int targets = [] # type: List[str] if len(target_langs) != 0: targets = [target for lang in target_langs for target in by_lang[lang]] if len(targets) == 0: # If we this linter has a list of languages, and # no files in those languages are to be checked, # then we can safely return success without # invoking the external linter. return 0 p = subprocess.Popen(command + targets, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) assert p.stdout # use of subprocess.PIPE indicates non-None for line in iter(p.stdout.readline, b''): print_err(name, color, line) return p.wait() # Linter exit code lint_functions[name] = run_linter external_linter('add_class', ['tools/find-add-class'], ['js']) external_linter('css', ['tools/check-css'], ['css']) external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet'], ['js']) external_linter('tslint', ['node', 'node_modules/.bin/tslint', '-c', 'static/ts/tslint.json'], ['ts']) external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp']) external_linter('templates', ['tools/check-templates'], ['handlebars', 'html']) external_linter('urls', ['tools/check-urls'], ['py']) external_linter('swagger', ['node', 'tools/check-swagger'], ['yaml']) # gitlint disabled until we can stabilize it more # if not args.no_gitlint: # external_linter('commit_messages', ['tools/commit-message-lint']) @lint def custom_py(): # type: () -> int failed = check_custom_checks_py() return 1 if failed else 0 @lint def custom_nonpy(): # type: () -> int failed = check_custom_checks_nonpy() return 1 if failed else 0 @lint def pyflakes(): # type: () -> int failed = check_pyflakes(args, by_lang) return 1 if failed else 0 @lint def pep8(): # type: () -> int failed = check_pep8(by_lang['py']) return 1 if failed else 0 failed = run_parallel(lint_functions) sys.exit(1 if failed else 0) if __name__ == '__main__': run()