zulip/tools/lint

#!/usr/bin/env python3
from __future__ import print_function
from __future__ import absolute_import
import logging
import os
import sys
import argparse
import subprocess

from linter_lib.printer import print_err, colors

# check for the venv
from lib import sanity_check
sanity_check.check_venv(__file__)

import lister
from typing import cast, Callable, Dict, Iterator, List


def run_parallel(lint_functions):
    # type: (Dict[str, Callable[[], int]]) -> bool
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            sys.stdout.flush()
            sys.stderr.flush()
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def run():
    # type: () -> None
    parser = argparse.ArgumentParser()
    parser.add_argument('--force', default=False,
                        action="store_true",
                        help='Run tests despite possible problems.')
    parser.add_argument('--full',
                        action='store_true',
                        help='Check some things we typically ignore')
    parser.add_argument('--no-gitlint',
                        action='store_true',
                        help='Disable gitlint')
    parser.add_argument('--modified', '-m',
                        action='store_true',
                        help='Only check modified files')
    parser.add_argument('--verbose', '-v',
                        action='store_true',
                        help='Print verbose timing output')
    parser.add_argument('targets',
                        nargs='*',
                        help='Specify directories to check')
    limited_tests_group = parser.add_mutually_exclusive_group()
    limited_tests_group.add_argument('--frontend',
                                     action='store_true',
                                     help='Only check files relevant to frontend')
    limited_tests_group.add_argument('--backend',
                                     action='store_true',
                                     help='Only check files relevant to backend')
    args = parser.parse_args()

    tools_dir = os.path.dirname(os.path.abspath(__file__))
    root_dir = os.path.dirname(tools_dir)
    sys.path.insert(0, root_dir)

    from tools.linter_lib.custom_check import build_custom_checkers
    from tools.linter_lib.exclude import EXCLUDED_FILES
    from tools.linter_lib.pyflakes import check_pyflakes
    from tools.linter_lib.pep8 import check_pep8

    from tools.lib.test_script import (
        get_provisioning_status,
    )

    os.chdir(root_dir)

    if not args.force:
        ok, msg = get_provisioning_status()
        if not ok:
            print(msg)
            print('If you really know what you are doing, use --force to run anyway.')
            sys.exit(1)

    backend_file_types = ['py', 'sh', 'pp', 'json', 'md', 'txt', 'text', 'yaml']
    frontend_file_types = ['js', 'css', 'handlebars', 'html']
    file_types = backend_file_types + frontend_file_types
    if args.backend:
        file_types = backend_file_types
    if args.frontend:
        file_types = frontend_file_types

    by_lang = cast(Dict[str, List[str]],
                   lister.list_files(args.targets, modified_only=args.modified,
                                     ftypes=file_types,
                                     use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES))

    # Invoke the appropriate lint checker for each language,
    # and also check files for extra whitespace.

    logging.basicConfig(format="%(asctime)s %(message)s")
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.INFO)
    else:
        logger.setLevel(logging.WARNING)

    check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)

    lint_functions = {}  # type: Dict[str, Callable[[], int]]

    def lint(func):
        # type: (Callable[[], int]) -> Callable[[], int]
        lint_functions[func.__name__] = func
        return func

    def external_linter(name, command, target_langs=[]):
        # type: (str, List[str], List[str]) -> None
        """Registers an external linter program to be run as part of the
        linter.  This program will be passed the subset of files being
        linted that have extensions in target_langs.  If there are no
        such files, exits without doing anything.

        If target_langs is empty, just runs the linter unconditionally.
        """
        color = next(colors)

        def run_linter():
            # type: () -> int
            targets = []  # type: List[str]
            if len(target_langs) != 0:
                targets = [target for lang in target_langs for target in by_lang[lang]]
                if len(targets) == 0:
                    # If we this linter has a list of languages, and
                    # no files in those languages are to be checked,
                    # then we can safely return success without
                    # invoking the external linter.
                    return 0

            p = subprocess.Popen(command + targets,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)

            assert p.stdout  # use of subprocess.PIPE indicates non-None
            for line in iter(p.stdout.readline, b''):
                print_err(name, color, line)

            return p.wait()  # Linter exit code

        lint_functions[name] = run_linter

    external_linter('add_class', ['tools/find-add-class'], ['js'])
    external_linter('css', ['tools/check-css'], ['css'])
    external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet'], ['js'])
    external_linter('tslint', ['node', 'node_modules/.bin/tslint', '-c',
                               'static/ts/tslint.json'], ['ts'])
    external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'])
    external_linter('templates', ['tools/check-templates'], ['handlebars', 'html'])
    external_linter('urls', ['tools/check-urls'], ['py'])
    external_linter('swagger', ['node', 'tools/check-swagger'], ['yaml'])

    # gitlint disabled until we can stabilize it more
    # if not args.no_gitlint:
    #     external_linter('commit_messages', ['tools/commit-message-lint'])

    @lint
    def custom_py():
        # type: () -> int
        failed = check_custom_checks_py()
        return 1 if failed else 0

    @lint
    def custom_nonpy():
        # type: () -> int
        failed = check_custom_checks_nonpy()
        return 1 if failed else 0

    @lint
    def pyflakes():
        # type: () -> int
        failed = check_pyflakes(args, by_lang)
        return 1 if failed else 0

    @lint
    def pep8():
        # type: () -> int
        failed = check_pep8(by_lang['py'])
        return 1 if failed else 0

    failed = run_parallel(lint_functions)

    sys.exit(1 if failed else 0)

if __name__ == '__main__':
    run()