zulip/tools/lint

#!/usr/bin/env python
from __future__ import print_function
from __future__ import absolute_import
import logging
import os
import sys
import optparse
import subprocess

from linter_lib.printer import print_err, colors

# check for the venv
from lib import sanity_check
sanity_check.check_venv(__file__)

import lister
from typing import cast, Callable, Dict, Iterator, List


def run_parallel(lint_functions):
    # type: (Dict[str, Callable[[], int]]) -> bool
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            sys.stdout.flush()
            sys.stderr.flush()
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def run():
    # type: () -> None
    parser = optparse.OptionParser()
    parser.add_option('--force', default=False,
                      action="store_true",
                      help='Run tests despite possible problems.')
    parser.add_option('--full',
                      action='store_true',
                      help='Check some things we typically ignore')
    parser.add_option('--pep8',
                      action='store_true',
                      help='Run the pep8 checker')
    parser.add_option('--no-gitlint',
                      action='store_true',
                      help='Disable gitlint')
    parser.add_option('--modified', '-m',
                      action='store_true',
                      help='Only check modified files')
    parser.add_option('--verbose', '-v',
                      action='store_true',
                      help='Print verbose timing output')
    (options, args) = parser.parse_args()

    tools_dir = os.path.dirname(os.path.abspath(__file__))
    root_dir = os.path.dirname(tools_dir)
    sys.path.insert(0, root_dir)

    from tools.linter_lib.custom_check import build_custom_checkers
    from tools.linter_lib.exclude import EXCLUDED_FILES
    from tools.linter_lib.pyflakes import check_pyflakes
    from tools.linter_lib.pep8 import check_pep8

    from tools.lib.test_script import (
        get_provisioning_status,
    )

    os.chdir(root_dir)

    if not options.force:
        ok, msg = get_provisioning_status()
        if not ok:
            print(msg)
            print('If you really know what you are doing, use --force to run anyway.')
            sys.exit(1)

    by_lang = cast(Dict[str, List[str]],
                   lister.list_files(args, modified_only=options.modified,
                                     ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars',
                                             'html', 'json', 'md', 'txt', 'text', 'yaml'],
                                     use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES))

    # Invoke the appropriate lint checker for each language,
    # and also check files for extra whitespace.

    logging.basicConfig(format="%(asctime)s %(message)s")
    logger = logging.getLogger()
    if options.verbose:
        logger.setLevel(logging.INFO)
    else:
        logger.setLevel(logging.WARNING)

    check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)

    lint_functions = {}  # type: Dict[str, Callable[[], int]]

    def lint(func):
        # type: (Callable[[], int]) -> Callable[[], int]
        lint_functions[func.__name__] = func
        return func

    def external_linter(name, command, target_langs=[]):
        # type: (str, List[str], List[str]) -> None
        """Registers an external linter program to be run as part of the
        linter.  This program will be passed the subset of files being
        linted that have extensions in target_langs.  If there are no
        such files, exits without doing anything.

        If target_langs is empty, just runs the linter unconditionally.
        """
        color = next(colors)

        def run_linter():
            # type: () -> int
            targets = []  # type: List[str]
            if len(target_langs) != 0:
                targets = [target for lang in target_langs for target in by_lang[lang]]
                if len(targets) == 0:
                    # If we this linter has a list of languages, and
                    # no files in those languages are to be checked,
                    # then we can safely return success without
                    # invoking the external linter.
                    return 0

            p = subprocess.Popen(command + targets,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)

            assert p.stdout  # use of subprocess.PIPE indicates non-None
            for line in iter(p.stdout.readline, b''):
                print_err(name, color, line)

            return p.wait()  # Linter exit code

        lint_functions[name] = run_linter

    external_linter('add_class', ['tools/find-add-class'])
    external_linter('css', ['tools/check-css'], ['css'])
    external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet'], ['js'])
    external_linter('tslint', ['node', 'node_modules/.bin/tslint', '-c',
                               'static/ts/tslint.json'], ['ts'])
    external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'])
    external_linter('templates', ['tools/check-templates'], ['handlebars', 'html'])
    external_linter('urls', ['tools/check-urls'])
    external_linter('swagger', ['node', 'tools/check-swagger'], ['yaml'])

    # gitlint disabled until we can stabilize it more
    # if not options.no_gitlint:
    #     external_linter('commit_messages', ['tools/commit-message-lint'])

    @lint
    def custom_py():
        # type: () -> int
        failed = check_custom_checks_py()
        return 1 if failed else 0

    @lint
    def custom_nonpy():
        # type: () -> int
        failed = check_custom_checks_nonpy()
        return 1 if failed else 0

    @lint
    def pyflakes():
        # type: () -> int
        failed = check_pyflakes(options, by_lang)
        return 1 if failed else 0

    if options.pep8:
        @lint
        def pep8():
            # type: () -> int
            failed = check_pep8(by_lang['py'])
            return 1 if failed else 0

    failed = run_parallel(lint_functions)

    sys.exit(1 if failed else 0)

if __name__ == '__main__':
    run()