zulip/tools/lint

#!/usr/bin/env python
from __future__ import print_function
from __future__ import absolute_import
from contextlib import contextmanager
import logging
import os
import sys
import optparse
import subprocess

# check for the venv
from lib import sanity_check
sanity_check.check_venv(__file__)

import lister
from typing import cast, Any, Callable, Dict, Iterator, List, Optional, Tuple

# Exclude some directories and files from lint checking
EXCLUDED_FILES = [
    # Third-party code that doesn't match our style
    "api/integrations/perforce/git_p4.py",
    "puppet/apt/.forge-release",
    "puppet/apt/README.md",
    "static/third",
    # Transifex syncs translation.json files without trailing
    # newlines; there's nothing other than trailing newlines we'd be
    # checking for in these anyway.
    "static/locale",
]

@contextmanager
def bright_red_output():
    # type: () -> Iterator[None]
    # Make the lint output bright red
    sys.stdout.write('\x1B[1;31m')
    sys.stdout.flush()
    try:
        yield
    finally:
        # Restore normal terminal colors
        sys.stdout.write('\x1B[0m')


def check_pyflakes(options, by_lang):
    # type: (Any, Dict[str, List[str]]) -> bool
    if len(by_lang['py']) == 0:
        return False
    failed = False
    pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
                                stdout = subprocess.PIPE,
                                stderr = subprocess.PIPE,
                                universal_newlines = True)

    # pyflakes writes some output (like syntax errors) to stderr. :/
    for pipe in (pyflakes.stdout, pyflakes.stderr):
        assert(pipe is not None)  # onvince mypy that pipe cannot be None
        for ln in pipe:
            if options.full or not (
                    ('imported but unused' in ln or
                     'redefinition of unused' in ln or
                     # Our ipython startup pythonrc file intentionally imports *
                     ("scripts/lib/pythonrc.py" in ln and
                      " import *' used; unable to detect undefined names" in ln) or
                     # Special dev_settings.py import
                     "from .prod_settings_template import *" in ln or
                     ("settings.py" in ln and
                      ("settings import *' used; unable to detect undefined names" in ln or
                       "may be undefined, or defined from star imports" in ln)) or
                     ("zerver/tornado/ioloop_logging.py" in ln and
                      "redefinition of function 'instrument_tornado_ioloop'" in ln) or
                     ("zephyr_mirror_backend.py:" in ln and
                      "redefinition of unused 'simplejson' from line" in ln))):
                sys.stdout.write(ln)
                failed = True
    return failed


def check_pep8(files):
    # type: (List[str]) -> bool

    def run_pycodestyle(files, ignored_rules):
        # type: (List[str], List[str]) -> bool
        failed = False
        pep8 = subprocess.Popen(
            ['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
        for pipe in (pep8.stdout, pep8.stderr):
            assert(pipe is not None)  # convince mypy that pipe cannot be None
            for ln in pipe:
                sys.stdout.write(ln)
                failed = True
        return failed

    failed = False
    ignored_rules = [
        # Each of these rules are ignored for the explained reason.

        # "multiple spaces before operator"
        # There are several typos here, but also several instances that are
        # being used for alignment in dict keys/values using the `dict`
        # constructor. We could fix the alignment cases by switching to the `{}`
        # constructor, but it makes fixing this rule a little less
        # straightforward.
        'E221',

        # 'missing whitespace around arithmetic operator'
        # This should possibly be cleaned up, though changing some of
        # these may make the code less readable.
        'E226',

        # "unexpected spaces around keyword / parameter equals"
        # Many of these should be fixed, but many are also being used for
        # alignment/making the code easier to read.
        'E251',

        # "block comment should start with '#'"
        # These serve to show which lines should be changed in files customized
        # by the user. We could probably resolve one of E265 or E266 by
        # standardizing on a single style for lines that the user might want to
        # change.
        'E265',

        # "too many leading '#' for block comment"
        # Most of these are there for valid reasons.
        'E266',

        # "expected 2 blank lines after class or function definition"
        # Zulip only uses 1 blank line after class/function
        # definitions; the PEP-8 recommendation results in super sparse code.
        'E302', 'E305',

        # "module level import not at top of file"
        # Most of these are there for valid reasons, though there might be a
        # few that could be eliminated.
        'E402',

        # "line too long"
        # Zulip is a bit less strict about line length, and has its
        # own check for this (see max_length)
        'E501',

        # "do not assign a lambda expression, use a def"
        # Fixing these would probably reduce readability in most cases.
        'E731',
    ]

    # TODO: Clear up this list of violations.
    IGNORE_FILES_PEPE261 = [
        'api/zulip/__init__.py',
        'tools/run-dev.py',
        'zerver/lib/bugdown/__init__.py',
        'zerver/models.py',
        'zerver/tests/test_bugdown.py',
        'zerver/tests/test_events.py',
        'zerver/tests/test_messages.py',
        'zerver/tests/test_narrow.py',
        'zerver/tests/test_outgoing_webhook_system.py',
        'zerver/tests/test_realm.py',
        'zerver/tests/test_signup.py',
        'zerver/tests/test_subs.py',
        'zerver/tests/test_upload.py',
        'zerver/tornado/socket.py',
        'zerver/tornado/websocket_client.py',
        'zerver/worker/queue_processors.py',
        'zilencer/management/commands/populate_db.py',
        'zproject/dev_settings.py',
        'zproject/prod_settings_template.py',
        'zproject/settings.py',
    ]

    filtered_files = [fn for fn in files if fn not in IGNORE_FILES_PEPE261]
    filtered_files_E261 = [fn for fn in files if fn in IGNORE_FILES_PEPE261]

    if len(files) == 0:
        return False
    if not len(filtered_files) == 0:
        failed = run_pycodestyle(filtered_files, ignored_rules)
    if not len(filtered_files_E261) == 0:
        # Adding an extra ignore rule for these files since they still remain in
        # violation of PEP-E261.
        failed = run_pycodestyle(filtered_files_E261, ignored_rules + ['E261'])

    return failed


def run_parallel(lint_functions):
    # type: (Dict[str, Callable[[], int]]) -> bool
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            sys.stdout.flush()
            sys.stderr.flush()
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def run():
    # type: () -> None
    parser = optparse.OptionParser()
    parser.add_option('--force', default=False,
                      action="store_true",
                      help='Run tests despite possible problems.')
    parser.add_option('--full',
                      action='store_true',
                      help='Check some things we typically ignore')
    parser.add_option('--pep8',
                      action='store_true',
                      help='Run the pep8 checker')
    parser.add_option('--no-gitlint',
                      action='store_true',
                      help='Disable gitlint')
    parser.add_option('--modified', '-m',
                      action='store_true',
                      help='Only check modified files')
    parser.add_option('--verbose', '-v',
                      action='store_true',
                      help='Print verbose timing output')
    (options, args) = parser.parse_args()

    tools_dir = os.path.dirname(os.path.abspath(__file__))
    root_dir = os.path.dirname(tools_dir)
    sys.path.insert(0, root_dir)

    from tools.linter_lib.custom_check import build_custom_checkers

    from tools.lib.test_script import (
        get_provisioning_status,
    )

    os.chdir(root_dir)

    if not options.force:
        ok, msg = get_provisioning_status()
        if not ok:
            print(msg)
            print('If you really know what you are doing, use --force to run anyway.')
            sys.exit(1)

    by_lang = cast(Dict[str, List[str]],
                   lister.list_files(args, modified_only=options.modified,
                                     ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars',
                                             'html', 'json', 'md', 'txt', 'text', 'yaml'],
                                     use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES))

    # Invoke the appropriate lint checker for each language,
    # and also check files for extra whitespace.

    logging.basicConfig(format="%(asctime)s %(message)s")
    logger = logging.getLogger()
    if options.verbose:
        logger.setLevel(logging.INFO)
    else:
        logger.setLevel(logging.WARNING)

    check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)

    lint_functions = {}  # type: Dict[str, Callable[[], int]]

    def lint(func):
        # type: (Callable[[], int]) -> Callable[[], int]
        lint_functions[func.__name__] = func
        return func

    def external_linter(name, command, target_langs=[]):
        # type: (str, List[str], List[str]) -> None
        """Registers an external linter program to be run as part of the
        linter.  This program will be passed the subset of files being
        linted that have extensions in target_langs.  If there are no
        such files, exits without doing anything.

        If target_langs is empty, just runs the linter unconditionally.
        """
        def run_linter():
            # type: () -> int
            if len(target_langs) == 0:
                return subprocess.call(command)
            targets = [target for lang in target_langs for target in by_lang[lang]]
            if len(targets) == 0:
                return 0
            return subprocess.call(command + targets)
        lint_functions[name] = run_linter

    with bright_red_output():
        external_linter('add_class', ['tools/find-add-class'])
        external_linter('css', ['tools/check-css'], ['css'])
        external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet'], ['js'])
        external_linter('tslint', ['node', 'node_modules/.bin/tslint', '-c',
                                   'static/ts/tslint.json'], ['ts'])
        external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'])
        external_linter('templates', ['tools/check-templates'], ['handlebars', 'html'])
        external_linter('urls', ['tools/check-urls'])
        external_linter('swagger', ['node', 'tools/check-swagger'], ['yaml'])

        # gitlint disabled until we can stabilize it more
#        if not options.no_gitlint:
#            external_linter('commit_messages', ['tools/commit-message-lint'])

        @lint
        def custom_py():
            # type: () -> int
            failed = check_custom_checks_py()
            return 1 if failed else 0

        @lint
        def custom_nonpy():
            # type: () -> int
            failed = check_custom_checks_nonpy()
            return 1 if failed else 0

        @lint
        def pyflakes():
            # type: () -> int
            failed = check_pyflakes(options, by_lang)
            return 1 if failed else 0

        if options.pep8:
            @lint
            def pep8():
                # type: () -> int
                failed = check_pep8(by_lang['py'])
                return 1 if failed else 0

        failed = run_parallel(lint_functions)

    sys.exit(1 if failed else 0)

if __name__ == '__main__':
    run()