zulip/tools/lint-all

#!/usr/bin/env python2.7
from __future__ import print_function
from __future__ import absolute_import
import os
import re
import sys
import optparse
import subprocess
import traceback

from os          import path
from collections import defaultdict
from six.moves import filter
from six.moves import map

import lister

parser = optparse.OptionParser()
parser.add_option('--full',
    action='store_true',
    help='Check some things we typically ignore')
parser.add_option('--modified', '-m',
    action='store_true',
    help='Only check modified files')
(options, args) = parser.parse_args()

os.chdir(path.join(path.dirname(__file__), '..'))


# Exclude some directories and files from lint checking

exclude = """
static/third
confirmation
frontend_tests/casperjs
zerver/migrations
node_modules
docs/html_unescape.py
zproject/test_settings.py
zproject/settings.py
tools/jslint/jslint.js
api/setup.py
api/integrations/perforce/git_p4.py
puppet/apt/.forge-release
puppet/puppet-common/tests/
""".split()

by_lang = lister.list_files(args, modified_only=options.modified, use_shebang=True,
    ftypes=['py', 'sh', 'js', 'pp'], group_by_ftype=True, exclude=exclude)

# Invoke the appropriate lint checker for each language,
# and also check files for extra whitespace.

import logging
logging.basicConfig(format="%(asctime)s %(message)s")
logger = logging.getLogger()
# Change this to logging.INFO to see performance data
logger.setLevel(logging.WARNING)

def check_pyflakes():
    if not by_lang['py']:
        return False
    failed = False
    pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE)

    # pyflakes writes some output (like syntax errors) to stderr. :/
    for pipe in (pyflakes.stdout, pyflakes.stderr):
        for ln in pipe:
            if options.full or not \
                    ('imported but unused' in ln or
                     'redefinition of unused' in ln or
                     "'from typing import *' used; unable to detect undefined names" in ln or
                     ("zerver/models.py" in ln and
                      " undefined name 'bugdown'" in ln) or
                     ("scripts/lib/pythonrc.py" in ln and
                      " import *' used; unable to detect undefined names" in ln) or
                     ("zerver/lib/tornado_ioloop_logging.py" in ln and
                      "redefinition of function 'instrument_tornado_ioloop'" in ln) or
                     ("zephyr_mirror_backend.py:" in ln and
                      "redefinition of unused 'simplejson' from line" in ln)):
                sys.stdout.write(ln)
                failed = True
    return failed

def custom_check_file(fn, rules, skip_rules=[]):
    failed = False
    lineFlag = False
    for i, line in enumerate(open(fn)):
        skip = False
        lineFlag = True
        for rule in skip_rules:
            if re.match(rule, line):
                skip = True
        if skip:
            continue
        for rule in rules:
            exclude_list = rule.get('exclude', set())
            if fn in exclude_list:
                continue
            try:
                if re.search(rule['pattern'], line.strip(rule.get('strip', None))):
                    sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1))
                    print(line)
                    failed = True
            except Exception:
                print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
                traceback.print_exc()
        lastLine = line
    if lineFlag and '\n' not in lastLine:
        print("No new line at the end of %s" % (fn,))
        failed = True
    return failed

whitespace_rules = [
    {'pattern': '\s+$',
     'strip': '\n',
     'description': 'Fix trailing whitespace'},
    {'pattern': '\t',
     'strip': '\n',
     'exclude': set(['zerver/lib/bugdown/codehilite.py']),
     'description': 'Fix tab-based whitespace'},
    ]
js_rules = [
    {'pattern': '[^_]function\(',
     'description': 'The keyword "function" should be followed by a space'},
    {'pattern': '.*blueslip.warning\(.*',
     'description': 'The module blueslip has no function warning, try using blueslip.warn'},
    {'pattern': '[)]{$',
     'description': 'Missing space between ) and {'},
    {'pattern': 'else{$',
     'description': 'Missing space between else and {'},
    ] + whitespace_rules
python_rules = [
    {'pattern': "'[^']*'\s+\([^']*$",
     'description': "Suspicious code with quoting around function name"},
    {'pattern': '"[^"]*"\s+\([^"]*$',
     'description': "Suspicious code with quoting around function name"},
    {'pattern': '".*"%\([a-z_].*\)?$',
     'description': 'Missing space around "%"'},
    {'pattern': "'.*'%\([a-z_].*\)?$",
     'description': 'Missing space around "%"'},
    # This next check could have false positives, but it seems pretty
    # rare; if we find any, they can be added to the exclude list for
    # this rule.
    {'pattern': '% [a-zA-Z0-9_.]*\)?$',
     'description': 'Used % comprehension without a tuple'},
    ] + whitespace_rules
python_line_skip_rules = [
    '\s*[*#]', # comments
    ]
bash_rules = [
    {'pattern': '#!.*sh [-xe]',
     'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'},
    ]

def check_custom_checks():
    failed = False

    for fn in by_lang['py']:
        if custom_check_file(fn, python_rules, skip_rules=python_line_skip_rules):
            failed = True

    for fn in by_lang['js']:
        if custom_check_file(fn, js_rules):
            failed = True

    for fn in by_lang['sh']:
        if custom_check_file(fn, bash_rules):
            failed = True

    return failed

lint_functions = {}

def run_parallel():
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def lint(func):
    lint_functions[func.__name__] = func
    return func

try:
    # Make the lint output bright red
    sys.stdout.write('\x1B[1;31m')
    sys.stdout.flush()

    @lint
    def templates():
        result = subprocess.call(['tools/check-templates'])
        return result

    @lint
    def jslint():
        result = subprocess.call(['tools/node', 'tools/jslint/check-all.js']
                                 + by_lang['js'])
        return result

    @lint
    def puppet():
        if not by_lang['pp']:
            return 0
        result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['pp'])
        return result

    @lint
    def custom():
        failed = check_custom_checks()
        return 1 if failed else 0

    @lint
    def pyflakes():
        failed = check_pyflakes()
        return 1 if failed else 0

    failed = run_parallel()

    sys.exit(1 if failed else 0)

finally:
    # Restore normal terminal colors
    sys.stdout.write('\x1B[0m')