zulip/tools/lint-all

#!/usr/bin/env python2.7
from __future__ import print_function
from __future__ import absolute_import
import os
import re
import sys
import optparse
import subprocess
import traceback

from os          import path
from collections import defaultdict
from six.moves import filter
from six.moves import map

parser = optparse.OptionParser()
parser.add_option('--full',
    action='store_true',
    help='Check some things we typically ignore')
parser.add_option('--modified', '-m',
    action='store_true',
    help='Only check modified files')
(options, args) = parser.parse_args()

os.chdir(path.join(path.dirname(__file__), '..'))


# Exclude some directories and files from lint checking

exclude_trees = """
static/third
confirmation
frontend_tests/casperjs
zerver/migrations
node_modules
""".split()

exclude_files = """
docs/html_unescape.py
zproject/test_settings.py
zproject/settings.py
tools/jslint/jslint.js
api/setup.py
api/integrations/perforce/git_p4.py
""".split()

if options.modified:
    # If the user specifies, use `git ls-files -m` to only check modified, non-staged
    # files in the current checkout.  This makes things fun faster.
    files = list(map(str.strip, subprocess.check_output(['git', 'ls-files', '-m']).split('\n')))
else:
    files = []

files += args

if not files and not options.modified:
    # If no files are specified on the command line, use the entire git checkout
    files = list(map(str.strip, subprocess.check_output(['git', 'ls-files']).split('\n')))

files = list(filter(bool, files)) # remove empty file caused by trailing \n

if not files:
    raise Exception('There are no files to check!')

# Categorize by language all files we want to check
by_lang   = defaultdict(list)

for filepath in files:
    if (not filepath or not path.isfile(filepath)
        or (filepath in exclude_files)
        or any(filepath.startswith(d+'/') for d in exclude_trees)):
        continue

    _, exn = path.splitext(filepath)
    if not exn:
        # No extension; look at the first line
        with open(filepath) as f:
            if re.match(r'^#!.*\bpython', f.readline()):
                exn = '.py'

    by_lang[exn].append(filepath)

by_lang['.sh'] = [_f for _f in map(str.strip, subprocess.check_output("grep --files-with-matches '#!.*\(ba\)\?sh' $(git ls-tree --name-only -r HEAD scripts/ tools/ bin/ | grep -v [.])", shell=True).split('\n')) if _f]

# Invoke the appropriate lint checker for each language,
# and also check files for extra whitespace.

import logging
logging.basicConfig(format="%(asctime)s %(message)s")
logger = logging.getLogger()
# Change this to logging.INFO to see performance data
logger.setLevel(logging.WARNING)

def check_pyflakes():
    if not by_lang['.py']:
        return False
    failed = False
    pyflakes = subprocess.Popen(['pyflakes'] + by_lang['.py'],
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE)

    # pyflakes writes some output (like syntax errors) to stderr. :/
    for pipe in (pyflakes.stdout, pyflakes.stderr):
        for ln in pipe:
            if options.full or not \
                    ('imported but unused' in ln or
                     'redefinition of unused' in ln or
                     "'from typing import *' used; unable to detect undefined names" in ln or
                     ("zerver/models.py" in ln and
                      " undefined name 'bugdown'" in ln) or
                     ("scripts/lib/pythonrc.py" in ln and
                      " import *' used; unable to detect undefined names" in ln) or
                     ("zerver/lib/tornado_ioloop_logging.py" in ln and
                      "redefinition of function 'instrument_tornado_ioloop'" in ln) or
                     ("zephyr_mirror_backend.py:" in ln and
                      "redefinition of unused 'simplejson' from line" in ln)):
                sys.stdout.write(ln)
                failed = True
    return failed

def custom_check_file(fn, rules, skip_rules=[]):
    failed = False
    lineFlag = False
    for i, line in enumerate(open(fn)):
        skip = False
        lineFlag = True
        for rule in skip_rules:
            if re.match(rule, line):
                skip = True
        if skip:
            continue
        for rule in rules:
            exclude_list = rule.get('exclude', set())
            if fn in exclude_list:
                continue
            try:
                if re.search(rule['pattern'], line.strip(rule.get('strip', None))):
                    sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1))
                    print(line)
                    failed = True
            except Exception:
                print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
                traceback.print_exc()
        lastLine = line
    if lineFlag and '\n' not in lastLine:
        print("No new line at the end of %s" % (fn,))
        failed = True
    return failed

whitespace_rules = [
    {'pattern': '\s+$',
     'strip': '\n',
     'description': 'Fix trailing whitespace'},
    {'pattern': '\t',
     'strip': '\n',
     'exclude': set(['zerver/lib/bugdown/codehilite.py']),
     'description': 'Fix tab-based whitespace'},
    ]
js_rules = [
    {'pattern': '[^_]function\(',
     'description': 'The keyword "function" should be followed by a space'},
    {'pattern': '.*blueslip.warning\(.*',
     'description': 'The module blueslip has no function warning, try using blueslip.warn'},
    {'pattern': '[)]{$',
     'description': 'Missing space between ) and {'},
    {'pattern': 'else{$',
     'description': 'Missing space between else and {'},
    ] + whitespace_rules
python_rules = [
    {'pattern': "'[^']*'\s+\([^']*$",
     'description': "Suspicious code with quoting around function name"},
    {'pattern': '"[^"]*"\s+\([^"]*$',
     'description': "Suspicious code with quoting around function name"},
    {'pattern': '".*"%\([a-z_].*\)?$',
     'description': 'Missing space around "%"'},
    {'pattern': "'.*'%\([a-z_].*\)?$",
     'description': 'Missing space around "%"'},
    # This next check could have false positives, but it seems pretty
    # rare; if we find any, they can be added to the exclude list for
    # this rule.
    {'pattern': '% [a-zA-Z0-9_.]*\)?$',
     'description': 'Used % comprehension without a tuple'},
    ] + whitespace_rules
python_line_skip_rules = [
    '\s*[*#]', # comments
    ]
bash_rules = [
    {'pattern': '#!.*sh [-xe]',
     'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'},
    ]

def check_custom_checks():
    failed = False

    for fn in by_lang['.py']:
        if custom_check_file(fn, python_rules, skip_rules=python_line_skip_rules):
            failed = True

    for fn in by_lang['.js']:
        if custom_check_file(fn, js_rules):
            failed = True

    for fn in by_lang['.sh']:
        if custom_check_file(fn, bash_rules):
            failed = True

    return failed

lint_functions = {}

def run_parallel():
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def lint(func):
    lint_functions[func.__name__] = func
    return func

try:
    # Make the lint output bright red
    sys.stdout.write('\x1B[1;31m')
    sys.stdout.flush()

    @lint
    def templates():
        result = subprocess.call(['tools/check-templates'])
        return result

    @lint
    def jslint():
        result = subprocess.call(['tools/node', 'tools/jslint/check-all.js']
                                 + by_lang['.js'])
        return result

    @lint
    def puppet():
        if not by_lang['.pp']:
            return 0
        result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['.pp'])
        return result

    @lint
    def custom():
        failed = check_custom_checks()
        return 1 if failed else 0

    @lint
    def pyflakes():
        failed = check_pyflakes()
        return 1 if failed else 0

    failed = run_parallel()

    sys.exit(1 if failed else 0)

finally:
    # Restore normal terminal colors
    sys.stdout.write('\x1B[0m')