zulip/tools/lint-all

#!/usr/bin/env python
import os
import re
import sys
import optparse
import subprocess

from os          import path
from collections import defaultdict

from zulip_tools import check_output

parser = optparse.OptionParser()
parser.add_option('--full',
    action='store_true',
    help='Check some things we typically ignore')
(options, args) = parser.parse_args()

os.chdir(path.join(path.dirname(__file__), '..'))


# Exclude some directories and files from lint checking

exclude_trees = """
static/third
confirmation
zerver/tests/frontend/casperjs
zerver/migrations
node_modules
""".split()

exclude_files = """
zproject/test_settings.py
zproject/settings.py
tools/jslint/jslint.js
""".split()


# Categorize by language all files known to Git

git_files = map(str.strip, check_output(['git', 'ls-files']).split('\n'))
by_lang   = defaultdict(list)

for filepath in git_files:
    if (not filepath or not path.isfile(filepath)
        or (filepath in exclude_files)
        or any(filepath.startswith(d+'/') for d in exclude_trees)):
        continue

    _, exn = path.splitext(filepath)
    if not exn:
        # No extension; look at the first line
        with file(filepath) as f:
            if re.match(r'^#!.*\bpython', f.readline()):
                exn = '.py'

    by_lang[exn].append(filepath)

def check_trailing_whitespace(fn):
    failed = False
    for i, line in enumerate(open(fn)):
        if re.search('\s+$', line.strip('\n')):
            sys.stdout.write('Fix whitespace at %s line %s\n' % (fn, i+1))
            failed = True
    return failed

def perform_extra_js_checks(fn):
    failed = False
    for i, line in enumerate(open(fn)):
        line = line.strip('\n')
        if re.search('[^_]function\(', line):
            sys.stdout.write('The keyword "function" should be followed by a space in %s line %s\n' % (fn, i+1))
            print line
            failed = True
    return failed


def check_python_gotchas(fn):
    '''
    Check for certain Python gotchas that pyflakes doesn't catch.
    '''
    failed = False
    for i, line in enumerate(open(fn)):
        line = line.strip('\n')

        # Hacks to skip lines that confuse our dirt simple code:
        if re.match('\s*[*#]', line):
            continue
        if 'help=' in line:
            continue

        gotcha_regexes = [
                "'[^']*'\s+\([^']*$", # 'foo'(2) makes no sense
                '"[^"]*"\s+\([^"]*$', # ditto
                '% [a-z_]*$', # "%s" % s [we prefer "%s" % (s,)]
        ]
        for gotcha_regex in gotcha_regexes:
            if re.search(gotcha_regex, line):
                sys.stdout.write('Suspicious code at %s line %s (regex=%r)\n' % (fn, i+1, gotcha_regex))
                print line
                failed = True
                break
    return failed

# Invoke the appropriate lint checker for each language,
# and also check files for extra whitespace.

import logging
logging.basicConfig(format="%(asctime)s %(message)s")
logger = logging.getLogger()
# Change this to logging.INFO to see performance data
logger.setLevel(logging.WARNING)

def check_pyflakes():
    failed = False
    pyflakes = subprocess.Popen(['pyflakes'] + by_lang['.py'],
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE)

    # pyflakes writes some output (like syntax errors) to stderr. :/
    for pipe in (pyflakes.stdout, pyflakes.stderr):
        for ln in pipe:
            if options.full or not \
                    ('imported but unused' in ln or
                     ("zephyr_mirror_backend.py:" in ln and
                      "redefinition of unused 'simplejson' from line" in ln)):
                sys.stdout.write(ln)
                failed = True
    return failed

def check_custom_checks():
    failed = False

    for fn in by_lang['.js'] + by_lang['.py']:
        if check_trailing_whitespace(fn):
            failed = True

    for fn in by_lang['.py']:
        if check_python_gotchas(fn):
            failed = True

    for fn in by_lang['.js']:
        if perform_extra_js_checks(fn):
            failed = True

    return failed

try:
    failed = False

    # Make the lint output bright red
    sys.stdout.write('\x1B[1;31m')
    sys.stdout.flush()

    check_handlebar_templates_pid = os.fork()
    if check_handlebar_templates_pid == 0:
        logging.info("start check-handlebar-templates")
        result = subprocess.call(['tools/check-handlebar-templates'])
        logging.info("finish check-handlebar-templates")
        os._exit(result)

    jslint_pid = os.fork()
    if jslint_pid == 0:
        logging.info("start jslint")
        result = subprocess.call(['tools/node', 'tools/jslint/check-all.js']
                                 + by_lang['.js'])
        logging.info("finish jslint")
        os._exit(result)

    puppet_pid = os.fork()
    if puppet_pid == 0:
        logging.info("start puppet")
        result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['.pp'])
        logging.info("finish puppet")
        os._exit(result)

    custom_pid = os.fork()
    if custom_pid == 0:
        logging.info("start custom")
        failed = check_custom_checks()
        logging.info("finish custom")
        os._exit(1 if failed else 0)

    pyflakes_pid = os.fork()
    if pyflakes_pid == 0:
        logging.info("start pyflakes")
        failed = check_pyflakes()
        logging.info("finish pyflakes")
        os._exit(1 if failed else 0)

    (_, status) = os.waitpid(check_handlebar_templates_pid, 0)
    if status != 0:
        failed = True

    (_, status) = os.waitpid(custom_pid, 0)
    if status != 0:
        failed = True

    (_, status) = os.waitpid(jslint_pid, 0)
    if status != 0:
        failed = True

    (_, status) = os.waitpid(pyflakes_pid, 0)
    if status != 0:
        failed = True

    (_, status) = os.waitpid(puppet_pid, 0)
    if status != 0:
        failed = True

    sys.exit(1 if failed else 0)

finally:
    # Restore normal terminal colors
    sys.stdout.write('\x1B[0m')