zulip/tools/lint-all

#!/usr/bin/env python2.7
from __future__ import print_function
from __future__ import absolute_import
import os
import re
import sys
import optparse
import subprocess
import traceback

from os          import path
from collections import defaultdict
from six.moves import filter
from six.moves import map

import lister

parser = optparse.OptionParser()
parser.add_option('--full',
    action='store_true',
    help='Check some things we typically ignore')
parser.add_option('--modified', '-m',
    action='store_true',
    help='Only check modified files')
parser.add_option('--verbose', '-v',
    action='store_true',
    help='Print verbose timing output')
(options, args) = parser.parse_args()

os.chdir(path.join(path.dirname(__file__), '..'))


# Exclude some directories and files from lint checking

exclude = """
static/third
confirmation
frontend_tests/casperjs
zerver/migrations
node_modules
docs/html_unescape.py
zproject/test_settings.py
zproject/settings.py
tools/jslint/jslint.js
api/setup.py
api/integrations/perforce/git_p4.py
puppet/apt/.forge-release
puppet/puppet-common/tests/
puppet/apt/README.md
""".split()

by_lang = lister.list_files(args, modified_only=options.modified, use_shebang=True,
    ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars', 'html', 'json', 'md', 'txt', 'text'],
    group_by_ftype=True, exclude=exclude)

# Invoke the appropriate lint checker for each language,
# and also check files for extra whitespace.

import logging
logging.basicConfig(format="%(asctime)s %(message)s")
logger = logging.getLogger()
if options.verbose:
    logger.setLevel(logging.INFO)
else:
    logger.setLevel(logging.WARNING)

def check_pyflakes():
    if not by_lang['py']:
        return False
    failed = False
    pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE)

    # pyflakes writes some output (like syntax errors) to stderr. :/
    for pipe in (pyflakes.stdout, pyflakes.stderr):
        for ln in pipe:
            if options.full or not \
                    ('imported but unused' in ln or
                     'redefinition of unused' in ln or
                     ("zerver/models.py" in ln and
                      " undefined name 'bugdown'" in ln) or
                     ("scripts/lib/pythonrc.py" in ln and
                      " import *' used; unable to detect undefined names" in ln) or
                     ("zerver/lib/tornado_ioloop_logging.py" in ln and
                      "redefinition of function 'instrument_tornado_ioloop'" in ln) or
                     ("zephyr_mirror_backend.py:" in ln and
                      "redefinition of unused 'simplejson' from line" in ln)):
                sys.stdout.write(ln)
                failed = True
    return failed

def custom_check_file(fn, rules, skip_rules=[]):
    failed = False
    lineFlag = False
    for i, line in enumerate(open(fn)):
        skip = False
        lineFlag = True
        for rule in skip_rules:
            if re.match(rule, line):
                skip = True
        if skip:
            continue
        for rule in rules:
            exclude_list = rule.get('exclude', set())
            if fn in exclude_list:
                continue
            try:
                if re.search(rule['pattern'], line.strip(rule.get('strip', None))):
                    sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1))
                    print(line)
                    failed = True
            except Exception:
                print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
                traceback.print_exc()
        lastLine = line
    if lineFlag and '\n' not in lastLine:
        print("No newline at the end of file.  Fix with `sed -i '$a\\' %s`" % (fn,))
        failed = True
    return failed

whitespace_rules = [
    # This linter should be first since bash_rules depends on it.
    {'pattern': '\s+$',
     'strip': '\n',
     'description': 'Fix trailing whitespace'},
    {'pattern': '\t',
     'strip': '\n',
     'exclude': set(['zerver/lib/bugdown/codehilite.py']),
     'description': 'Fix tab-based whitespace'},
    ]
js_rules = [
    {'pattern': '[^_]function\(',
     'description': 'The keyword "function" should be followed by a space'},
    {'pattern': '.*blueslip.warning\(.*',
     'description': 'The module blueslip has no function warning, try using blueslip.warn'},
    {'pattern': '[)]{$',
     'description': 'Missing space between ) and {'},
    # This rule is constructed with + to avoid triggering on itself
    {'pattern': " =" + '[^ =>~"]',
     'description': 'Missing whitespace after "="'},
    {'pattern': 'else{$',
     'description': 'Missing space between else and {'},
    ] + whitespace_rules
python_rules = [
    {'pattern': '^(?!#)@login_required',
     'description': '@login_required is unsupported; use @zulip_login_required'},
    {'pattern': '".*"%\([a-z_].*\)?$',
     'description': 'Missing space around "%"'},
    {'pattern': "'.*'%\([a-z_].*\)?$",
     'description': 'Missing space around "%"'},
    # This rule is constructed with + to avoid triggering on itself
    {'pattern': " =" + '[^ =>~"]',
     'description': 'Missing whitespace after "="'},
    {'pattern': '":\w[^"]*$',
     'description': 'Missing whitespace after ":"'},
    {'pattern': "':\w[^']*$",
     'description': 'Missing whitespace after ":"'},
    {'pattern': "^\s+[#]\w",
     'strip': '\n',
     'description': 'Missing whitespace after "#"'},
    {'pattern': ", [)]",
     'description': 'Unnecessary whitespace between "," and ")"'},
    {'pattern': "%  [(]",
     'description': 'Unnecessary whitespace between "%" and "("'},
    # This next check could have false positives, but it seems pretty
    # rare; if we find any, they can be added to the exclude list for
    # this rule.
    {'pattern': '% [a-zA-Z0-9_.]*\)?$',
     'description': 'Used % comprehension without a tuple'},
    ] + whitespace_rules
bash_rules = [
    {'pattern': '#!.*sh [-xe]',
     'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'},
    ] + whitespace_rules[0:1]
css_rules = [
    {'pattern': '^[^:]*:\S[^:]*;$',
     'description': "Missing whitespace after : in CSS"},
    {'pattern': '{\w',
     'description': "Missing whitespace after '{' in CSS (should be newline)."},
    ] + whitespace_rules
handlebars_rules = whitespace_rules
html_rules = whitespace_rules
json_rules = [] # just fix newlines at ends of files
markdown_rules = whitespace_rules
txt_rules = whitespace_rules

def check_custom_checks():
    failed = False

    for fn in by_lang['py']:
        if custom_check_file(fn, python_rules):
            failed = True

    for fn in by_lang['js']:
        if custom_check_file(fn, js_rules):
            failed = True

    for fn in by_lang['sh']:
        if custom_check_file(fn, bash_rules):
            failed = True

    for fn in by_lang['css']:
        if custom_check_file(fn, css_rules):
            failed = True

    for fn in by_lang['handlebars']:
        if custom_check_file(fn, handlebars_rules):
            failed = True

    for fn in by_lang['html']:
        if custom_check_file(fn, html_rules):
            failed = True

    for fn in by_lang['json']:
        if custom_check_file(fn, json_rules):
            failed = True

    for fn in by_lang['md']:
        if custom_check_file(fn, markdown_rules):
            failed = True

    for fn in by_lang['txt'] + by_lang['text']:
        if custom_check_file(fn, txt_rules):
            failed = True

    return failed

lint_functions = {}

def run_parallel():
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def lint(func):
    lint_functions[func.__name__] = func
    return func

try:
    # Make the lint output bright red
    sys.stdout.write('\x1B[1;31m')
    sys.stdout.flush()

    @lint
    def templates():
        result = subprocess.call(['tools/check-templates'])
        return result

    @lint
    def jslint():
        result = subprocess.call(['tools/node', 'tools/jslint/check-all.js']
                                 + by_lang['js'])
        return result

    @lint
    def puppet():
        if not by_lang['pp']:
            return 0
        result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['pp'])
        return result

    @lint
    def custom():
        failed = check_custom_checks()
        return 1 if failed else 0

    @lint
    def pyflakes():
        failed = check_pyflakes()
        return 1 if failed else 0

    failed = run_parallel()

    sys.exit(1 if failed else 0)

finally:
    # Restore normal terminal colors
    sys.stdout.write('\x1B[0m')