zulip/tools/lint-all

#!/usr/bin/env python
from __future__ import print_function
from __future__ import absolute_import
import os
import re
import sys
import optparse
import subprocess
import traceback
try:
    import lister
except ImportError as e:
    print("ImportError: {}".format(e))
    print("You need to run the Zulip linters inside a Zulip dev environment.")
    print("If you are using Vagrant, you can `vagrant ssh` to enter the Vagrant guest.")
    sys.exit(1)

parser = optparse.OptionParser()
parser.add_option('--full',
    action='store_true',
    help='Check some things we typically ignore')
parser.add_option('--modified', '-m',
    action='store_true',
    help='Only check modified files')
parser.add_option('--verbose', '-v',
    action='store_true',
    help='Print verbose timing output')
(options, args) = parser.parse_args()

os.chdir(os.path.join(os.path.dirname(__file__), '..'))


# Exclude some directories and files from lint checking

exclude = """
static/third
frontend_tests/casperjs
zerver/migrations
node_modules
docs/html_unescape.py
zproject/test_settings.py
zproject/dev_settings.py
zproject/settings.py
tools/jslint/jslint.js
api/setup.py
api/integrations/perforce/git_p4.py
puppet/apt/.forge-release
puppet/puppet-common/tests/
puppet/apt/README.md
static/locale
""".split()

by_lang = lister.list_files(args, modified_only=options.modified, use_shebang=True,
    ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars', 'html', 'json', 'md', 'txt', 'text'],
    group_by_ftype=True, exclude=exclude)

# Invoke the appropriate lint checker for each language,
# and also check files for extra whitespace.

import logging
logging.basicConfig(format="%(asctime)s %(message)s")
logger = logging.getLogger()
if options.verbose:
    logger.setLevel(logging.INFO)
else:
    logger.setLevel(logging.WARNING)

def check_pyflakes():
    if not by_lang['py']:
        return False
    failed = False
    pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE,
        universal_newlines = True)

    # pyflakes writes some output (like syntax errors) to stderr. :/
    for pipe in (pyflakes.stdout, pyflakes.stderr):
        for ln in pipe:
            if options.full or not \
                    ('imported but unused' in ln or
                     'redefinition of unused' in ln or
                     ("zerver/models.py" in ln and
                      " undefined name 'bugdown'" in ln) or
                     ("scripts/lib/pythonrc.py" in ln and
                      " import *' used; unable to detect undefined names" in ln) or
                     ("zerver/lib/tornado_ioloop_logging.py" in ln and
                      "redefinition of function 'instrument_tornado_ioloop'" in ln) or
                     ("zephyr_mirror_backend.py:" in ln and
                      "redefinition of unused 'simplejson' from line" in ln)):
                sys.stdout.write(ln)
                failed = True
    return failed

def custom_check_file(fn, rules, skip_rules=None):
    failed = False
    lineFlag = False
    for i, line in enumerate(open(fn)):
        line_newline_stripped = line.strip('\n')
        line_fully_stripped = line_newline_stripped.strip()
        skip = False
        lineFlag = True
        for rule in skip_rules or []:
            if re.match(rule, line):
                skip = True
        if skip:
            continue
        for rule in rules:
            exclude_list = rule.get('exclude', set())
            if fn in exclude_list:
                continue
            exclude_list = rule.get('exclude_line', set())
            if (fn, line_fully_stripped) in exclude_list:
                continue
            try:
                line_to_check = line_fully_stripped
                if rule.get('strip') is not None:
                    if rule['strip'] == '\n':
                        line_to_check = line_newline_stripped
                    else:
                        raise Exception("Invalid strip rule")
                if re.search(rule['pattern'], line_to_check):
                    sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1))
                    print(line)
                    failed = True
            except Exception:
                print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
                traceback.print_exc()
        lastLine = line
    if lineFlag and '\n' not in lastLine:
        print("No newline at the end of file.  Fix with `sed -i '$a\\' %s`" % (fn,))
        failed = True
    return failed

whitespace_rules = [
    # This linter should be first since bash_rules depends on it.
    {'pattern': '\s+$',
     'strip': '\n',
     'description': 'Fix trailing whitespace'},
    {'pattern': '\t',
     'strip': '\n',
     'exclude': set(['zerver/lib/bugdown/codehilite.py']),
     'description': 'Fix tab-based whitespace'},
    ]
js_rules = [
    {'pattern': '[^_]function\(',
     'description': 'The keyword "function" should be followed by a space'},
    {'pattern': '.*blueslip.warning\(.*',
     'description': 'The module blueslip has no function warning, try using blueslip.warn'},
    {'pattern': '[)]{$',
     'description': 'Missing space between ) and {'},
    {'pattern': '["\']json/',
     'description': 'Relative URL for JSON route not supported by i18n'},
    # This rule is constructed with + to avoid triggering on itself
    {'pattern': " =" + '[^ =>~"]',
     'description': 'Missing whitespace after "="'},
    {'pattern': '^[ ]*//[A-Za-z0-9]',
     'description': 'Missing space after // in comment'},
    {'pattern': 'if[(]',
     'description': 'Missing space between if and ('},
    {'pattern': 'else{$',
     'description': 'Missing space between else and {'},
    {'pattern': '^else {$',
     'description': 'Write JS else statements on same line as }'},
    {'pattern': '^else if',
     'description': 'Write JS else statements on same line as }'},
    {'pattern': 'button\.text\(["\']',
     'exclude': set(['tools/lint-all',
                     'frontend_tests/node_tests/templates.js']),
     'description': 'Argument to button.text should be a literal string enclosed by i18n.t()'},
    {'pattern': 'compose_error\(["\']',
     'exclude': set(['tools/lint-all']),
     'description': 'Argument to compose_error should be a literal string enclosed '
                    'by i18n.t()'},
    ] + whitespace_rules
python_rules = [
    {'pattern': '^(?!#)@login_required',
     'description': '@login_required is unsupported; use @zulip_login_required'},
    {'pattern': '".*"%\([a-z_].*\)?$',
     'description': 'Missing space around "%"'},
    {'pattern': "'.*'%\([a-z_].*\)?$",
     'exclude': set(['tools/lint-all']),
     'exclude_line': set([
         ('zerver/views/users.py',
          "return json_error(_(\"Email '%(email)s' does not belong to domain '%(domain)s'\") %"),
     ]),
     'description': 'Missing space around "%"'},
    # This rule is constructed with + to avoid triggering on itself
    {'pattern': " =" + '[^ =>~"]',
     'description': 'Missing whitespace after "="'},
    {'pattern': '":\w[^"]*$',
     'description': 'Missing whitespace after ":"'},
    {'pattern': "':\w[^']*$",
     'description': 'Missing whitespace after ":"'},
    {'pattern': "^\s+[#]\w",
     'strip': '\n',
     'description': 'Missing whitespace after "#"'},
    {'pattern': "== None",
     'exclude': 'tools/lint-all',
     'description': 'Use `is None` to check whether something is None'},
    {'pattern': "type:[(]",
     'description': 'Missing whitespace after ":" in type annotation'},
    {'pattern': "#type",
     'exclude': 'tools/lint-all',
     'description': 'Missing whitespace after "#" in type annotation'},
    {'pattern': ", [)]",
     'description': 'Unnecessary whitespace between "," and ")"'},
    {'pattern': "%  [(]",
     'description': 'Unnecessary whitespace between "%" and "("'},
    # This next check could have false positives, but it seems pretty
    # rare; if we find any, they can be added to the exclude list for
    # this rule.
    {'pattern': '% [a-zA-Z0-9_.]*\)?$',
     'description': 'Used % comprehension without a tuple'},
    # To avoid json_error(_variable) and json_error(_(variable))
    {'pattern': '\Wjson_error\(_\(?\w+\)',
     'exclude': set(['tools/lint-all']),
     'description': 'Argument to json_error should be a literal string enclosed by _()'},
    {'pattern': '\Wjson_error\([^_].+\)',
     'exclude': set(['tools/lint-all']),
     'exclude_line': set([
         # function definition
         ('zerver/lib/response.py', 'def json_error(msg, data=None, status=400):'),
         # No need to worry about the following as the translation strings
         # are already captured
         ('zerver/middleware.py',
          'return json_error(exception.to_json_error_msg(), status=status_code)'),
         ('zerver/tornadoviews.py', 'return json_error(result["message"])'),
         ('zerver/views/__init__.py',
          'return json_error(data=error_data, msg=ret_error)'),
         ('zerver/views/streams.py', 'return json_error(property_conversion)'),
         # We can't do anything about this.
         ('zerver/views/realm_emoji.py', 'return json_error(e.messages[0])'),
     ]),
     'description': 'Argument to json_error should a literal string enclosed by _()'},
    # To avoid JsonableError(_variable) and JsonableError(_(variable))
    {'pattern': '\WJsonableError\(_\(?\w.+\)',
     'exclude': set(['tools/lint-all']),
     'description': 'Argument to JsonableError should be a literal string enclosed by _()'},
    {'pattern': '\WJsonableError\([^_].+\)',
     'exclude': set(['tools/lint-all']),
     'exclude_line': set([
         # class definition
         ('zerver/lib/request.py', 'class JsonableError(Exception):'),
         # No need to worry about the following as the translation strings
         # are already captured
         ('zerver/decorator.py', 'raise JsonableError(reason % (role,))'),
         ('zerver/lib/actions.py', 'raise JsonableError(e.messages[0])'),
         ('zerver/views/messages.py', 'raise JsonableError(error)'),
         ('zerver/lib/request.py', 'raise JsonableError(error)'),
     ]),
     'description': 'Argument to JsonableError should be a literal string enclosed by _()'},
    {'pattern': '([a-zA-Z0-9_]+)=REQ\([\'"]\\1[\'"]',
     'description': 'REQ\'s first argument already defaults to parameter name'},
    ] + whitespace_rules
bash_rules = [
    {'pattern': '#!.*sh [-xe]',
     'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'},
    ] + whitespace_rules[0:1]
css_rules = [
    {'pattern': '^[^:]*:\S[^:]*;$',
     'description': "Missing whitespace after : in CSS"},
    {'pattern': '{\w',
     'description': "Missing whitespace after '{' in CSS (should be newline)."},
    ] + whitespace_rules
handlebars_rules = whitespace_rules
html_rules = whitespace_rules + [
    {'pattern': 'placeholder="[^{]',
     'description': "`placeholder` value should be translatable."},
    {'pattern': "placeholder='[^{]",
     'description': "`placeholder` value should be translatable."},
    ]
json_rules = [] # just fix newlines at ends of files
markdown_rules = whitespace_rules
txt_rules = whitespace_rules

def check_custom_checks_py():
    failed = False

    for fn in by_lang['py']:
        if custom_check_file(fn, python_rules):
            failed = True
    return failed

def check_custom_checks_nonpy():
    failed = False

    for fn in by_lang['js']:
        if custom_check_file(fn, js_rules):
            failed = True

    for fn in by_lang['sh']:
        if custom_check_file(fn, bash_rules):
            failed = True

    for fn in by_lang['css']:
        if custom_check_file(fn, css_rules):
            failed = True

    for fn in by_lang['handlebars']:
        if custom_check_file(fn, handlebars_rules):
            failed = True

    for fn in by_lang['html']:
        if custom_check_file(fn, html_rules):
            failed = True

    for fn in by_lang['json']:
        if custom_check_file(fn, json_rules):
            failed = True

    for fn in by_lang['md']:
        if custom_check_file(fn, markdown_rules):
            failed = True

    for fn in by_lang['txt'] + by_lang['text']:
        if custom_check_file(fn, txt_rules):
            failed = True

    return failed

lint_functions = {}

def run_parallel():
    pids = []
    for name, func in lint_functions.items():
        pid = os.fork()
        if pid == 0:
            logging.info("start " + name)
            result = func()
            logging.info("finish " + name)
            os._exit(result)
        pids.append(pid)
    failed = False

    for pid in pids:
        (_, status) = os.waitpid(pid, 0)
        if status != 0:
            failed = True
    return failed

def lint(func):
    lint_functions[func.__name__] = func
    return func

try:
    # Make the lint output bright red
    sys.stdout.write('\x1B[1;31m')
    sys.stdout.flush()

    @lint
    def templates():
        result = subprocess.call(['tools/check-templates'])
        return result

    @lint
    def jslint():
        result = subprocess.call(['tools/node', 'tools/jslint/check-all.js']
                                 + by_lang['js'])
        return result

    @lint
    def puppet():
        if not by_lang['pp']:
            return 0
        result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['pp'])
        return result

    @lint
    def custom_py():
        failed = check_custom_checks_py()
        return 1 if failed else 0

    @lint
    def custom_nonpy():
        failed = check_custom_checks_nonpy()
        return 1 if failed else 0

    @lint
    def pyflakes():
        failed = check_pyflakes()
        return 1 if failed else 0

    failed = run_parallel()

    sys.exit(1 if failed else 0)

finally:
    # Restore normal terminal colors
    sys.stdout.write('\x1B[0m')