#!/usr/bin/env python2.7 from __future__ import print_function from __future__ import absolute_import import os import re import sys import optparse import subprocess import traceback from os import path from collections import defaultdict from six.moves import filter from six.moves import map parser = optparse.OptionParser() parser.add_option('--full', action='store_true', help='Check some things we typically ignore') parser.add_option('--modified', '-m', action='store_true', help='Only check modified files') (options, args) = parser.parse_args() os.chdir(path.join(path.dirname(__file__), '..')) # Exclude some directories and files from lint checking exclude_trees = """ static/third confirmation frontend_tests/casperjs zerver/migrations node_modules """.split() exclude_files = """ docs/html_unescape.py zproject/test_settings.py zproject/settings.py tools/jslint/jslint.js api/setup.py api/integrations/perforce/git_p4.py """.split() if options.modified: # If the user specifies, use `git ls-files -m` to only check modified, non-staged # files in the current checkout. This makes things fun faster. files = list(map(str.strip, subprocess.check_output(['git', 'ls-files', '-m']).split('\n'))) else: files = [] files += args if not files and not options.modified: # If no files are specified on the command line, use the entire git checkout files = list(map(str.strip, subprocess.check_output(['git', 'ls-files']).split('\n'))) files = list(filter(bool, files)) # remove empty file caused by trailing \n if not files: raise Exception('There are no files to check!') # Categorize by language all files we want to check by_lang = defaultdict(list) for filepath in files: if (not filepath or not path.isfile(filepath) or (filepath in exclude_files) or any(filepath.startswith(d+'/') for d in exclude_trees)): continue _, exn = path.splitext(filepath) if not exn: # No extension; look at the first line with open(filepath) as f: if re.match(r'^#!.*\bpython', f.readline()): exn = '.py' by_lang[exn].append(filepath) by_lang['.sh'] = [_f for _f in map(str.strip, subprocess.check_output("grep --files-with-matches '#!.*\(ba\)\?sh' $(git ls-tree --name-only -r HEAD scripts/ tools/ bin/ | grep -v [.])", shell=True).split('\n')) if _f] # Invoke the appropriate lint checker for each language, # and also check files for extra whitespace. import logging logging.basicConfig(format="%(asctime)s %(message)s") logger = logging.getLogger() # Change this to logging.INFO to see performance data logger.setLevel(logging.WARNING) def check_pyflakes(): if not by_lang['.py']: return False failed = False pyflakes = subprocess.Popen(['pyflakes'] + by_lang['.py'], stdout = subprocess.PIPE, stderr = subprocess.PIPE) # pyflakes writes some output (like syntax errors) to stderr. :/ for pipe in (pyflakes.stdout, pyflakes.stderr): for ln in pipe: if options.full or not \ ('imported but unused' in ln or 'redefinition of unused' in ln or "'from typing import *' used; unable to detect undefined names" in ln or ("zerver/models.py" in ln and " undefined name 'bugdown'" in ln) or ("scripts/lib/pythonrc.py" in ln and " import *' used; unable to detect undefined names" in ln) or ("zerver/lib/tornado_ioloop_logging.py" in ln and "redefinition of function 'instrument_tornado_ioloop'" in ln) or ("zephyr_mirror_backend.py:" in ln and "redefinition of unused 'simplejson' from line" in ln)): sys.stdout.write(ln) failed = True return failed def custom_check_file(fn, rules, skip_rules=[]): failed = False lineFlag = False for i, line in enumerate(open(fn)): skip = False lineFlag = True for rule in skip_rules: if re.match(rule, line): skip = True if skip: continue for rule in rules: exclude_list = rule.get('exclude', set()) if fn in exclude_list: continue try: if re.search(rule['pattern'], line.strip(rule.get('strip', None))): sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1)) print(line) failed = True except Exception: print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1)) traceback.print_exc() lastLine = line if lineFlag and '\n' not in lastLine: print("No new line at the end of %s" % (fn,)) failed = True return failed whitespace_rules = [ {'pattern': '\s+$', 'strip': '\n', 'description': 'Fix trailing whitespace'}, {'pattern': '\t', 'strip': '\n', 'exclude': set(['zerver/lib/bugdown/codehilite.py']), 'description': 'Fix tab-based whitespace'}, ] js_rules = [ {'pattern': '[^_]function\(', 'description': 'The keyword "function" should be followed by a space'}, {'pattern': '.*blueslip.warning\(.*', 'description': 'The module blueslip has no function warning, try using blueslip.warn'}, {'pattern': '[)]{$', 'description': 'Missing space between ) and {'}, {'pattern': 'else{$', 'description': 'Missing space between else and {'}, ] + whitespace_rules python_rules = [ {'pattern': "'[^']*'\s+\([^']*$", 'description': "Suspicious code with quoting around function name"}, {'pattern': '"[^"]*"\s+\([^"]*$', 'description': "Suspicious code with quoting around function name"}, {'pattern': '".*"%\([a-z_].*\)?$', 'description': 'Missing space around "%"'}, {'pattern': "'.*'%\([a-z_].*\)?$", 'description': 'Missing space around "%"'}, # This next check could have false positives, but it seems pretty # rare; if we find any, they can be added to the exclude list for # this rule. {'pattern': '% [a-zA-Z0-9_.]*\)?$', 'description': 'Used % comprehension without a tuple'}, ] + whitespace_rules python_line_skip_rules = [ '\s*[*#]', # comments ] bash_rules = [ {'pattern': '#!.*sh [-xe]', 'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'}, ] def check_custom_checks(): failed = False for fn in by_lang['.py']: if custom_check_file(fn, python_rules, skip_rules=python_line_skip_rules): failed = True for fn in by_lang['.js']: if custom_check_file(fn, js_rules): failed = True for fn in by_lang['.sh']: if custom_check_file(fn, bash_rules): failed = True return failed lint_functions = {} def run_parallel(): pids = [] for name, func in lint_functions.items(): pid = os.fork() if pid == 0: logging.info("start " + name) result = func() logging.info("finish " + name) os._exit(result) pids.append(pid) failed = False for pid in pids: (_, status) = os.waitpid(pid, 0) if status != 0: failed = True return failed def lint(func): lint_functions[func.__name__] = func return func try: # Make the lint output bright red sys.stdout.write('\x1B[1;31m') sys.stdout.flush() @lint def templates(): result = subprocess.call(['tools/check-templates']) return result @lint def jslint(): result = subprocess.call(['tools/node', 'tools/jslint/check-all.js'] + by_lang['.js']) return result @lint def puppet(): if not by_lang['.pp']: return 0 result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['.pp']) return result @lint def custom(): failed = check_custom_checks() return 1 if failed else 0 @lint def pyflakes(): failed = check_pyflakes() return 1 if failed else 0 failed = run_parallel() sys.exit(1 if failed else 0) finally: # Restore normal terminal colors sys.stdout.write('\x1B[0m')