#!/usr/bin/env python2.7 from __future__ import print_function from __future__ import absolute_import import os import re import sys import optparse import subprocess import traceback from os import path from collections import defaultdict from six.moves import filter from six.moves import map import lister parser = optparse.OptionParser() parser.add_option('--full', action='store_true', help='Check some things we typically ignore') parser.add_option('--modified', '-m', action='store_true', help='Only check modified files') (options, args) = parser.parse_args() os.chdir(path.join(path.dirname(__file__), '..')) # Exclude some directories and files from lint checking exclude = """ static/third confirmation frontend_tests/casperjs zerver/migrations node_modules docs/html_unescape.py zproject/test_settings.py zproject/settings.py tools/jslint/jslint.js api/setup.py api/integrations/perforce/git_p4.py puppet/apt/.forge-release puppet/puppet-common/tests/ """.split() by_lang = lister.list_files(args, modified_only=options.modified, use_shebang=True, ftypes=['py', 'sh', 'js', 'pp'], group_by_ftype=True, exclude=exclude) # Invoke the appropriate lint checker for each language, # and also check files for extra whitespace. import logging logging.basicConfig(format="%(asctime)s %(message)s") logger = logging.getLogger() # Change this to logging.INFO to see performance data logger.setLevel(logging.WARNING) def check_pyflakes(): if not by_lang['py']: return False failed = False pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'], stdout = subprocess.PIPE, stderr = subprocess.PIPE) # pyflakes writes some output (like syntax errors) to stderr. :/ for pipe in (pyflakes.stdout, pyflakes.stderr): for ln in pipe: if options.full or not \ ('imported but unused' in ln or 'redefinition of unused' in ln or "'from typing import *' used; unable to detect undefined names" in ln or ("zerver/models.py" in ln and " undefined name 'bugdown'" in ln) or ("scripts/lib/pythonrc.py" in ln and " import *' used; unable to detect undefined names" in ln) or ("zerver/lib/tornado_ioloop_logging.py" in ln and "redefinition of function 'instrument_tornado_ioloop'" in ln) or ("zephyr_mirror_backend.py:" in ln and "redefinition of unused 'simplejson' from line" in ln)): sys.stdout.write(ln) failed = True return failed def custom_check_file(fn, rules, skip_rules=[]): failed = False lineFlag = False for i, line in enumerate(open(fn)): skip = False lineFlag = True for rule in skip_rules: if re.match(rule, line): skip = True if skip: continue for rule in rules: exclude_list = rule.get('exclude', set()) if fn in exclude_list: continue try: if re.search(rule['pattern'], line.strip(rule.get('strip', None))): sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1)) print(line) failed = True except Exception: print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1)) traceback.print_exc() lastLine = line if lineFlag and '\n' not in lastLine: print("No new line at the end of %s" % (fn,)) failed = True return failed whitespace_rules = [ {'pattern': '\s+$', 'strip': '\n', 'description': 'Fix trailing whitespace'}, {'pattern': '\t', 'strip': '\n', 'exclude': set(['zerver/lib/bugdown/codehilite.py']), 'description': 'Fix tab-based whitespace'}, ] js_rules = [ {'pattern': '[^_]function\(', 'description': 'The keyword "function" should be followed by a space'}, {'pattern': '.*blueslip.warning\(.*', 'description': 'The module blueslip has no function warning, try using blueslip.warn'}, {'pattern': '[)]{$', 'description': 'Missing space between ) and {'}, {'pattern': 'else{$', 'description': 'Missing space between else and {'}, ] + whitespace_rules python_rules = [ {'pattern': "'[^']*'\s+\([^']*$", 'description': "Suspicious code with quoting around function name"}, {'pattern': '"[^"]*"\s+\([^"]*$', 'description': "Suspicious code with quoting around function name"}, {'pattern': '".*"%\([a-z_].*\)?$', 'description': 'Missing space around "%"'}, {'pattern': "'.*'%\([a-z_].*\)?$", 'description': 'Missing space around "%"'}, # This next check could have false positives, but it seems pretty # rare; if we find any, they can be added to the exclude list for # this rule. {'pattern': '% [a-zA-Z0-9_.]*\)?$', 'description': 'Used % comprehension without a tuple'}, ] + whitespace_rules python_line_skip_rules = [ '\s*[*#]', # comments ] bash_rules = [ {'pattern': '#!.*sh [-xe]', 'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'}, ] def check_custom_checks(): failed = False for fn in by_lang['py']: if custom_check_file(fn, python_rules, skip_rules=python_line_skip_rules): failed = True for fn in by_lang['js']: if custom_check_file(fn, js_rules): failed = True for fn in by_lang['sh']: if custom_check_file(fn, bash_rules): failed = True return failed lint_functions = {} def run_parallel(): pids = [] for name, func in lint_functions.items(): pid = os.fork() if pid == 0: logging.info("start " + name) result = func() logging.info("finish " + name) os._exit(result) pids.append(pid) failed = False for pid in pids: (_, status) = os.waitpid(pid, 0) if status != 0: failed = True return failed def lint(func): lint_functions[func.__name__] = func return func try: # Make the lint output bright red sys.stdout.write('\x1B[1;31m') sys.stdout.flush() @lint def templates(): result = subprocess.call(['tools/check-templates']) return result @lint def jslint(): result = subprocess.call(['tools/node', 'tools/jslint/check-all.js'] + by_lang['js']) return result @lint def puppet(): if not by_lang['pp']: return 0 result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['pp']) return result @lint def custom(): failed = check_custom_checks() return 1 if failed else 0 @lint def pyflakes(): failed = check_pyflakes() return 1 if failed else 0 failed = run_parallel() sys.exit(1 if failed else 0) finally: # Restore normal terminal colors sys.stdout.write('\x1B[0m')