mirror of https://github.com/zulip/zulip.git
264 lines
7.8 KiB
Python
Executable File
264 lines
7.8 KiB
Python
Executable File
#!/usr/bin/env python2.7
|
|
from __future__ import print_function
|
|
from __future__ import absolute_import
|
|
import os
|
|
import re
|
|
import sys
|
|
import optparse
|
|
import subprocess
|
|
import traceback
|
|
|
|
from os import path
|
|
from collections import defaultdict
|
|
from six.moves import filter
|
|
from six.moves import map
|
|
|
|
parser = optparse.OptionParser()
|
|
parser.add_option('--full',
|
|
action='store_true',
|
|
help='Check some things we typically ignore')
|
|
parser.add_option('--modified', '-m',
|
|
action='store_true',
|
|
help='Only check modified files')
|
|
(options, args) = parser.parse_args()
|
|
|
|
os.chdir(path.join(path.dirname(__file__), '..'))
|
|
|
|
|
|
# Exclude some directories and files from lint checking
|
|
|
|
exclude_trees = """
|
|
static/third
|
|
confirmation
|
|
frontend_tests/casperjs
|
|
zerver/migrations
|
|
node_modules
|
|
""".split()
|
|
|
|
exclude_files = """
|
|
docs/html_unescape.py
|
|
zproject/test_settings.py
|
|
zproject/settings.py
|
|
tools/jslint/jslint.js
|
|
api/setup.py
|
|
api/integrations/perforce/git_p4.py
|
|
""".split()
|
|
|
|
if options.modified:
|
|
# If the user specifies, use `git ls-files -m` to only check modified, non-staged
|
|
# files in the current checkout. This makes things fun faster.
|
|
files = list(map(str.strip, subprocess.check_output(['git', 'ls-files', '-m']).split('\n')))
|
|
else:
|
|
files = []
|
|
|
|
files += args
|
|
|
|
if not files and not options.modified:
|
|
# If no files are specified on the command line, use the entire git checkout
|
|
files = list(map(str.strip, subprocess.check_output(['git', 'ls-files']).split('\n')))
|
|
|
|
files = list(filter(bool, files)) # remove empty file caused by trailing \n
|
|
|
|
if not files:
|
|
raise Exception('There are no files to check!')
|
|
|
|
# Categorize by language all files we want to check
|
|
by_lang = defaultdict(list)
|
|
|
|
for filepath in files:
|
|
if (not filepath or not path.isfile(filepath)
|
|
or (filepath in exclude_files)
|
|
or any(filepath.startswith(d+'/') for d in exclude_trees)):
|
|
continue
|
|
|
|
_, exn = path.splitext(filepath)
|
|
if not exn:
|
|
# No extension; look at the first line
|
|
with open(filepath) as f:
|
|
if re.match(r'^#!.*\bpython', f.readline()):
|
|
exn = '.py'
|
|
|
|
by_lang[exn].append(filepath)
|
|
|
|
by_lang['.sh'] = [_f for _f in map(str.strip, subprocess.check_output("grep --files-with-matches '#!.*\(ba\)\?sh' $(find scripts/ tools/ bin/ -type f | grep -v [.])", shell=True).split('\n')) if _f]
|
|
|
|
# Invoke the appropriate lint checker for each language,
|
|
# and also check files for extra whitespace.
|
|
|
|
import logging
|
|
logging.basicConfig(format="%(asctime)s %(message)s")
|
|
logger = logging.getLogger()
|
|
# Change this to logging.INFO to see performance data
|
|
logger.setLevel(logging.WARNING)
|
|
|
|
def check_pyflakes():
|
|
if not by_lang['.py']:
|
|
return False
|
|
failed = False
|
|
pyflakes = subprocess.Popen(['pyflakes'] + by_lang['.py'],
|
|
stdout = subprocess.PIPE,
|
|
stderr = subprocess.PIPE)
|
|
|
|
# pyflakes writes some output (like syntax errors) to stderr. :/
|
|
for pipe in (pyflakes.stdout, pyflakes.stderr):
|
|
for ln in pipe:
|
|
if options.full or not \
|
|
('imported but unused' in ln or
|
|
'redefinition of unused' in ln or
|
|
("zerver/models.py" in ln and
|
|
" undefined name 'bugdown'" in ln) or
|
|
("zerver/lib/tornado_ioloop_logging.py" in ln and
|
|
"redefinition of function 'instrument_tornado_ioloop'" in ln) or
|
|
("zephyr_mirror_backend.py:" in ln and
|
|
"redefinition of unused 'simplejson' from line" in ln)):
|
|
sys.stdout.write(ln)
|
|
failed = True
|
|
return failed
|
|
|
|
def custom_check_file(fn, rules, skip_rules=[]):
|
|
failed = False
|
|
for i, line in enumerate(open(fn)):
|
|
skip = False
|
|
for rule in skip_rules:
|
|
if re.match(rule, line):
|
|
skip = True
|
|
if skip:
|
|
continue
|
|
for rule in rules:
|
|
exclude_list = rule.get('exclude', set())
|
|
if fn in exclude_list:
|
|
continue
|
|
try:
|
|
if re.search(rule['pattern'], line.strip(rule.get('strip', None))):
|
|
sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1))
|
|
print(line)
|
|
failed = True
|
|
except Exception:
|
|
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
|
|
traceback.print_exc()
|
|
return failed
|
|
|
|
whitespace_rules = [
|
|
{'pattern': '\s+$',
|
|
'strip': '\n',
|
|
'description': 'Fix trailing whitespace'},
|
|
{'pattern': '\t',
|
|
'strip': '\n',
|
|
'exclude': set(['zerver/lib/bugdown/codehilite.py']),
|
|
'description': 'Fix tab-based whitespace'},
|
|
]
|
|
js_rules = [
|
|
{'pattern': '[^_]function\(',
|
|
'description': 'The keyword "function" should be followed by a space'},
|
|
{'pattern': '.*blueslip.warning\(.*',
|
|
'description': 'The module blueslip has no function warning, try using blueslip.warn'},
|
|
{'pattern': '[)]{$',
|
|
'description': 'Missing space between ) and {'},
|
|
{'pattern': 'else{$',
|
|
'description': 'Missing space between else and {'},
|
|
] + whitespace_rules
|
|
python_rules = [
|
|
{'pattern': "'[^']*'\s+\([^']*$",
|
|
'description': "Suspicious code with quoting around function name"},
|
|
{'pattern': '"[^"]*"\s+\([^"]*$',
|
|
'description': "Suspicious code with quoting around function name"},
|
|
{'pattern': '".*"%\([a-z_].*\)?$',
|
|
'description': 'Missing space around "%"'},
|
|
{'pattern': "'.*'%\([a-z_].*\)?$",
|
|
'description': 'Missing space around "%"'},
|
|
# This next check could have false positives, but it seems pretty
|
|
# rare; if we find any, they can be added to the exclude list for
|
|
# this rule.
|
|
{'pattern': '% [a-zA-Z0-9_.]*\)?$',
|
|
'description': 'Used % comprehension without a tuple'},
|
|
] + whitespace_rules
|
|
python_line_skip_rules = [
|
|
'\s*[*#]', # comments
|
|
]
|
|
bash_rules = [
|
|
{'pattern': '#!.*sh [-xe]',
|
|
'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches to set -x|set -e'},
|
|
]
|
|
|
|
def check_custom_checks():
|
|
failed = False
|
|
|
|
for fn in by_lang['.py']:
|
|
if custom_check_file(fn, python_rules, skip_rules=python_line_skip_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['.js']:
|
|
if custom_check_file(fn, js_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['.sh']:
|
|
if custom_check_file(fn, bash_rules):
|
|
failed = True
|
|
|
|
return failed
|
|
|
|
lint_functions = {}
|
|
|
|
def run_parallel():
|
|
pids = []
|
|
for name, func in lint_functions.items():
|
|
pid = os.fork()
|
|
if pid == 0:
|
|
logging.info("start " + name)
|
|
result = func()
|
|
logging.info("finish " + name)
|
|
os._exit(result)
|
|
pids.append(pid)
|
|
failed = False
|
|
|
|
for pid in pids:
|
|
(_, status) = os.waitpid(pid, 0)
|
|
if status != 0:
|
|
failed = True
|
|
return failed
|
|
|
|
def lint(func):
|
|
lint_functions[func.__name__] = func
|
|
return func
|
|
|
|
try:
|
|
# Make the lint output bright red
|
|
sys.stdout.write('\x1B[1;31m')
|
|
sys.stdout.flush()
|
|
|
|
@lint
|
|
def templates():
|
|
result = subprocess.call(['tools/check-templates'])
|
|
return result
|
|
|
|
@lint
|
|
def jslint():
|
|
result = subprocess.call(['tools/node', 'tools/jslint/check-all.js']
|
|
+ by_lang['.js'])
|
|
return result
|
|
|
|
@lint
|
|
def puppet():
|
|
if not by_lang['.pp']:
|
|
return 0
|
|
result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['.pp'])
|
|
return result
|
|
|
|
@lint
|
|
def custom():
|
|
failed = check_custom_checks()
|
|
return 1 if failed else 0
|
|
|
|
@lint
|
|
def pyflakes():
|
|
failed = check_pyflakes()
|
|
return 1 if failed else 0
|
|
|
|
failed = run_parallel()
|
|
|
|
sys.exit(1 if failed else 0)
|
|
|
|
finally:
|
|
# Restore normal terminal colors
|
|
sys.stdout.write('\x1B[0m')
|