2016-05-25 15:53:13 +02:00
|
|
|
#!/usr/bin/env python
|
2016-03-10 17:15:34 +01:00
|
|
|
from __future__ import print_function
|
2016-03-10 18:22:27 +01:00
|
|
|
from __future__ import absolute_import
|
2016-08-18 20:05:04 +02:00
|
|
|
from contextlib import contextmanager
|
2016-08-18 18:48:08 +02:00
|
|
|
import logging
|
2013-02-19 04:40:57 +01:00
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
import optparse
|
|
|
|
import subprocess
|
2016-08-18 18:42:17 +02:00
|
|
|
|
2017-02-05 21:24:28 +01:00
|
|
|
# check for the venv
|
|
|
|
from lib import sanity_check
|
|
|
|
sanity_check.check_venv(__file__)
|
|
|
|
|
|
|
|
import lister
|
|
|
|
from typing import cast, Any, Callable, Dict, Iterator, List, Optional, Tuple
|
2016-03-22 21:20:33 +01:00
|
|
|
|
2016-08-18 20:05:04 +02:00
|
|
|
@contextmanager
|
|
|
|
def bright_red_output():
|
2016-11-24 19:12:55 +01:00
|
|
|
# type: () -> Iterator[None]
|
2016-08-18 20:05:04 +02:00
|
|
|
# Make the lint output bright red
|
|
|
|
sys.stdout.write('\x1B[1;31m')
|
|
|
|
sys.stdout.flush()
|
|
|
|
try:
|
|
|
|
yield
|
|
|
|
finally:
|
|
|
|
# Restore normal terminal colors
|
|
|
|
sys.stdout.write('\x1B[0m')
|
|
|
|
|
|
|
|
|
2016-08-18 18:46:04 +02:00
|
|
|
def check_pyflakes(options, by_lang):
|
|
|
|
# type: (Any, Dict[str, List[str]]) -> bool
|
2017-03-19 21:09:54 +01:00
|
|
|
if len(by_lang['py']) == 0:
|
2016-08-18 18:46:04 +02:00
|
|
|
return False
|
|
|
|
failed = False
|
|
|
|
pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
|
2016-12-11 14:30:45 +01:00
|
|
|
stdout = subprocess.PIPE,
|
|
|
|
stderr = subprocess.PIPE,
|
|
|
|
universal_newlines = True)
|
2016-08-18 18:46:04 +02:00
|
|
|
|
|
|
|
# pyflakes writes some output (like syntax errors) to stderr. :/
|
|
|
|
for pipe in (pyflakes.stdout, pyflakes.stderr):
|
2017-06-04 11:50:28 +02:00
|
|
|
assert(pipe is not None) # onvince mypy that pipe cannot be None
|
2016-08-18 18:46:04 +02:00
|
|
|
for ln in pipe:
|
2016-12-28 04:39:44 +01:00
|
|
|
if options.full or not (
|
2016-08-18 18:46:04 +02:00
|
|
|
('imported but unused' in ln or
|
|
|
|
'redefinition of unused' in ln or
|
2016-12-28 04:39:44 +01:00
|
|
|
# Our ipython startup pythonrc file intentionally imports *
|
2016-08-18 18:46:04 +02:00
|
|
|
("scripts/lib/pythonrc.py" in ln and
|
|
|
|
" import *' used; unable to detect undefined names" in ln) or
|
2016-12-28 04:39:44 +01:00
|
|
|
# Special dev_settings.py import
|
|
|
|
"from .prod_settings_template import *" in ln or
|
2016-12-28 04:52:25 +01:00
|
|
|
("settings.py" in ln and
|
|
|
|
("settings import *' used; unable to detect undefined names" in ln or
|
|
|
|
"may be undefined, or defined from star imports" in ln)) or
|
2016-11-27 04:40:18 +01:00
|
|
|
("zerver/tornado/ioloop_logging.py" in ln and
|
2016-08-18 18:46:04 +02:00
|
|
|
"redefinition of function 'instrument_tornado_ioloop'" in ln) or
|
|
|
|
("zephyr_mirror_backend.py:" in ln and
|
2016-12-28 04:39:44 +01:00
|
|
|
"redefinition of unused 'simplejson' from line" in ln))):
|
2016-08-18 18:46:04 +02:00
|
|
|
sys.stdout.write(ln)
|
|
|
|
failed = True
|
|
|
|
return failed
|
|
|
|
|
2016-11-09 14:07:29 +01:00
|
|
|
|
|
|
|
def check_pep8(files):
|
|
|
|
# type: (List[str]) -> bool
|
2017-06-04 13:10:32 +02:00
|
|
|
|
|
|
|
def run_pycodestyle(files, ignored_rules):
|
|
|
|
# type: (List[str], List[str]) -> bool
|
|
|
|
failed = False
|
|
|
|
pep8 = subprocess.Popen(
|
|
|
|
['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))],
|
|
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
|
|
|
for pipe in (pep8.stdout, pep8.stderr):
|
|
|
|
assert(pipe is not None) # convince mypy that pipe cannot be None
|
|
|
|
for ln in pipe:
|
|
|
|
sys.stdout.write(ln)
|
|
|
|
failed = True
|
|
|
|
return failed
|
|
|
|
|
2016-11-09 14:07:29 +01:00
|
|
|
failed = False
|
|
|
|
ignored_rules = [
|
2017-01-24 05:19:58 +01:00
|
|
|
# Each of these rules are ignored for the explained reason.
|
|
|
|
|
|
|
|
# "multiple spaces before operator"
|
|
|
|
# There are several typos here, but also several instances that are
|
|
|
|
# being used for alignment in dict keys/values using the `dict`
|
|
|
|
# constructor. We could fix the alignment cases by switching to the `{}`
|
|
|
|
# constructor, but it makes fixing this rule a little less
|
|
|
|
# straightforward.
|
|
|
|
'E221',
|
|
|
|
|
2017-01-24 06:43:33 +01:00
|
|
|
# 'missing whitespace around arithmetic operator'
|
|
|
|
# This should possibly be cleaned up, though changing some of
|
|
|
|
# these may make the code less readable.
|
|
|
|
'E226',
|
|
|
|
|
2017-01-24 05:19:58 +01:00
|
|
|
# "unexpected spaces around keyword / parameter equals"
|
|
|
|
# Many of these should be fixed, but many are also being used for
|
|
|
|
# alignment/making the code easier to read.
|
|
|
|
'E251',
|
|
|
|
|
|
|
|
# "block comment should start with '#'"
|
|
|
|
# These serve to show which lines should be changed in files customized
|
|
|
|
# by the user. We could probably resolve one of E265 or E266 by
|
|
|
|
# standardizing on a single style for lines that the user might want to
|
|
|
|
# change.
|
|
|
|
'E265',
|
|
|
|
|
|
|
|
# "too many leading '#' for block comment"
|
|
|
|
# Most of these are there for valid reasons.
|
|
|
|
'E266',
|
|
|
|
|
2017-01-24 06:43:33 +01:00
|
|
|
# "expected 2 blank lines after class or function definition"
|
|
|
|
# Zulip only uses 1 blank line after class/function
|
|
|
|
# definitions; the PEP-8 recommendation results in super sparse code.
|
|
|
|
'E302', 'E305',
|
|
|
|
|
2017-01-24 05:19:58 +01:00
|
|
|
# "module level import not at top of file"
|
|
|
|
# Most of these are there for valid reasons, though there might be a
|
2017-01-24 06:43:33 +01:00
|
|
|
# few that could be eliminated.
|
2017-01-24 05:19:58 +01:00
|
|
|
'E402',
|
|
|
|
|
2017-01-24 06:43:33 +01:00
|
|
|
# "line too long"
|
|
|
|
# Zulip is a bit less strict about line length, and has its
|
|
|
|
# own check for this (see max_length)
|
|
|
|
'E501',
|
|
|
|
|
2017-01-24 05:19:58 +01:00
|
|
|
# "do not assign a lambda expression, use a def"
|
|
|
|
# Fixing these would probably reduce readability in most cases.
|
|
|
|
'E731',
|
2016-11-09 14:07:29 +01:00
|
|
|
]
|
2017-06-04 13:10:32 +02:00
|
|
|
|
|
|
|
# TODO: Clear up this list of violations.
|
|
|
|
IGNORE_FILES_PEPE261 = [
|
|
|
|
'api/zulip/__init__.py',
|
|
|
|
'tools/run-dev.py',
|
|
|
|
'zerver/lib/bugdown/__init__.py',
|
|
|
|
'zerver/models.py',
|
|
|
|
'zerver/tests/test_bugdown.py',
|
|
|
|
'zerver/tests/test_events.py',
|
|
|
|
'zerver/tests/test_messages.py',
|
|
|
|
'zerver/tests/test_narrow.py',
|
|
|
|
'zerver/tests/test_outgoing_webhook_system.py',
|
|
|
|
'zerver/tests/test_realm.py',
|
|
|
|
'zerver/tests/test_signup.py',
|
|
|
|
'zerver/tests/test_subs.py',
|
|
|
|
'zerver/tests/test_upload.py',
|
|
|
|
'zerver/tornado/socket.py',
|
|
|
|
'zerver/tornado/websocket_client.py',
|
|
|
|
'zerver/worker/queue_processors.py',
|
|
|
|
'zilencer/management/commands/populate_db.py',
|
|
|
|
'zproject/dev_settings.py',
|
|
|
|
'zproject/prod_settings_template.py',
|
|
|
|
'zproject/settings.py',
|
|
|
|
]
|
|
|
|
|
|
|
|
filtered_files = [fn for fn in files if fn not in IGNORE_FILES_PEPE261]
|
|
|
|
filtered_files_E261 = [fn for fn in files if fn in IGNORE_FILES_PEPE261]
|
|
|
|
|
2017-02-07 22:19:43 +01:00
|
|
|
if len(files) == 0:
|
|
|
|
return False
|
2017-06-04 13:10:32 +02:00
|
|
|
if not len(filtered_files) == 0:
|
|
|
|
failed = run_pycodestyle(filtered_files, ignored_rules)
|
|
|
|
if not len(filtered_files_E261) == 0:
|
|
|
|
# Adding an extra ignore rule for these files since they still remain in
|
|
|
|
# violation of PEP-E261.
|
|
|
|
failed = run_pycodestyle(filtered_files_E261, ignored_rules + ['E261'])
|
|
|
|
|
2016-11-09 14:07:29 +01:00
|
|
|
return failed
|
|
|
|
|
|
|
|
|
2016-08-18 18:48:08 +02:00
|
|
|
def run_parallel(lint_functions):
|
|
|
|
# type: (Dict[str, Callable[[], int]]) -> bool
|
|
|
|
pids = []
|
|
|
|
for name, func in lint_functions.items():
|
|
|
|
pid = os.fork()
|
|
|
|
if pid == 0:
|
|
|
|
logging.info("start " + name)
|
|
|
|
result = func()
|
|
|
|
logging.info("finish " + name)
|
|
|
|
sys.stdout.flush()
|
|
|
|
sys.stderr.flush()
|
|
|
|
os._exit(result)
|
|
|
|
pids.append(pid)
|
|
|
|
failed = False
|
|
|
|
|
|
|
|
for pid in pids:
|
|
|
|
(_, status) = os.waitpid(pid, 0)
|
|
|
|
if status != 0:
|
|
|
|
failed = True
|
|
|
|
return failed
|
|
|
|
|
2016-08-18 20:05:42 +02:00
|
|
|
def run():
|
|
|
|
# type: () -> None
|
|
|
|
parser = optparse.OptionParser()
|
2016-11-23 18:28:21 +01:00
|
|
|
parser.add_option('--force', default=False,
|
2016-12-11 14:30:45 +01:00
|
|
|
action="store_true",
|
|
|
|
help='Run tests despite possible problems.')
|
2016-08-18 20:05:42 +02:00
|
|
|
parser.add_option('--full',
|
2016-12-11 14:30:45 +01:00
|
|
|
action='store_true',
|
|
|
|
help='Check some things we typically ignore')
|
2016-11-09 14:07:29 +01:00
|
|
|
parser.add_option('--pep8',
|
2016-12-11 14:30:45 +01:00
|
|
|
action='store_true',
|
|
|
|
help='Run the pep8 checker')
|
2017-03-22 21:18:48 +01:00
|
|
|
parser.add_option('--no-gitlint',
|
|
|
|
action='store_true',
|
|
|
|
help='Disable gitlint')
|
2016-08-18 20:05:42 +02:00
|
|
|
parser.add_option('--modified', '-m',
|
2016-12-11 14:30:45 +01:00
|
|
|
action='store_true',
|
|
|
|
help='Only check modified files')
|
2016-08-18 20:05:42 +02:00
|
|
|
parser.add_option('--verbose', '-v',
|
2016-12-11 14:30:45 +01:00
|
|
|
action='store_true',
|
|
|
|
help='Print verbose timing output')
|
2016-08-18 20:05:42 +02:00
|
|
|
(options, args) = parser.parse_args()
|
|
|
|
|
2016-11-23 18:28:21 +01:00
|
|
|
tools_dir = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
root_dir = os.path.dirname(tools_dir)
|
|
|
|
sys.path.insert(0, root_dir)
|
|
|
|
|
2017-06-05 16:29:04 +02:00
|
|
|
from tools.linter_lib.custom_check import build_custom_checkers
|
2017-06-05 16:34:21 +02:00
|
|
|
from tools.linter_lib.exclude import EXCLUDED_FILES
|
2017-06-05 16:29:04 +02:00
|
|
|
|
2016-11-23 18:28:21 +01:00
|
|
|
from tools.lib.test_script import (
|
|
|
|
get_provisioning_status,
|
|
|
|
)
|
|
|
|
|
|
|
|
os.chdir(root_dir)
|
|
|
|
|
|
|
|
if not options.force:
|
|
|
|
ok, msg = get_provisioning_status()
|
|
|
|
if not ok:
|
|
|
|
print(msg)
|
|
|
|
print('If you really know what you are doing, use --force to run anyway.')
|
|
|
|
sys.exit(1)
|
2016-08-18 20:05:42 +02:00
|
|
|
|
2016-12-11 14:30:45 +01:00
|
|
|
by_lang = cast(Dict[str, List[str]],
|
|
|
|
lister.list_files(args, modified_only=options.modified,
|
|
|
|
ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars',
|
2017-01-24 02:10:38 +01:00
|
|
|
'html', 'json', 'md', 'txt', 'text', 'yaml'],
|
2016-12-11 14:30:45 +01:00
|
|
|
use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES))
|
2016-08-18 20:05:42 +02:00
|
|
|
|
|
|
|
# Invoke the appropriate lint checker for each language,
|
|
|
|
# and also check files for extra whitespace.
|
|
|
|
|
|
|
|
logging.basicConfig(format="%(asctime)s %(message)s")
|
|
|
|
logger = logging.getLogger()
|
|
|
|
if options.verbose:
|
|
|
|
logger.setLevel(logging.INFO)
|
|
|
|
else:
|
|
|
|
logger.setLevel(logging.WARNING)
|
|
|
|
|
|
|
|
check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)
|
|
|
|
|
2017-05-17 23:02:38 +02:00
|
|
|
lint_functions = {} # type: Dict[str, Callable[[], int]]
|
2016-08-18 18:42:17 +02:00
|
|
|
|
|
|
|
def lint(func):
|
|
|
|
# type: (Callable[[], int]) -> Callable[[], int]
|
|
|
|
lint_functions[func.__name__] = func
|
|
|
|
return func
|
|
|
|
|
2017-03-19 21:25:20 +01:00
|
|
|
def external_linter(name, command, target_langs=[]):
|
2017-06-04 01:02:24 +02:00
|
|
|
# type: (str, List[str], List[str]) -> None
|
2017-06-02 23:48:40 +02:00
|
|
|
"""Registers an external linter program to be run as part of the
|
|
|
|
linter. This program will be passed the subset of files being
|
|
|
|
linted that have extensions in target_langs. If there are no
|
|
|
|
such files, exits without doing anything.
|
|
|
|
|
|
|
|
If target_langs is empty, just runs the linter unconditionally.
|
|
|
|
"""
|
2017-03-19 21:25:20 +01:00
|
|
|
def run_linter():
|
2016-11-04 23:15:18 +01:00
|
|
|
# type: () -> int
|
2017-03-19 21:25:20 +01:00
|
|
|
if len(target_langs) == 0:
|
|
|
|
return subprocess.call(command)
|
|
|
|
targets = [target for lang in target_langs for target in by_lang[lang]]
|
|
|
|
if len(targets) == 0:
|
2016-11-19 01:18:08 +01:00
|
|
|
return 0
|
2017-03-19 21:25:20 +01:00
|
|
|
return subprocess.call(command + targets)
|
|
|
|
lint_functions[name] = run_linter
|
2016-11-04 23:15:18 +01:00
|
|
|
|
2017-03-19 21:25:20 +01:00
|
|
|
with bright_red_output():
|
|
|
|
external_linter('add_class', ['tools/find-add-class'])
|
|
|
|
external_linter('css', ['tools/check-css'], ['css'])
|
|
|
|
external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet'], ['js'])
|
2017-05-24 20:31:00 +02:00
|
|
|
external_linter('tslint', ['node', 'node_modules/.bin/tslint', '-c',
|
|
|
|
'static/ts/tslint.json'], ['ts'])
|
2017-03-19 21:25:20 +01:00
|
|
|
external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'])
|
|
|
|
external_linter('templates', ['tools/check-templates'], ['handlebars', 'html'])
|
|
|
|
external_linter('urls', ['tools/check-urls'])
|
2017-06-02 23:12:26 +02:00
|
|
|
external_linter('swagger', ['node', 'tools/check-swagger'], ['yaml'])
|
2016-08-18 18:42:17 +02:00
|
|
|
|
2017-04-28 21:25:53 +02:00
|
|
|
# gitlint disabled until we can stabilize it more
|
|
|
|
# if not options.no_gitlint:
|
|
|
|
# external_linter('commit_messages', ['tools/commit-message-lint'])
|
2017-03-22 21:18:48 +01:00
|
|
|
|
2016-08-18 18:42:17 +02:00
|
|
|
@lint
|
|
|
|
def custom_py():
|
|
|
|
# type: () -> int
|
|
|
|
failed = check_custom_checks_py()
|
|
|
|
return 1 if failed else 0
|
|
|
|
|
|
|
|
@lint
|
|
|
|
def custom_nonpy():
|
|
|
|
# type: () -> int
|
|
|
|
failed = check_custom_checks_nonpy()
|
|
|
|
return 1 if failed else 0
|
|
|
|
|
|
|
|
@lint
|
|
|
|
def pyflakes():
|
|
|
|
# type: () -> int
|
2016-08-18 18:46:04 +02:00
|
|
|
failed = check_pyflakes(options, by_lang)
|
2016-08-18 18:42:17 +02:00
|
|
|
return 1 if failed else 0
|
|
|
|
|
2016-11-09 14:07:29 +01:00
|
|
|
if options.pep8:
|
|
|
|
@lint
|
|
|
|
def pep8():
|
|
|
|
# type: () -> int
|
|
|
|
failed = check_pep8(by_lang['py'])
|
|
|
|
return 1 if failed else 0
|
|
|
|
|
2016-08-18 18:48:08 +02:00
|
|
|
failed = run_parallel(lint_functions)
|
2016-08-18 18:42:17 +02:00
|
|
|
|
2016-08-18 20:05:04 +02:00
|
|
|
sys.exit(1 if failed else 0)
|
2016-08-18 18:42:17 +02:00
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
run()
|