zulip/tools/lint

327 lines
12 KiB
Plaintext
Raw Normal View History

#!/usr/bin/env python
from __future__ import print_function
from __future__ import absolute_import
from contextlib import contextmanager
import logging
import os
import sys
import optparse
import subprocess
# check for the venv
from lib import sanity_check
sanity_check.check_venv(__file__)
import lister
from typing import cast, Any, Callable, Dict, Iterator, List, Optional, Tuple
@contextmanager
def bright_red_output():
# type: () -> Iterator[None]
# Make the lint output bright red
sys.stdout.write('\x1B[1;31m')
sys.stdout.flush()
try:
yield
finally:
# Restore normal terminal colors
sys.stdout.write('\x1B[0m')
def check_pyflakes(options, by_lang):
# type: (Any, Dict[str, List[str]]) -> bool
if len(by_lang['py']) == 0:
return False
failed = False
pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
universal_newlines = True)
# pyflakes writes some output (like syntax errors) to stderr. :/
for pipe in (pyflakes.stdout, pyflakes.stderr):
assert(pipe is not None) # onvince mypy that pipe cannot be None
for ln in pipe:
if options.full or not (
('imported but unused' in ln or
'redefinition of unused' in ln or
# Our ipython startup pythonrc file intentionally imports *
("scripts/lib/pythonrc.py" in ln and
" import *' used; unable to detect undefined names" in ln) or
# Special dev_settings.py import
"from .prod_settings_template import *" in ln or
("settings.py" in ln and
("settings import *' used; unable to detect undefined names" in ln or
"may be undefined, or defined from star imports" in ln)) or
("zerver/tornado/ioloop_logging.py" in ln and
"redefinition of function 'instrument_tornado_ioloop'" in ln) or
("zephyr_mirror_backend.py:" in ln and
"redefinition of unused 'simplejson' from line" in ln))):
sys.stdout.write(ln)
failed = True
return failed
def check_pep8(files):
# type: (List[str]) -> bool
def run_pycodestyle(files, ignored_rules):
# type: (List[str], List[str]) -> bool
failed = False
pep8 = subprocess.Popen(
['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
for pipe in (pep8.stdout, pep8.stderr):
assert(pipe is not None) # convince mypy that pipe cannot be None
for ln in pipe:
sys.stdout.write(ln)
failed = True
return failed
failed = False
ignored_rules = [
# Each of these rules are ignored for the explained reason.
# "multiple spaces before operator"
# There are several typos here, but also several instances that are
# being used for alignment in dict keys/values using the `dict`
# constructor. We could fix the alignment cases by switching to the `{}`
# constructor, but it makes fixing this rule a little less
# straightforward.
'E221',
# 'missing whitespace around arithmetic operator'
# This should possibly be cleaned up, though changing some of
# these may make the code less readable.
'E226',
# "unexpected spaces around keyword / parameter equals"
# Many of these should be fixed, but many are also being used for
# alignment/making the code easier to read.
'E251',
# "block comment should start with '#'"
# These serve to show which lines should be changed in files customized
# by the user. We could probably resolve one of E265 or E266 by
# standardizing on a single style for lines that the user might want to
# change.
'E265',
# "too many leading '#' for block comment"
# Most of these are there for valid reasons.
'E266',
# "expected 2 blank lines after class or function definition"
# Zulip only uses 1 blank line after class/function
# definitions; the PEP-8 recommendation results in super sparse code.
'E302', 'E305',
# "module level import not at top of file"
# Most of these are there for valid reasons, though there might be a
# few that could be eliminated.
'E402',
# "line too long"
# Zulip is a bit less strict about line length, and has its
# own check for this (see max_length)
'E501',
# "do not assign a lambda expression, use a def"
# Fixing these would probably reduce readability in most cases.
'E731',
]
# TODO: Clear up this list of violations.
IGNORE_FILES_PEPE261 = [
'api/zulip/__init__.py',
'tools/run-dev.py',
'zerver/lib/bugdown/__init__.py',
'zerver/models.py',
'zerver/tests/test_bugdown.py',
'zerver/tests/test_events.py',
'zerver/tests/test_messages.py',
'zerver/tests/test_narrow.py',
'zerver/tests/test_outgoing_webhook_system.py',
'zerver/tests/test_realm.py',
'zerver/tests/test_signup.py',
'zerver/tests/test_subs.py',
'zerver/tests/test_upload.py',
'zerver/tornado/socket.py',
'zerver/tornado/websocket_client.py',
'zerver/worker/queue_processors.py',
'zilencer/management/commands/populate_db.py',
'zproject/dev_settings.py',
'zproject/prod_settings_template.py',
'zproject/settings.py',
]
filtered_files = [fn for fn in files if fn not in IGNORE_FILES_PEPE261]
filtered_files_E261 = [fn for fn in files if fn in IGNORE_FILES_PEPE261]
if len(files) == 0:
return False
if not len(filtered_files) == 0:
failed = run_pycodestyle(filtered_files, ignored_rules)
if not len(filtered_files_E261) == 0:
# Adding an extra ignore rule for these files since they still remain in
# violation of PEP-E261.
failed = run_pycodestyle(filtered_files_E261, ignored_rules + ['E261'])
return failed
def run_parallel(lint_functions):
# type: (Dict[str, Callable[[], int]]) -> bool
pids = []
for name, func in lint_functions.items():
pid = os.fork()
if pid == 0:
logging.info("start " + name)
result = func()
logging.info("finish " + name)
sys.stdout.flush()
sys.stderr.flush()
os._exit(result)
pids.append(pid)
failed = False
for pid in pids:
(_, status) = os.waitpid(pid, 0)
if status != 0:
failed = True
return failed
def run():
# type: () -> None
parser = optparse.OptionParser()
parser.add_option('--force', default=False,
action="store_true",
help='Run tests despite possible problems.')
parser.add_option('--full',
action='store_true',
help='Check some things we typically ignore')
parser.add_option('--pep8',
action='store_true',
help='Run the pep8 checker')
parser.add_option('--no-gitlint',
action='store_true',
help='Disable gitlint')
parser.add_option('--modified', '-m',
action='store_true',
help='Only check modified files')
parser.add_option('--verbose', '-v',
action='store_true',
help='Print verbose timing output')
(options, args) = parser.parse_args()
tools_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.dirname(tools_dir)
sys.path.insert(0, root_dir)
from tools.linter_lib.custom_check import build_custom_checkers
2017-06-05 16:34:21 +02:00
from tools.linter_lib.exclude import EXCLUDED_FILES
from tools.lib.test_script import (
get_provisioning_status,
)
os.chdir(root_dir)
if not options.force:
ok, msg = get_provisioning_status()
if not ok:
print(msg)
print('If you really know what you are doing, use --force to run anyway.')
sys.exit(1)
by_lang = cast(Dict[str, List[str]],
lister.list_files(args, modified_only=options.modified,
ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars',
'html', 'json', 'md', 'txt', 'text', 'yaml'],
use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES))
# Invoke the appropriate lint checker for each language,
# and also check files for extra whitespace.
logging.basicConfig(format="%(asctime)s %(message)s")
logger = logging.getLogger()
if options.verbose:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.WARNING)
check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)
lint_functions = {} # type: Dict[str, Callable[[], int]]
def lint(func):
# type: (Callable[[], int]) -> Callable[[], int]
lint_functions[func.__name__] = func
return func
def external_linter(name, command, target_langs=[]):
# type: (str, List[str], List[str]) -> None
"""Registers an external linter program to be run as part of the
linter. This program will be passed the subset of files being
linted that have extensions in target_langs. If there are no
such files, exits without doing anything.
If target_langs is empty, just runs the linter unconditionally.
"""
def run_linter():
# type: () -> int
if len(target_langs) == 0:
return subprocess.call(command)
targets = [target for lang in target_langs for target in by_lang[lang]]
if len(targets) == 0:
return 0
return subprocess.call(command + targets)
lint_functions[name] = run_linter
with bright_red_output():
external_linter('add_class', ['tools/find-add-class'])
external_linter('css', ['tools/check-css'], ['css'])
external_linter('eslint', ['node', 'node_modules/.bin/eslint', '--quiet'], ['js'])
external_linter('tslint', ['node', 'node_modules/.bin/tslint', '-c',
'static/ts/tslint.json'], ['ts'])
external_linter('puppet', ['puppet', 'parser', 'validate'], ['pp'])
external_linter('templates', ['tools/check-templates'], ['handlebars', 'html'])
external_linter('urls', ['tools/check-urls'])
external_linter('swagger', ['node', 'tools/check-swagger'], ['yaml'])
# gitlint disabled until we can stabilize it more
# if not options.no_gitlint:
# external_linter('commit_messages', ['tools/commit-message-lint'])
@lint
def custom_py():
# type: () -> int
failed = check_custom_checks_py()
return 1 if failed else 0
@lint
def custom_nonpy():
# type: () -> int
failed = check_custom_checks_nonpy()
return 1 if failed else 0
@lint
def pyflakes():
# type: () -> int
failed = check_pyflakes(options, by_lang)
return 1 if failed else 0
if options.pep8:
@lint
def pep8():
# type: () -> int
failed = check_pep8(by_lang['py'])
return 1 if failed else 0
failed = run_parallel(lint_functions)
sys.exit(1 if failed else 0)
if __name__ == '__main__':
run()