mirror of https://github.com/zulip/zulip.git
zulint: Use zulint from the extracted repository.
zulint will be added as a "third-party" dependency in zulip from now on. See the new project at https://github.com/zulip/zulint for more details.
This commit is contained in:
Aman
Tim Abbott
parent
618d026941
commit
2183a74040
|
|
@ -52,4 +52,7 @@ python-digitalocean==1.14.0
|
|||
# Needed for updating the locked pip dependencies
|
||||
pip-tools==2.0.2
|
||||
|
||||
# zulip's linting framework - zulint
|
||||
-e git+https://github.com/zulip/zulint@master#egg=zulint==0.0.1
|
||||
|
||||
-r mypy.in
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ git+https://github.com/zulip/libthumbor.git@60ed2431c07686a12f2770b2d852c5650f3c
|
|||
git+https://github.com/zulip/line_profiler.git#egg=line_profiler==2.1.2.zulip1
|
||||
git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1
|
||||
git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git
|
||||
git+https://github.com/zulip/zulint@master#egg=zulint==0.0.1
|
||||
git+https://github.com/zulip/python-zulip-api.git@0.6.1#egg=zulip==0.6.1_git&subdirectory=zulip
|
||||
git+https://github.com/zulip/python-zulip-api.git@0.6.1#egg=zulip_bots==0.6.1+git&subdirectory=zulip_bots
|
||||
alabaster==0.7.12 # via sphinx
|
||||
|
|
|
|||
|
|
@ -335,7 +335,6 @@ python_rules = RuleList(
|
|||
'include_only': set([
|
||||
'scripts/',
|
||||
'puppet/',
|
||||
'tools/zulint/',
|
||||
]),
|
||||
'exclude': set([
|
||||
# Not important, but should fix
|
||||
|
|
@ -499,7 +498,6 @@ python_rules = RuleList(
|
|||
# We are likely to want to keep these dirs Python 2+3 compatible,
|
||||
# since the plan includes extracting them to a separate project eventually.
|
||||
'tools/lib',
|
||||
'tools/zulint',
|
||||
# TODO: Update our migrations from Text->str.
|
||||
'zerver/migrations/',
|
||||
# thumbor is (currently) python2 only
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from zulint.custom_rules import RuleList
|
|||
from linter_lib.custom_check import python_rules, non_py_rules
|
||||
|
||||
ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
|
||||
CHECK_MESSAGE = "Fix the corresponding rule in `tools/zulint/custom_rules.py`."
|
||||
CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`."
|
||||
|
||||
class TestRuleList(TestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
# zulint
|
||||
|
||||
zulint is a lightweight linting framework designed for complex
|
||||
applications using a mix of third-party linters and custom rules.
|
||||
|
||||
## Why zulint
|
||||
|
||||
Modern full-stack web applications generally involve code written in
|
||||
several programming languages, each of which have their own standard
|
||||
linter tools. For example, [Zulip](https://zulipchat.com) uses Python
|
||||
(mypy/pyflake/pycodestyle), JavaScript (eslint), CSS (stylelint),
|
||||
puppet (puppet-lint), shell (shellcheck), and several more. For many
|
||||
codebases, this results in linting being an unpleasantly slow
|
||||
experience, resulting in even more unpleasant secondary problems like
|
||||
developers merging code that doesn't pass lint, not enforcing linter
|
||||
rules, and debates about whether a useful linter is "worth the time".
|
||||
|
||||
Zulint is the linter framework we built for Zulip to create a
|
||||
reliable, lightning-fast linter experience to solve these problems.
|
||||
It has the following features:
|
||||
|
||||
- Integrates with `git` to only checks files in source control (not
|
||||
automatically generated, untracked, or .gitignore files).
|
||||
- Runs the linters in parallel, so you only have to wait for the
|
||||
slowest linter. For Zulip, this is a ~4x performance improvement
|
||||
over running our third-party linters in series.
|
||||
- Produduces easy-to-read, clear terminal output, with each
|
||||
independent linter given its own color.
|
||||
- Can check just modified files, or even as a `pre-commit` hook, only
|
||||
checking files that have changed (and only starting linters which
|
||||
check files that have changed).
|
||||
- Handles all the annoying details of flushing stdout and managing
|
||||
color codes.
|
||||
- Highly configurable.
|
||||
- Integrate a third-party linter with just a couple lines of code.
|
||||
- Every feature supports convenient include/exclude rules.
|
||||
- Add custom lint rules with a powerful regular expression
|
||||
framework. E.g. in Zulip, we want all access to `Message` objects
|
||||
in views code to be done via our `access_message_by_id` functions
|
||||
(which do security checks to ensure the user the request is being
|
||||
done on behalf of has access to the message), and that is enforced
|
||||
in part by custom regular expression lint rules. This system is
|
||||
optimized Python: Zulip has a few hundred custom linter rules of
|
||||
this type.
|
||||
- Easily add custom options to check subsets of your codebase,
|
||||
subsets of rules, etc.
|
||||
- Has a nice automated testing framework for custom lint rules, so you
|
||||
can make sure your rules actually work.
|
||||
|
||||
This codebase has been in production use in Zulip for several years,
|
||||
but only in 2019 was generalized for use by other projects. Its API
|
||||
to be beta and may change (with notice in the release notes) if we
|
||||
discover a better API, and patches to further extend it for more use
|
||||
cases are encouraged.
|
||||
|
||||
## Using zulint
|
||||
|
||||
Once a project is setup with zulint, you'll have a top-level linter
|
||||
script with at least the following options:
|
||||
|
||||
```
|
||||
(zulip-py3-venv) tabbott@coset:~/zulip$ ./tools/lint --help
|
||||
usage: lint [-h] [--force] [--full] [--modified] [--verbose-timing]
|
||||
[--skip SKIP] [--only ONLY] [--list] [--groups GROUPS]
|
||||
[targets [targets ...]]
|
||||
|
||||
positional arguments:
|
||||
targets Specify directories to check
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--force Run tests despite possible problems.
|
||||
--modified, -m Only check modified files
|
||||
--verbose-timing, -vt
|
||||
Print verbose timing output
|
||||
--skip SKIP Specify linters to skip, eg: --skip=mypy,gitlint
|
||||
--only ONLY Specify linters to run, eg: --only=mypy,gitlint
|
||||
--list, -l List all the registered linters
|
||||
--groups GROUPS, -g GROUPS
|
||||
Only run linter for languages in the group(s), e.g.:
|
||||
--groups=backend,other_group
|
||||
```
|
||||
|
||||
### pre-commit hook mode
|
||||
|
||||
See https://github.com/zulip/zulip/blob/master/tools/pre-commit for an
|
||||
example pre-commit hook (Zulip's has some extra complexity because we
|
||||
use Vagrant from our development environment, and want to be able to
|
||||
run the hook from outside Vagrant).
|
||||
|
||||
## Adding zulint to a codebase
|
||||
|
||||
TODO. Will roughly include `pip install zulint`, copying an example
|
||||
`lint` script, and adding your rules.
|
||||
|
||||
|
||||
## Adding third-party linters
|
||||
|
||||
TODO: Document the linter_config API.
|
||||
|
||||
## Writing custom rules
|
||||
|
||||
TODO: Document all the features of the `RuleList` and `custom_check` system.
|
||||
|
||||
|
|
@ -1,185 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
if False:
|
||||
# See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from zulint.printer import print_err, colors, BOLDRED, BLUE, GREEN, ENDC
|
||||
from zulint import lister
|
||||
|
||||
def add_default_linter_arguments(parser):
|
||||
# type: (argparse.ArgumentParser) -> None
|
||||
parser.add_argument('--modified', '-m',
|
||||
action='store_true',
|
||||
help='Only check modified files')
|
||||
parser.add_argument('--verbose-timing', '-vt',
|
||||
action='store_true',
|
||||
help='Print verbose timing output')
|
||||
parser.add_argument('targets',
|
||||
nargs='*',
|
||||
help='Specify directories to check')
|
||||
parser.add_argument('--skip',
|
||||
default=[],
|
||||
type=split_arg_into_list,
|
||||
help='Specify linters to skip, eg: --skip=mypy,gitlint')
|
||||
parser.add_argument('--only',
|
||||
default=[],
|
||||
type=split_arg_into_list,
|
||||
help='Specify linters to run, eg: --only=mypy,gitlint')
|
||||
parser.add_argument('--list', '-l',
|
||||
action='store_true',
|
||||
help='List all the registered linters')
|
||||
parser.add_argument('--list-groups', '-lg',
|
||||
action='store_true',
|
||||
help='List all the registered linter groups')
|
||||
parser.add_argument('--groups', '-g',
|
||||
default=[],
|
||||
type=split_arg_into_list,
|
||||
help='Only run linter for languages in the group(s), e.g.: '
|
||||
'--groups=backend,frontend')
|
||||
parser.add_argument('--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Print verbose output where available')
|
||||
parser.add_argument('--fix',
|
||||
action='store_true',
|
||||
help='Automatically fix problems where supported')
|
||||
|
||||
def split_arg_into_list(arg):
|
||||
# type: (str) -> List[str]
|
||||
return [linter for linter in arg.split(',')]
|
||||
|
||||
def run_parallel(lint_functions):
|
||||
# type: (Dict[str, Callable[[], int]]) -> bool
|
||||
pids = []
|
||||
for name, func in lint_functions.items():
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
logging.info("start " + name)
|
||||
result = func()
|
||||
logging.info("finish " + name)
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
os._exit(result)
|
||||
pids.append(pid)
|
||||
failed = False
|
||||
|
||||
for pid in pids:
|
||||
(_, status) = os.waitpid(pid, 0)
|
||||
if status != 0:
|
||||
failed = True
|
||||
return failed
|
||||
|
||||
class LinterConfig:
|
||||
lint_functions = {} # type: Dict[str, Callable[[], int]]
|
||||
lint_descriptions = {} # type: Dict[str, str]
|
||||
|
||||
def __init__(self, args):
|
||||
# type: (argparse.Namespace) -> None
|
||||
self.args = args
|
||||
self.by_lang = {} # type: Dict[str, List[str]]
|
||||
self.groups = {} # type: Dict[str, List[str]]
|
||||
|
||||
def list_files(self, file_types=[], groups={}, use_shebang=True, group_by_ftype=True, exclude=[]):
|
||||
# type: (List[str], Dict[str, List[str]], bool, bool, List[str]) -> Dict[str, List[str]]
|
||||
assert file_types or groups, "Atleast one of `file_types` or `groups` must be specified."
|
||||
|
||||
self.groups = groups
|
||||
if self.args.groups:
|
||||
file_types = [ft for group in self.args.groups for ft in groups[group]]
|
||||
else:
|
||||
file_types.extend({ft for group in groups.values() for ft in group})
|
||||
|
||||
self.by_lang = lister.list_files(self.args.targets, modified_only=self.args.modified,
|
||||
ftypes=file_types, use_shebang=use_shebang,
|
||||
group_by_ftype=group_by_ftype, exclude=exclude)
|
||||
return self.by_lang
|
||||
|
||||
def lint(self, func):
|
||||
# type: (Callable[[], int]) -> Callable[[], int]
|
||||
self.lint_functions[func.__name__] = func
|
||||
self.lint_descriptions[func.__name__] = func.__doc__ if func.__doc__ else "External Linter"
|
||||
return func
|
||||
|
||||
def external_linter(self, name, command, target_langs=[], pass_targets=True, fix_arg=None,
|
||||
description="External Linter"):
|
||||
# type: (str, List[str], List[str], bool, Optional[str], str) -> None
|
||||
"""Registers an external linter program to be run as part of the
|
||||
linter. This program will be passed the subset of files being
|
||||
linted that have extensions in target_langs. If there are no
|
||||
such files, exits without doing anything.
|
||||
|
||||
If target_langs is empty, just runs the linter unconditionally.
|
||||
"""
|
||||
self.lint_descriptions[name] = description
|
||||
color = next(colors)
|
||||
|
||||
def run_linter():
|
||||
# type: () -> int
|
||||
targets = [] # type: List[str]
|
||||
if len(target_langs) != 0:
|
||||
targets = [target for lang in target_langs for target in self.by_lang[lang]]
|
||||
if len(targets) == 0:
|
||||
# If this linter has a list of languages, and
|
||||
# no files in those languages are to be checked,
|
||||
# then we can safely return success without
|
||||
# invoking the external linter.
|
||||
return 0
|
||||
|
||||
if self.args.fix and fix_arg:
|
||||
command.append(fix_arg)
|
||||
|
||||
if pass_targets:
|
||||
full_command = command + targets
|
||||
else:
|
||||
full_command = command
|
||||
p = subprocess.Popen(full_command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
assert p.stdout # use of subprocess.PIPE indicates non-None
|
||||
for line in iter(p.stdout.readline, b''):
|
||||
print_err(name, color, line)
|
||||
|
||||
return p.wait() # Linter exit code
|
||||
|
||||
self.lint_functions[name] = run_linter
|
||||
|
||||
def set_logger(self):
|
||||
# type: () -> None
|
||||
logging.basicConfig(format="%(asctime)s %(message)s")
|
||||
logger = logging.getLogger()
|
||||
if self.args.verbose_timing:
|
||||
logger.setLevel(logging.INFO)
|
||||
else:
|
||||
logger.setLevel(logging.WARNING)
|
||||
|
||||
def do_lint(self):
|
||||
# type: () -> None
|
||||
assert not self.args.only or not self.args.skip, "Only one of --only or --skip can be used at once."
|
||||
if self.args.only:
|
||||
self.lint_functions = {linter: self.lint_functions[linter] for linter in self.args.only}
|
||||
for linter in self.args.skip:
|
||||
del self.lint_functions[linter]
|
||||
if self.args.list:
|
||||
print("{}{:<15} {} {}".format(BOLDRED, 'Linter', 'Description', ENDC))
|
||||
for linter, desc in self.lint_descriptions.items():
|
||||
print("{}{:<15} {}{}{}".format(BLUE, linter, GREEN, desc, ENDC))
|
||||
sys.exit()
|
||||
if self.args.list_groups:
|
||||
print("{}{:<15} {} {}".format(BOLDRED, 'Linter Group', 'File types', ENDC))
|
||||
for group, file_types in self.groups.items():
|
||||
print("{}{:<15} {}{}{}".format(BLUE, group, GREEN, ", ".join(file_types), ENDC))
|
||||
sys.exit()
|
||||
self.set_logger()
|
||||
|
||||
failed = run_parallel(self.lint_functions)
|
||||
sys.exit(1 if failed else 0)
|
||||
|
|
@ -1,243 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
|
||||
import re
|
||||
import traceback
|
||||
|
||||
from zulint.printer import print_err, colors, GREEN, ENDC, MAGENTA, BLUE, YELLOW
|
||||
|
||||
if False:
|
||||
from typing import Any, Dict, List, Optional, Tuple, Iterable
|
||||
|
||||
Rule = List[Dict[str, Any]]
|
||||
LineTup = Tuple[int, str, str, str]
|
||||
|
||||
|
||||
class RuleList:
|
||||
"""Defines and runs custom linting rules for the specified language."""
|
||||
|
||||
def __init__(self, langs, rules, max_length=None, length_exclude=[], shebang_rules=[],
|
||||
exclude_files_in=None):
|
||||
# type: (List[str], Rule, Optional[int], List[str], Rule, Optional[str]) -> None
|
||||
self.langs = langs
|
||||
self.rules = rules
|
||||
self.max_length = max_length
|
||||
self.length_exclude = length_exclude
|
||||
self.shebang_rules = shebang_rules
|
||||
# Exclude the files in this folder from rules
|
||||
self.exclude_files_in = "\\"
|
||||
self.verbose = False
|
||||
|
||||
def get_line_info_from_file(self, fn):
|
||||
# type: (str) -> List[LineTup]
|
||||
line_tups = []
|
||||
for i, line in enumerate(open(fn)):
|
||||
line_newline_stripped = line.strip('\n')
|
||||
line_fully_stripped = line_newline_stripped.strip()
|
||||
if line_fully_stripped.endswith(' # nolint'):
|
||||
continue
|
||||
tup = (i, line, line_newline_stripped, line_fully_stripped)
|
||||
line_tups.append(tup)
|
||||
return line_tups
|
||||
|
||||
def get_rules_applying_to_fn(self, fn, rules):
|
||||
# type: (str, Rule) -> Rule
|
||||
rules_to_apply = []
|
||||
for rule in rules:
|
||||
excluded = False
|
||||
for item in rule.get('exclude', set()):
|
||||
if fn.startswith(item):
|
||||
excluded = True
|
||||
break
|
||||
if excluded:
|
||||
continue
|
||||
if rule.get("include_only"):
|
||||
found = False
|
||||
for item in rule.get("include_only", set()):
|
||||
if item in fn:
|
||||
found = True
|
||||
if not found:
|
||||
continue
|
||||
rules_to_apply.append(rule)
|
||||
|
||||
return rules_to_apply
|
||||
|
||||
def check_file_for_pattern(self,
|
||||
fn,
|
||||
line_tups,
|
||||
identifier,
|
||||
color,
|
||||
rule):
|
||||
# type: (str, List[LineTup], str, Optional[Iterable[str]], Dict[str, Any]) -> bool
|
||||
|
||||
'''
|
||||
DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING.
|
||||
|
||||
This function gets called ~40k times, once per file per regex.
|
||||
|
||||
Inside it's doing a regex check for every line in the file, so
|
||||
it's important to do things like pre-compiling regexes.
|
||||
|
||||
DO NOT INLINE THIS FUNCTION.
|
||||
|
||||
We need to see it show up in profiles, and the function call
|
||||
overhead will never be a bottleneck.
|
||||
'''
|
||||
exclude_lines = {
|
||||
line for
|
||||
(exclude_fn, line) in rule.get('exclude_line', set())
|
||||
if exclude_fn == fn
|
||||
}
|
||||
|
||||
pattern = re.compile(rule['pattern'])
|
||||
strip_rule = rule.get('strip') # type: Optional[str]
|
||||
|
||||
ok = True
|
||||
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
|
||||
if line_fully_stripped in exclude_lines:
|
||||
exclude_lines.remove(line_fully_stripped)
|
||||
continue
|
||||
try:
|
||||
line_to_check = line_fully_stripped
|
||||
if strip_rule is not None:
|
||||
if strip_rule == '\n':
|
||||
line_to_check = line_newline_stripped
|
||||
else:
|
||||
raise Exception("Invalid strip rule")
|
||||
if pattern.search(line_to_check):
|
||||
if rule.get("exclude_pattern"):
|
||||
if re.search(rule['exclude_pattern'], line_to_check):
|
||||
continue
|
||||
self.print_error(rule, line, identifier, color, fn, i+1)
|
||||
ok = False
|
||||
except Exception:
|
||||
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
|
||||
traceback.print_exc()
|
||||
|
||||
if exclude_lines:
|
||||
print('Please remove exclusions for file %s: %s' % (fn, exclude_lines))
|
||||
|
||||
return ok
|
||||
|
||||
def print_error(self, rule, line, identifier, color, fn, line_number):
|
||||
# type: (Dict[str, Any], str, str, Optional[Iterable[str]], str, int) -> None
|
||||
print_err(identifier, color, '{} {}at {} line {}:'.format(
|
||||
YELLOW + rule['description'], BLUE, fn, line_number))
|
||||
print_err(identifier, color, line)
|
||||
if self.verbose:
|
||||
if rule.get('good_lines'):
|
||||
print_err(identifier, color, GREEN + " Good code: {}{}".format(
|
||||
(YELLOW + " | " + GREEN).join(rule['good_lines']), ENDC))
|
||||
if rule.get('bad_lines'):
|
||||
print_err(identifier, color, MAGENTA + " Bad code: {}{}".format(
|
||||
(YELLOW + " | " + MAGENTA).join(rule['bad_lines']), ENDC))
|
||||
print_err(identifier, color, "")
|
||||
|
||||
def check_file_for_long_lines(self,
|
||||
fn,
|
||||
max_length,
|
||||
line_tups):
|
||||
# type: (str, int, List[LineTup]) -> bool
|
||||
ok = True
|
||||
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
|
||||
if isinstance(line, bytes):
|
||||
line_length = len(line.decode("utf-8"))
|
||||
else:
|
||||
line_length = len(line)
|
||||
if (line_length > max_length and
|
||||
'# type' not in line and 'test' not in fn and 'example' not in fn and
|
||||
# Don't throw errors for markdown format URLs
|
||||
not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and
|
||||
# Don't throw errors for URLs in code comments
|
||||
not re.search(r"[#].*http.*", line) and
|
||||
not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and
|
||||
"# ignorelongline" not in line and 'migrations' not in fn):
|
||||
print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped))
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
def custom_check_file(self,
|
||||
fn,
|
||||
identifier,
|
||||
color,
|
||||
max_length=None):
|
||||
# type: (str, str, Optional[Iterable[str]], Optional[int]) -> bool
|
||||
failed = False
|
||||
|
||||
line_tups = self.get_line_info_from_file(fn=fn)
|
||||
|
||||
rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.rules)
|
||||
|
||||
for rule in rules_to_apply:
|
||||
ok = self.check_file_for_pattern(
|
||||
fn=fn,
|
||||
line_tups=line_tups,
|
||||
identifier=identifier,
|
||||
color=color,
|
||||
rule=rule,
|
||||
)
|
||||
if not ok:
|
||||
failed = True
|
||||
|
||||
# TODO: Move the below into more of a framework.
|
||||
firstline = None
|
||||
lastLine = None
|
||||
if line_tups:
|
||||
# line_fully_stripped for the first line.
|
||||
firstline = line_tups[0][3]
|
||||
lastLine = line_tups[-1][1]
|
||||
|
||||
if max_length is not None:
|
||||
ok = self.check_file_for_long_lines(
|
||||
fn=fn,
|
||||
max_length=max_length,
|
||||
line_tups=line_tups,
|
||||
)
|
||||
if not ok:
|
||||
failed = True
|
||||
|
||||
if firstline:
|
||||
shebang_rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.shebang_rules)
|
||||
for rule in shebang_rules_to_apply:
|
||||
if re.search(rule['pattern'], firstline):
|
||||
self.print_error(rule, firstline, identifier, color, fn, 1)
|
||||
failed = True
|
||||
|
||||
if lastLine and ('\n' not in lastLine):
|
||||
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
|
||||
failed = True
|
||||
|
||||
return failed
|
||||
|
||||
def check(self, by_lang, verbose=False):
|
||||
# type: (Dict[str, List[str]], bool) -> bool
|
||||
# By default, a rule applies to all files within the extension for
|
||||
# which it is specified (e.g. all .py files)
|
||||
# There are three operators we can use to manually include or exclude files from linting for a rule:
|
||||
# 'exclude': 'set([<path>, ...])' - if <path> is a filename, excludes that file.
|
||||
# if <path> is a directory, excludes all files
|
||||
# directly below the directory <path>.
|
||||
# 'exclude_line': 'set([(<path>, <line>), ...])' - excludes all lines matching <line>
|
||||
# in the file <path> from linting.
|
||||
# 'include_only': 'set([<path>, ...])' - includes only those files where <path> is a
|
||||
# substring of the filepath.
|
||||
failed = False
|
||||
self.verbose = verbose
|
||||
for lang in self.langs:
|
||||
color = next(colors)
|
||||
for fn in by_lang[lang]:
|
||||
if fn.startswith(self.exclude_files_in) or ('custom_check.py' in fn):
|
||||
# This is a bit of a hack, but it generally really doesn't
|
||||
# work to check the file that defines all the things to check for.
|
||||
#
|
||||
# TODO: Migrate this to looking at __module__ type attributes.
|
||||
continue
|
||||
max_length = None
|
||||
if fn not in self.length_exclude:
|
||||
max_length = self.max_length
|
||||
if self.custom_check_file(fn, lang, color, max_length=max_length):
|
||||
failed = True
|
||||
|
||||
return failed
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
if False:
|
||||
# See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts
|
||||
from typing import List, Tuple
|
||||
|
||||
from zulint.printer import print_err, colors
|
||||
|
||||
|
||||
def run_pycodestyle(files, ignored_rules):
|
||||
# type: (List[str], List[str]) -> bool
|
||||
if len(files) == 0:
|
||||
return False
|
||||
|
||||
failed = False
|
||||
color = next(colors)
|
||||
pep8 = subprocess.Popen(
|
||||
['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
assert pep8.stdout is not None # Implied by use of subprocess.PIPE
|
||||
for line in iter(pep8.stdout.readline, b''):
|
||||
print_err('pep8', color, line)
|
||||
failed = True
|
||||
return failed
|
||||
|
||||
|
||||
def run_pyflakes(files, options, suppress_patterns=[]):
|
||||
# type: (List[str], argparse.Namespace, List[Tuple[str, str]]) -> bool
|
||||
if len(files) == 0:
|
||||
return False
|
||||
failed = False
|
||||
color = next(colors)
|
||||
pyflakes = subprocess.Popen(['pyflakes'] + files,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE
|
||||
|
||||
def suppress_line(line: str) -> bool:
|
||||
for file_pattern, line_pattern in suppress_patterns:
|
||||
if file_pattern in line and line_pattern in line:
|
||||
return True
|
||||
return False
|
||||
|
||||
for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines():
|
||||
if not suppress_line(ln):
|
||||
print_err('pyflakes', color, ln)
|
||||
failed = True
|
||||
return failed
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import re
|
||||
from collections import defaultdict
|
||||
import argparse
|
||||
from six.moves import filter
|
||||
|
||||
if False:
|
||||
# See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts
|
||||
from typing import Union, List, Dict
|
||||
|
||||
def get_ftype(fpath, use_shebang):
|
||||
# type: (str, bool) -> str
|
||||
ext = os.path.splitext(fpath)[1]
|
||||
if ext:
|
||||
return ext[1:]
|
||||
elif use_shebang:
|
||||
# opening a file may throw an OSError
|
||||
with open(fpath) as f:
|
||||
first_line = f.readline()
|
||||
if re.search(r'^#!.*\bpython', first_line):
|
||||
return 'py'
|
||||
elif re.search(r'^#!.*sh', first_line):
|
||||
return 'sh'
|
||||
elif re.search(r'^#!.*\bperl', first_line):
|
||||
return 'pl'
|
||||
elif re.search(r'^#!.*\bnode', first_line):
|
||||
return 'js'
|
||||
elif re.search(r'^#!.*\bruby', first_line):
|
||||
return 'rb'
|
||||
elif re.search(r'^#!.*\btail', first_line):
|
||||
return '' # do not lint these scripts.
|
||||
elif re.search(r'^#!', first_line):
|
||||
print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr)
|
||||
return ''
|
||||
else:
|
||||
return ''
|
||||
else:
|
||||
return ''
|
||||
|
||||
def list_files(targets=[], ftypes=[], use_shebang=True,
|
||||
modified_only=False, exclude=[], group_by_ftype=False,
|
||||
extless_only=False):
|
||||
# type: (List[str], List[str], bool, bool, List[str], bool, bool) -> Union[Dict[str, List[str]], List[str]]
|
||||
"""
|
||||
List files tracked by git.
|
||||
|
||||
Returns a list of files which are either in targets or in directories in targets.
|
||||
If targets is [], list of all tracked files in current directory is returned.
|
||||
|
||||
Other arguments:
|
||||
ftypes - List of file types on which to filter the search.
|
||||
If ftypes is [], all files are included.
|
||||
use_shebang - Determine file type of extensionless files from their shebang.
|
||||
modified_only - Only include files which have been modified.
|
||||
exclude - List of files or directories to be excluded, relative to repository root.
|
||||
group_by_ftype - If True, returns a dict of lists keyed by file type.
|
||||
If False, returns a flat list of files.
|
||||
extless_only - Only include extensionless files in output.
|
||||
"""
|
||||
ftypes = [x.strip('.') for x in ftypes]
|
||||
ftypes_set = set(ftypes)
|
||||
|
||||
# Really this is all bytes -- it's a file path -- but we get paths in
|
||||
# sys.argv as str, so that battle is already lost. Settle for hoping
|
||||
# everything is UTF-8.
|
||||
repository_root = subprocess.check_output(['git', 'rev-parse',
|
||||
'--show-toplevel']).strip().decode('utf-8')
|
||||
exclude_abspaths = [os.path.abspath(os.path.join(repository_root, fpath)) for fpath in exclude]
|
||||
|
||||
cmdline = ['git', 'ls-files'] + targets
|
||||
if modified_only:
|
||||
cmdline.append('-m')
|
||||
|
||||
files_gen = (x.strip() for x in subprocess.check_output(cmdline, universal_newlines=True).split('\n'))
|
||||
# throw away empty lines and non-files (like symlinks)
|
||||
files = list(filter(os.path.isfile, files_gen))
|
||||
|
||||
result_dict = defaultdict(list) # type: Dict[str, List[str]]
|
||||
result_list = [] # type: List[str]
|
||||
|
||||
for fpath in files:
|
||||
# this will take a long time if exclude is very large
|
||||
ext = os.path.splitext(fpath)[1]
|
||||
if extless_only and ext:
|
||||
continue
|
||||
absfpath = os.path.abspath(fpath)
|
||||
if any(absfpath == expath or absfpath.startswith(os.path.abspath(expath) + os.sep)
|
||||
for expath in exclude_abspaths):
|
||||
continue
|
||||
|
||||
if ftypes or group_by_ftype:
|
||||
try:
|
||||
filetype = get_ftype(fpath, use_shebang)
|
||||
except (OSError, UnicodeDecodeError) as e:
|
||||
etype = e.__class__.__name__
|
||||
print('Error: %s while determining type of file "%s":' % (etype, fpath), file=sys.stderr)
|
||||
print(e, file=sys.stderr)
|
||||
filetype = ''
|
||||
if ftypes and filetype not in ftypes_set:
|
||||
continue
|
||||
|
||||
if group_by_ftype:
|
||||
result_dict[filetype].append(fpath)
|
||||
else:
|
||||
result_list.append(fpath)
|
||||
|
||||
if group_by_ftype:
|
||||
return result_dict
|
||||
else:
|
||||
return result_list
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="List files tracked by git and optionally filter by type")
|
||||
parser.add_argument('targets', nargs='*', default=[],
|
||||
help='''files and directories to include in the result.
|
||||
If this is not specified, the current directory is used''')
|
||||
parser.add_argument('-m', '--modified', action='store_true', default=False,
|
||||
help='list only modified files')
|
||||
parser.add_argument('-f', '--ftypes', nargs='+', default=[],
|
||||
help="list of file types to filter on. "
|
||||
"All files are included if this option is absent")
|
||||
parser.add_argument('--ext-only', dest='extonly', action='store_true', default=False,
|
||||
help='only use extension to determine file type')
|
||||
parser.add_argument('--exclude', nargs='+', default=[],
|
||||
help='list of files and directories to exclude from results, relative to repo root')
|
||||
parser.add_argument('--extless-only', dest='extless_only', action='store_true', default=False,
|
||||
help='only include extensionless files in output')
|
||||
args = parser.parse_args()
|
||||
listing = list_files(targets=args.targets, ftypes=args.ftypes, use_shebang=not args.extonly,
|
||||
modified_only=args.modified, exclude=args.exclude, extless_only=args.extless_only)
|
||||
for l in listing:
|
||||
print(l)
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
from itertools import cycle
|
||||
if False:
|
||||
# See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts
|
||||
from typing import Union, Text
|
||||
|
||||
# Terminal Color codes for use in differentiatng linters
|
||||
BOLDRED = '\x1B[1;31m'
|
||||
GREEN = '\x1b[32m'
|
||||
YELLOW = '\x1b[33m'
|
||||
BLUE = '\x1b[34m'
|
||||
MAGENTA = '\x1b[35m'
|
||||
CYAN = '\x1b[36m'
|
||||
ENDC = '\033[0m'
|
||||
|
||||
colors = cycle([GREEN, YELLOW, BLUE, MAGENTA, CYAN])
|
||||
|
||||
def print_err(name, color, line):
|
||||
# type: (str, str, Union[Text, bytes]) -> None
|
||||
|
||||
# Decode with UTF-8 if in Python 3 and `line` is of bytes type.
|
||||
# (Python 2 does this automatically)
|
||||
if sys.version_info[0] == 3 and isinstance(line, bytes):
|
||||
line = line.decode('utf-8')
|
||||
|
||||
print('{}{}{}|{end} {}{}{end}'.format(
|
||||
color,
|
||||
name,
|
||||
' ' * max(0, 10 - len(name)),
|
||||
BOLDRED,
|
||||
line.rstrip(),
|
||||
end=ENDC)
|
||||
)
|
||||
|
||||
# Python 2's print function does not have a `flush` option.
|
||||
sys.stdout.flush()
|
||||
|
|
@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea
|
|||
# historical commits sharing the same major version, in which case a
|
||||
# minor version bump suffices.
|
||||
|
||||
PROVISION_VERSION = '44.1'
|
||||
PROVISION_VERSION = '45.0'
|
||||
|
|
|
|||
Loading…
Reference in New Issue