From 2183a74040cd2fc1db3bdfb931c26a6ac25eee08 Mon Sep 17 00:00:00 2001 From: Aman Date: Fri, 26 Jul 2019 08:15:18 +0530 Subject: [PATCH] zulint: Use zulint from the extracted repository. zulint will be added as a "third-party" dependency in zulip from now on. See the new project at https://github.com/zulip/zulint for more details. --- requirements/dev.in | 3 + requirements/dev.txt | 1 + tools/linter_lib/custom_check.py | 2 - tools/tests/test_zulint_custom_rules.py | 2 +- tools/zulint/README.md | 104 ---------- tools/zulint/__init__.py | 0 tools/zulint/command.py | 185 ------------------ tools/zulint/custom_rules.py | 243 ------------------------ tools/zulint/linters.py | 51 ----- tools/zulint/lister.py | 138 -------------- tools/zulint/printer.py | 39 ---- version.py | 2 +- 12 files changed, 6 insertions(+), 764 deletions(-) delete mode 100644 tools/zulint/README.md delete mode 100644 tools/zulint/__init__.py delete mode 100644 tools/zulint/command.py delete mode 100644 tools/zulint/custom_rules.py delete mode 100644 tools/zulint/linters.py delete mode 100755 tools/zulint/lister.py delete mode 100644 tools/zulint/printer.py diff --git a/requirements/dev.in b/requirements/dev.in index 208141019c..f07cbee74a 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -52,4 +52,7 @@ python-digitalocean==1.14.0 # Needed for updating the locked pip dependencies pip-tools==2.0.2 +# zulip's linting framework - zulint +-e git+https://github.com/zulip/zulint@master#egg=zulint==0.0.1 + -r mypy.in diff --git a/requirements/dev.txt b/requirements/dev.txt index de1db9bcb2..66e708ab12 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -13,6 +13,7 @@ git+https://github.com/zulip/libthumbor.git@60ed2431c07686a12f2770b2d852c5650f3c git+https://github.com/zulip/line_profiler.git#egg=line_profiler==2.1.2.zulip1 git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1 git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git +git+https://github.com/zulip/zulint@master#egg=zulint==0.0.1 git+https://github.com/zulip/python-zulip-api.git@0.6.1#egg=zulip==0.6.1_git&subdirectory=zulip git+https://github.com/zulip/python-zulip-api.git@0.6.1#egg=zulip_bots==0.6.1+git&subdirectory=zulip_bots alabaster==0.7.12 # via sphinx diff --git a/tools/linter_lib/custom_check.py b/tools/linter_lib/custom_check.py index ca3753c915..13559a858d 100644 --- a/tools/linter_lib/custom_check.py +++ b/tools/linter_lib/custom_check.py @@ -335,7 +335,6 @@ python_rules = RuleList( 'include_only': set([ 'scripts/', 'puppet/', - 'tools/zulint/', ]), 'exclude': set([ # Not important, but should fix @@ -499,7 +498,6 @@ python_rules = RuleList( # We are likely to want to keep these dirs Python 2+3 compatible, # since the plan includes extracting them to a separate project eventually. 'tools/lib', - 'tools/zulint', # TODO: Update our migrations from Text->str. 'zerver/migrations/', # thumbor is (currently) python2 only diff --git a/tools/tests/test_zulint_custom_rules.py b/tools/tests/test_zulint_custom_rules.py index fc3182e7f5..f6b215c7d1 100644 --- a/tools/tests/test_zulint_custom_rules.py +++ b/tools/tests/test_zulint_custom_rules.py @@ -7,7 +7,7 @@ from zulint.custom_rules import RuleList from linter_lib.custom_check import python_rules, non_py_rules ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..')) -CHECK_MESSAGE = "Fix the corresponding rule in `tools/zulint/custom_rules.py`." +CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`." class TestRuleList(TestCase): diff --git a/tools/zulint/README.md b/tools/zulint/README.md deleted file mode 100644 index b2f4d6411d..0000000000 --- a/tools/zulint/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# zulint - -zulint is a lightweight linting framework designed for complex -applications using a mix of third-party linters and custom rules. - -## Why zulint - -Modern full-stack web applications generally involve code written in -several programming languages, each of which have their own standard -linter tools. For example, [Zulip](https://zulipchat.com) uses Python -(mypy/pyflake/pycodestyle), JavaScript (eslint), CSS (stylelint), -puppet (puppet-lint), shell (shellcheck), and several more. For many -codebases, this results in linting being an unpleasantly slow -experience, resulting in even more unpleasant secondary problems like -developers merging code that doesn't pass lint, not enforcing linter -rules, and debates about whether a useful linter is "worth the time". - -Zulint is the linter framework we built for Zulip to create a -reliable, lightning-fast linter experience to solve these problems. -It has the following features: - -- Integrates with `git` to only checks files in source control (not - automatically generated, untracked, or .gitignore files). -- Runs the linters in parallel, so you only have to wait for the - slowest linter. For Zulip, this is a ~4x performance improvement - over running our third-party linters in series. -- Produduces easy-to-read, clear terminal output, with each - independent linter given its own color. -- Can check just modified files, or even as a `pre-commit` hook, only - checking files that have changed (and only starting linters which - check files that have changed). -- Handles all the annoying details of flushing stdout and managing - color codes. -- Highly configurable. - - Integrate a third-party linter with just a couple lines of code. - - Every feature supports convenient include/exclude rules. - - Add custom lint rules with a powerful regular expression - framework. E.g. in Zulip, we want all access to `Message` objects - in views code to be done via our `access_message_by_id` functions - (which do security checks to ensure the user the request is being - done on behalf of has access to the message), and that is enforced - in part by custom regular expression lint rules. This system is - optimized Python: Zulip has a few hundred custom linter rules of - this type. - - Easily add custom options to check subsets of your codebase, - subsets of rules, etc. -- Has a nice automated testing framework for custom lint rules, so you - can make sure your rules actually work. - -This codebase has been in production use in Zulip for several years, -but only in 2019 was generalized for use by other projects. Its API -to be beta and may change (with notice in the release notes) if we -discover a better API, and patches to further extend it for more use -cases are encouraged. - -## Using zulint - -Once a project is setup with zulint, you'll have a top-level linter -script with at least the following options: - -``` -(zulip-py3-venv) tabbott@coset:~/zulip$ ./tools/lint --help -usage: lint [-h] [--force] [--full] [--modified] [--verbose-timing] - [--skip SKIP] [--only ONLY] [--list] [--groups GROUPS] - [targets [targets ...]] - -positional arguments: - targets Specify directories to check - -optional arguments: - -h, --help show this help message and exit - --force Run tests despite possible problems. - --modified, -m Only check modified files - --verbose-timing, -vt - Print verbose timing output - --skip SKIP Specify linters to skip, eg: --skip=mypy,gitlint - --only ONLY Specify linters to run, eg: --only=mypy,gitlint - --list, -l List all the registered linters - --groups GROUPS, -g GROUPS - Only run linter for languages in the group(s), e.g.: - --groups=backend,other_group -``` - -### pre-commit hook mode - -See https://github.com/zulip/zulip/blob/master/tools/pre-commit for an -example pre-commit hook (Zulip's has some extra complexity because we -use Vagrant from our development environment, and want to be able to -run the hook from outside Vagrant). - -## Adding zulint to a codebase - -TODO. Will roughly include `pip install zulint`, copying an example -`lint` script, and adding your rules. - - -## Adding third-party linters - -TODO: Document the linter_config API. - -## Writing custom rules - -TODO: Document all the features of the `RuleList` and `custom_check` system. - diff --git a/tools/zulint/__init__.py b/tools/zulint/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/zulint/command.py b/tools/zulint/command.py deleted file mode 100644 index c2da447a67..0000000000 --- a/tools/zulint/command.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import print_function -from __future__ import absolute_import - -import argparse -import logging -import os -import subprocess -import sys - -if False: - # See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts - from typing import Callable, Dict, List, Optional - -from zulint.printer import print_err, colors, BOLDRED, BLUE, GREEN, ENDC -from zulint import lister - -def add_default_linter_arguments(parser): - # type: (argparse.ArgumentParser) -> None - parser.add_argument('--modified', '-m', - action='store_true', - help='Only check modified files') - parser.add_argument('--verbose-timing', '-vt', - action='store_true', - help='Print verbose timing output') - parser.add_argument('targets', - nargs='*', - help='Specify directories to check') - parser.add_argument('--skip', - default=[], - type=split_arg_into_list, - help='Specify linters to skip, eg: --skip=mypy,gitlint') - parser.add_argument('--only', - default=[], - type=split_arg_into_list, - help='Specify linters to run, eg: --only=mypy,gitlint') - parser.add_argument('--list', '-l', - action='store_true', - help='List all the registered linters') - parser.add_argument('--list-groups', '-lg', - action='store_true', - help='List all the registered linter groups') - parser.add_argument('--groups', '-g', - default=[], - type=split_arg_into_list, - help='Only run linter for languages in the group(s), e.g.: ' - '--groups=backend,frontend') - parser.add_argument('--verbose', '-v', - action='store_true', - help='Print verbose output where available') - parser.add_argument('--fix', - action='store_true', - help='Automatically fix problems where supported') - -def split_arg_into_list(arg): - # type: (str) -> List[str] - return [linter for linter in arg.split(',')] - -def run_parallel(lint_functions): - # type: (Dict[str, Callable[[], int]]) -> bool - pids = [] - for name, func in lint_functions.items(): - pid = os.fork() - if pid == 0: - logging.info("start " + name) - result = func() - logging.info("finish " + name) - sys.stdout.flush() - sys.stderr.flush() - os._exit(result) - pids.append(pid) - failed = False - - for pid in pids: - (_, status) = os.waitpid(pid, 0) - if status != 0: - failed = True - return failed - -class LinterConfig: - lint_functions = {} # type: Dict[str, Callable[[], int]] - lint_descriptions = {} # type: Dict[str, str] - - def __init__(self, args): - # type: (argparse.Namespace) -> None - self.args = args - self.by_lang = {} # type: Dict[str, List[str]] - self.groups = {} # type: Dict[str, List[str]] - - def list_files(self, file_types=[], groups={}, use_shebang=True, group_by_ftype=True, exclude=[]): - # type: (List[str], Dict[str, List[str]], bool, bool, List[str]) -> Dict[str, List[str]] - assert file_types or groups, "Atleast one of `file_types` or `groups` must be specified." - - self.groups = groups - if self.args.groups: - file_types = [ft for group in self.args.groups for ft in groups[group]] - else: - file_types.extend({ft for group in groups.values() for ft in group}) - - self.by_lang = lister.list_files(self.args.targets, modified_only=self.args.modified, - ftypes=file_types, use_shebang=use_shebang, - group_by_ftype=group_by_ftype, exclude=exclude) - return self.by_lang - - def lint(self, func): - # type: (Callable[[], int]) -> Callable[[], int] - self.lint_functions[func.__name__] = func - self.lint_descriptions[func.__name__] = func.__doc__ if func.__doc__ else "External Linter" - return func - - def external_linter(self, name, command, target_langs=[], pass_targets=True, fix_arg=None, - description="External Linter"): - # type: (str, List[str], List[str], bool, Optional[str], str) -> None - """Registers an external linter program to be run as part of the - linter. This program will be passed the subset of files being - linted that have extensions in target_langs. If there are no - such files, exits without doing anything. - - If target_langs is empty, just runs the linter unconditionally. - """ - self.lint_descriptions[name] = description - color = next(colors) - - def run_linter(): - # type: () -> int - targets = [] # type: List[str] - if len(target_langs) != 0: - targets = [target for lang in target_langs for target in self.by_lang[lang]] - if len(targets) == 0: - # If this linter has a list of languages, and - # no files in those languages are to be checked, - # then we can safely return success without - # invoking the external linter. - return 0 - - if self.args.fix and fix_arg: - command.append(fix_arg) - - if pass_targets: - full_command = command + targets - else: - full_command = command - p = subprocess.Popen(full_command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - assert p.stdout # use of subprocess.PIPE indicates non-None - for line in iter(p.stdout.readline, b''): - print_err(name, color, line) - - return p.wait() # Linter exit code - - self.lint_functions[name] = run_linter - - def set_logger(self): - # type: () -> None - logging.basicConfig(format="%(asctime)s %(message)s") - logger = logging.getLogger() - if self.args.verbose_timing: - logger.setLevel(logging.INFO) - else: - logger.setLevel(logging.WARNING) - - def do_lint(self): - # type: () -> None - assert not self.args.only or not self.args.skip, "Only one of --only or --skip can be used at once." - if self.args.only: - self.lint_functions = {linter: self.lint_functions[linter] for linter in self.args.only} - for linter in self.args.skip: - del self.lint_functions[linter] - if self.args.list: - print("{}{:<15} {} {}".format(BOLDRED, 'Linter', 'Description', ENDC)) - for linter, desc in self.lint_descriptions.items(): - print("{}{:<15} {}{}{}".format(BLUE, linter, GREEN, desc, ENDC)) - sys.exit() - if self.args.list_groups: - print("{}{:<15} {} {}".format(BOLDRED, 'Linter Group', 'File types', ENDC)) - for group, file_types in self.groups.items(): - print("{}{:<15} {}{}{}".format(BLUE, group, GREEN, ", ".join(file_types), ENDC)) - sys.exit() - self.set_logger() - - failed = run_parallel(self.lint_functions) - sys.exit(1 if failed else 0) diff --git a/tools/zulint/custom_rules.py b/tools/zulint/custom_rules.py deleted file mode 100644 index 32e161f030..0000000000 --- a/tools/zulint/custom_rules.py +++ /dev/null @@ -1,243 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import print_function -from __future__ import absolute_import - -import re -import traceback - -from zulint.printer import print_err, colors, GREEN, ENDC, MAGENTA, BLUE, YELLOW - -if False: - from typing import Any, Dict, List, Optional, Tuple, Iterable - - Rule = List[Dict[str, Any]] - LineTup = Tuple[int, str, str, str] - - -class RuleList: - """Defines and runs custom linting rules for the specified language.""" - - def __init__(self, langs, rules, max_length=None, length_exclude=[], shebang_rules=[], - exclude_files_in=None): - # type: (List[str], Rule, Optional[int], List[str], Rule, Optional[str]) -> None - self.langs = langs - self.rules = rules - self.max_length = max_length - self.length_exclude = length_exclude - self.shebang_rules = shebang_rules - # Exclude the files in this folder from rules - self.exclude_files_in = "\\" - self.verbose = False - - def get_line_info_from_file(self, fn): - # type: (str) -> List[LineTup] - line_tups = [] - for i, line in enumerate(open(fn)): - line_newline_stripped = line.strip('\n') - line_fully_stripped = line_newline_stripped.strip() - if line_fully_stripped.endswith(' # nolint'): - continue - tup = (i, line, line_newline_stripped, line_fully_stripped) - line_tups.append(tup) - return line_tups - - def get_rules_applying_to_fn(self, fn, rules): - # type: (str, Rule) -> Rule - rules_to_apply = [] - for rule in rules: - excluded = False - for item in rule.get('exclude', set()): - if fn.startswith(item): - excluded = True - break - if excluded: - continue - if rule.get("include_only"): - found = False - for item in rule.get("include_only", set()): - if item in fn: - found = True - if not found: - continue - rules_to_apply.append(rule) - - return rules_to_apply - - def check_file_for_pattern(self, - fn, - line_tups, - identifier, - color, - rule): - # type: (str, List[LineTup], str, Optional[Iterable[str]], Dict[str, Any]) -> bool - - ''' - DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING. - - This function gets called ~40k times, once per file per regex. - - Inside it's doing a regex check for every line in the file, so - it's important to do things like pre-compiling regexes. - - DO NOT INLINE THIS FUNCTION. - - We need to see it show up in profiles, and the function call - overhead will never be a bottleneck. - ''' - exclude_lines = { - line for - (exclude_fn, line) in rule.get('exclude_line', set()) - if exclude_fn == fn - } - - pattern = re.compile(rule['pattern']) - strip_rule = rule.get('strip') # type: Optional[str] - - ok = True - for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: - if line_fully_stripped in exclude_lines: - exclude_lines.remove(line_fully_stripped) - continue - try: - line_to_check = line_fully_stripped - if strip_rule is not None: - if strip_rule == '\n': - line_to_check = line_newline_stripped - else: - raise Exception("Invalid strip rule") - if pattern.search(line_to_check): - if rule.get("exclude_pattern"): - if re.search(rule['exclude_pattern'], line_to_check): - continue - self.print_error(rule, line, identifier, color, fn, i+1) - ok = False - except Exception: - print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1)) - traceback.print_exc() - - if exclude_lines: - print('Please remove exclusions for file %s: %s' % (fn, exclude_lines)) - - return ok - - def print_error(self, rule, line, identifier, color, fn, line_number): - # type: (Dict[str, Any], str, str, Optional[Iterable[str]], str, int) -> None - print_err(identifier, color, '{} {}at {} line {}:'.format( - YELLOW + rule['description'], BLUE, fn, line_number)) - print_err(identifier, color, line) - if self.verbose: - if rule.get('good_lines'): - print_err(identifier, color, GREEN + " Good code: {}{}".format( - (YELLOW + " | " + GREEN).join(rule['good_lines']), ENDC)) - if rule.get('bad_lines'): - print_err(identifier, color, MAGENTA + " Bad code: {}{}".format( - (YELLOW + " | " + MAGENTA).join(rule['bad_lines']), ENDC)) - print_err(identifier, color, "") - - def check_file_for_long_lines(self, - fn, - max_length, - line_tups): - # type: (str, int, List[LineTup]) -> bool - ok = True - for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: - if isinstance(line, bytes): - line_length = len(line.decode("utf-8")) - else: - line_length = len(line) - if (line_length > max_length and - '# type' not in line and 'test' not in fn and 'example' not in fn and - # Don't throw errors for markdown format URLs - not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and - # Don't throw errors for URLs in code comments - not re.search(r"[#].*http.*", line) and - not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and - "# ignorelongline" not in line and 'migrations' not in fn): - print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped)) - ok = False - return ok - - def custom_check_file(self, - fn, - identifier, - color, - max_length=None): - # type: (str, str, Optional[Iterable[str]], Optional[int]) -> bool - failed = False - - line_tups = self.get_line_info_from_file(fn=fn) - - rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.rules) - - for rule in rules_to_apply: - ok = self.check_file_for_pattern( - fn=fn, - line_tups=line_tups, - identifier=identifier, - color=color, - rule=rule, - ) - if not ok: - failed = True - - # TODO: Move the below into more of a framework. - firstline = None - lastLine = None - if line_tups: - # line_fully_stripped for the first line. - firstline = line_tups[0][3] - lastLine = line_tups[-1][1] - - if max_length is not None: - ok = self.check_file_for_long_lines( - fn=fn, - max_length=max_length, - line_tups=line_tups, - ) - if not ok: - failed = True - - if firstline: - shebang_rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.shebang_rules) - for rule in shebang_rules_to_apply: - if re.search(rule['pattern'], firstline): - self.print_error(rule, firstline, identifier, color, fn, 1) - failed = True - - if lastLine and ('\n' not in lastLine): - print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,)) - failed = True - - return failed - - def check(self, by_lang, verbose=False): - # type: (Dict[str, List[str]], bool) -> bool - # By default, a rule applies to all files within the extension for - # which it is specified (e.g. all .py files) - # There are three operators we can use to manually include or exclude files from linting for a rule: - # 'exclude': 'set([, ...])' - if is a filename, excludes that file. - # if is a directory, excludes all files - # directly below the directory . - # 'exclude_line': 'set([(, ), ...])' - excludes all lines matching - # in the file from linting. - # 'include_only': 'set([, ...])' - includes only those files where is a - # substring of the filepath. - failed = False - self.verbose = verbose - for lang in self.langs: - color = next(colors) - for fn in by_lang[lang]: - if fn.startswith(self.exclude_files_in) or ('custom_check.py' in fn): - # This is a bit of a hack, but it generally really doesn't - # work to check the file that defines all the things to check for. - # - # TODO: Migrate this to looking at __module__ type attributes. - continue - max_length = None - if fn not in self.length_exclude: - max_length = self.max_length - if self.custom_check_file(fn, lang, color, max_length=max_length): - failed = True - - return failed diff --git a/tools/zulint/linters.py b/tools/zulint/linters.py deleted file mode 100644 index f3c5dc9122..0000000000 --- a/tools/zulint/linters.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import - -import argparse -import subprocess -if False: - # See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts - from typing import List, Tuple - -from zulint.printer import print_err, colors - - -def run_pycodestyle(files, ignored_rules): - # type: (List[str], List[str]) -> bool - if len(files) == 0: - return False - - failed = False - color = next(colors) - pep8 = subprocess.Popen( - ['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - assert pep8.stdout is not None # Implied by use of subprocess.PIPE - for line in iter(pep8.stdout.readline, b''): - print_err('pep8', color, line) - failed = True - return failed - - -def run_pyflakes(files, options, suppress_patterns=[]): - # type: (List[str], argparse.Namespace, List[Tuple[str, str]]) -> bool - if len(files) == 0: - return False - failed = False - color = next(colors) - pyflakes = subprocess.Popen(['pyflakes'] + files, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE - - def suppress_line(line: str) -> bool: - for file_pattern, line_pattern in suppress_patterns: - if file_pattern in line and line_pattern in line: - return True - return False - - for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines(): - if not suppress_line(ln): - print_err('pyflakes', color, ln) - failed = True - return failed diff --git a/tools/zulint/lister.py b/tools/zulint/lister.py deleted file mode 100755 index 95f7ae7568..0000000000 --- a/tools/zulint/lister.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import print_function -from __future__ import absolute_import - -import os -import sys -import subprocess -import re -from collections import defaultdict -import argparse -from six.moves import filter - -if False: - # See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts - from typing import Union, List, Dict - -def get_ftype(fpath, use_shebang): - # type: (str, bool) -> str - ext = os.path.splitext(fpath)[1] - if ext: - return ext[1:] - elif use_shebang: - # opening a file may throw an OSError - with open(fpath) as f: - first_line = f.readline() - if re.search(r'^#!.*\bpython', first_line): - return 'py' - elif re.search(r'^#!.*sh', first_line): - return 'sh' - elif re.search(r'^#!.*\bperl', first_line): - return 'pl' - elif re.search(r'^#!.*\bnode', first_line): - return 'js' - elif re.search(r'^#!.*\bruby', first_line): - return 'rb' - elif re.search(r'^#!.*\btail', first_line): - return '' # do not lint these scripts. - elif re.search(r'^#!', first_line): - print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr) - return '' - else: - return '' - else: - return '' - -def list_files(targets=[], ftypes=[], use_shebang=True, - modified_only=False, exclude=[], group_by_ftype=False, - extless_only=False): - # type: (List[str], List[str], bool, bool, List[str], bool, bool) -> Union[Dict[str, List[str]], List[str]] - """ - List files tracked by git. - - Returns a list of files which are either in targets or in directories in targets. - If targets is [], list of all tracked files in current directory is returned. - - Other arguments: - ftypes - List of file types on which to filter the search. - If ftypes is [], all files are included. - use_shebang - Determine file type of extensionless files from their shebang. - modified_only - Only include files which have been modified. - exclude - List of files or directories to be excluded, relative to repository root. - group_by_ftype - If True, returns a dict of lists keyed by file type. - If False, returns a flat list of files. - extless_only - Only include extensionless files in output. - """ - ftypes = [x.strip('.') for x in ftypes] - ftypes_set = set(ftypes) - - # Really this is all bytes -- it's a file path -- but we get paths in - # sys.argv as str, so that battle is already lost. Settle for hoping - # everything is UTF-8. - repository_root = subprocess.check_output(['git', 'rev-parse', - '--show-toplevel']).strip().decode('utf-8') - exclude_abspaths = [os.path.abspath(os.path.join(repository_root, fpath)) for fpath in exclude] - - cmdline = ['git', 'ls-files'] + targets - if modified_only: - cmdline.append('-m') - - files_gen = (x.strip() for x in subprocess.check_output(cmdline, universal_newlines=True).split('\n')) - # throw away empty lines and non-files (like symlinks) - files = list(filter(os.path.isfile, files_gen)) - - result_dict = defaultdict(list) # type: Dict[str, List[str]] - result_list = [] # type: List[str] - - for fpath in files: - # this will take a long time if exclude is very large - ext = os.path.splitext(fpath)[1] - if extless_only and ext: - continue - absfpath = os.path.abspath(fpath) - if any(absfpath == expath or absfpath.startswith(os.path.abspath(expath) + os.sep) - for expath in exclude_abspaths): - continue - - if ftypes or group_by_ftype: - try: - filetype = get_ftype(fpath, use_shebang) - except (OSError, UnicodeDecodeError) as e: - etype = e.__class__.__name__ - print('Error: %s while determining type of file "%s":' % (etype, fpath), file=sys.stderr) - print(e, file=sys.stderr) - filetype = '' - if ftypes and filetype not in ftypes_set: - continue - - if group_by_ftype: - result_dict[filetype].append(fpath) - else: - result_list.append(fpath) - - if group_by_ftype: - return result_dict - else: - return result_list - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="List files tracked by git and optionally filter by type") - parser.add_argument('targets', nargs='*', default=[], - help='''files and directories to include in the result. - If this is not specified, the current directory is used''') - parser.add_argument('-m', '--modified', action='store_true', default=False, - help='list only modified files') - parser.add_argument('-f', '--ftypes', nargs='+', default=[], - help="list of file types to filter on. " - "All files are included if this option is absent") - parser.add_argument('--ext-only', dest='extonly', action='store_true', default=False, - help='only use extension to determine file type') - parser.add_argument('--exclude', nargs='+', default=[], - help='list of files and directories to exclude from results, relative to repo root') - parser.add_argument('--extless-only', dest='extless_only', action='store_true', default=False, - help='only include extensionless files in output') - args = parser.parse_args() - listing = list_files(targets=args.targets, ftypes=args.ftypes, use_shebang=not args.extonly, - modified_only=args.modified, exclude=args.exclude, extless_only=args.extless_only) - for l in listing: - print(l) diff --git a/tools/zulint/printer.py b/tools/zulint/printer.py deleted file mode 100644 index e041e950bf..0000000000 --- a/tools/zulint/printer.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import - -import sys -from itertools import cycle -if False: - # See https://zulip.readthedocs.io/en/latest/testing/mypy.html#mypy-in-production-scripts - from typing import Union, Text - -# Terminal Color codes for use in differentiatng linters -BOLDRED = '\x1B[1;31m' -GREEN = '\x1b[32m' -YELLOW = '\x1b[33m' -BLUE = '\x1b[34m' -MAGENTA = '\x1b[35m' -CYAN = '\x1b[36m' -ENDC = '\033[0m' - -colors = cycle([GREEN, YELLOW, BLUE, MAGENTA, CYAN]) - -def print_err(name, color, line): - # type: (str, str, Union[Text, bytes]) -> None - - # Decode with UTF-8 if in Python 3 and `line` is of bytes type. - # (Python 2 does this automatically) - if sys.version_info[0] == 3 and isinstance(line, bytes): - line = line.decode('utf-8') - - print('{}{}{}|{end} {}{}{end}'.format( - color, - name, - ' ' * max(0, 10 - len(name)), - BOLDRED, - line.rstrip(), - end=ENDC) - ) - - # Python 2's print function does not have a `flush` option. - sys.stdout.flush() diff --git a/version.py b/version.py index 7b72b2bd9c..5c1c1cc04a 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '44.1' +PROVISION_VERSION = '45.0'