From 40a2ffad98ab6cd0cbb6f5a8225bf01f28aba8cc Mon Sep 17 00:00:00 2001 From: Steve Howell Date: Mon, 5 Jun 2017 08:29:04 -0600 Subject: [PATCH] Extract tools/linter_lib/custom_check.py. As part of extracting this, we exempt the library from all custom checks on itself. This is expedient, since a lot of our custom checks are naive about whether things are in strings, and it is also a pain to configure individual rules. --- tools/lint | 457 +------------------------------ tools/linter_lib/__init__.py | 0 tools/linter_lib/custom_check.py | 452 ++++++++++++++++++++++++++++++ 3 files changed, 454 insertions(+), 455 deletions(-) create mode 100644 tools/linter_lib/__init__.py create mode 100644 tools/linter_lib/custom_check.py diff --git a/tools/lint b/tools/lint index 021ca0ca00..61628ca139 100755 --- a/tools/lint +++ b/tools/lint @@ -4,11 +4,9 @@ from __future__ import absolute_import from contextlib import contextmanager import logging import os -import re import sys import optparse import subprocess -import traceback # check for the venv from lib import sanity_check @@ -206,459 +204,6 @@ def run_parallel(lint_functions): failed = True return failed -def build_custom_checkers(by_lang): - # type: (Dict[str, List[str]]) -> Tuple[Callable[[], bool], Callable[[], bool]] - RuleList = List[Dict[str, Any]] - - def custom_check_file(fn, rules, skip_rules=None, max_length=None): - # type: (str, RuleList, Optional[Any], Optional[int]) -> bool - failed = False - lineFlag = False - for i, line in enumerate(open(fn)): - line_newline_stripped = line.strip('\n') - line_fully_stripped = line_newline_stripped.strip() - skip = False - lineFlag = True - for rule in skip_rules or []: - if re.match(rule, line): - skip = True - if line_fully_stripped.endswith(' # nolint'): - continue - if skip: - continue - for rule in rules: - exclude_list = rule.get('exclude', set()) - if fn in exclude_list or os.path.dirname(fn) in exclude_list: - continue - exclude_list = rule.get('exclude_line', set()) - if (fn, line_fully_stripped) in exclude_list: - continue - if rule.get("include_only"): - found = False - for item in rule.get("include_only", set()): - if item in fn: - found = True - if not found: - continue - try: - line_to_check = line_fully_stripped - if rule.get('strip') is not None: - if rule['strip'] == '\n': - line_to_check = line_newline_stripped - else: - raise Exception("Invalid strip rule") - if re.search(rule['pattern'], line_to_check): - sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1)) - print(line) - failed = True - except Exception: - print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1)) - traceback.print_exc() - if isinstance(line, bytes): - line_length = len(line.decode("utf-8")) - else: - line_length = len(line) - if (max_length is not None and line_length > max_length and - '# type' not in line and 'test' not in fn and 'example' not in fn and - not re.match("\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and - "#ignorelongline" not in line and 'migrations' not in fn): - print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped)) - failed = True - lastLine = line - if lineFlag and '\n' not in lastLine: - print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,)) - failed = True - return failed - - whitespace_rules = [ - # This linter should be first since bash_rules depends on it. - {'pattern': '\s+$', - 'strip': '\n', - 'description': 'Fix trailing whitespace'}, - {'pattern': '\t', - 'strip': '\n', - 'exclude': set(['zerver/lib/bugdown/codehilite.py', - 'tools/travis/success-http-headers.txt']), - 'description': 'Fix tab-based whitespace'}, - ] # type: RuleList - markdown_whitespace_rules = list([rule for rule in whitespace_rules if rule['pattern'] != '\s+$']) + [ - # Two spaces trailing a line with other content is okay--it's a markdown line break. - # This rule finds one space trailing a non-space, three or more trailing spaces, and - # spaces on an empty line. - {'pattern': '((?~"]', - 'description': 'Missing whitespace after "="'}, - {'pattern': '^[ ]*//[A-Za-z0-9]', - 'description': 'Missing space after // in comment'}, - {'pattern': 'if[(]', - 'description': 'Missing space between if and ('}, - {'pattern': 'else{$', - 'description': 'Missing space between else and {'}, - {'pattern': '^else {$', - 'description': 'Write JS else statements on same line as }'}, - {'pattern': '^else if', - 'description': 'Write JS else statements on same line as }'}, - {'pattern': 'console[.][a-z]', - 'exclude': set(['static/js/blueslip.js', - 'frontend_tests/zjsunit', - 'frontend_tests/casper_lib/common.js', - 'frontend_tests/node_tests', - 'static/js/debug.js']), - 'description': 'console.log and similar should not be used in webapp'}, - {'pattern': 'i18n[.]t', - 'include_only': set(['static/js/portico']), - 'description': 'i18n.t is not available in portico pages yet'}, - {'pattern': '[.]text\(["\'][a-zA-Z]', - 'description': 'Strings passed to $().text should be wrapped in i18n.t() for internationalization'}, - {'pattern': 'compose_error\(["\']', - 'exclude': set(['tools/lint']), - 'description': 'Argument to compose_error should be a literal string enclosed ' - 'by i18n.t()'}, - {'pattern': 'ui.report_success\(', - 'exclude': set(['tools/lint']), - 'description': 'Deprecated function, use ui_report.success.'}, - {'pattern': 'report.success\(["\']', - 'exclude': set(['tools/lint']), - 'description': 'Argument to report_success should be a literal string enclosed ' - 'by i18n.t()'}, - {'pattern': 'ui.report_error\(', - 'exclude': set(['tools/lint']), - 'description': 'Deprecated function, use ui_report.error.'}, - {'pattern': 'report.error\(["\']', - 'exclude': set(['tools/lint']), - 'description': 'Argument to report_error should be a literal string enclosed ' - 'by i18n.t()'}, - ]) + whitespace_rules - python_rules = cast(RuleList, [ - {'pattern': '^(?!#)@login_required', - 'description': '@login_required is unsupported; use @zulip_login_required'}, - {'pattern': '".*"%\([a-z_].*\)?$', - 'description': 'Missing space around "%"'}, - {'pattern': "'.*'%\([a-z_].*\)?$", - 'exclude': set(['tools/lint', - 'analytics/lib/counts.py', - 'analytics/tests/test_counts.py', - ]), - 'exclude_line': set([ - ('zerver/views/users.py', - "return json_error(_(\"Email '%(email)s' not allowed for realm '%(realm)s'\") %"), - ('zproject/settings.py', - "'format': '%(asctime)s %(levelname)-8s %(message)s'"), - ]), - 'description': 'Missing space around "%"'}, - # This rule is constructed with + to avoid triggering on itself - {'pattern': " =" + '[^ =>~"]', - 'description': 'Missing whitespace after "="'}, - {'pattern': '":\w[^"]*$', - 'description': 'Missing whitespace after ":"'}, - {'pattern': "':\w[^']*$", - 'description': 'Missing whitespace after ":"'}, - {'pattern': "^\s+[#]\w", - 'strip': '\n', - 'description': 'Missing whitespace after "#"'}, - {'pattern': "assertEquals[(]", - 'description': 'Use assertEqual, not assertEquals (which is deprecated).'}, - {'pattern': "== None", - 'exclude': 'tools/lint', - 'description': 'Use `is None` to check whether something is None'}, - {'pattern': "type:[(]", - 'description': 'Missing whitespace after ":" in type annotation'}, - {'pattern': "# type [(]", - 'description': 'Missing : after type in type annotation'}, - {'pattern': "#type", - 'exclude': 'tools/lint', - 'description': 'Missing whitespace after "#" in type annotation'}, - {'pattern': 'if[(]', - 'description': 'Missing space between if and ('}, - {'pattern': ", [)]", - 'description': 'Unnecessary whitespace between "," and ")"'}, - {'pattern': "% [(]", - 'description': 'Unnecessary whitespace between "%" and "("'}, - # This next check could have false positives, but it seems pretty - # rare; if we find any, they can be added to the exclude list for - # this rule. - {'pattern': ' % [a-zA-Z0-9_.]*\)?$', - 'exclude_line': set([ - ('tools/tests/test_template_parser.py', '{% foo'), - ]), - 'description': 'Used % comprehension without a tuple'}, - {'pattern': '.*%s.* % \([a-zA-Z0-9_.]*\)$', - 'description': 'Used % comprehension without a tuple'}, - {'pattern': 'django.utils.translation', - 'include_only': set(['test']), - 'description': 'Test strings should not be tagged for translationx'}, - {'pattern': 'json_success\({}\)', - 'description': 'Use json_success() to return nothing'}, - # To avoid json_error(_variable) and json_error(_(variable)) - {'pattern': '\Wjson_error\(_\(?\w+\)', - 'exclude': set(['tools/lint', 'zerver/tests']), - 'description': 'Argument to json_error should be a literal string enclosed by _()'}, - {'pattern': '\Wjson_error\([\'"].+[),]$', - 'exclude': set(['tools/lint', 'zerver/tests']), - 'exclude_line': set([ - # We don't want this string tagged for translation. - ('zerver/views/compatibility.py', 'return json_error("Client is too old")'), - ]), - 'description': 'Argument to json_error should a literal string enclosed by _()'}, - # To avoid JsonableError(_variable) and JsonableError(_(variable)) - {'pattern': '\WJsonableError\(_\(?\w.+\)', - 'exclude': set(['tools/lint', 'zerver/tests']), - 'description': 'Argument to JsonableError should be a literal string enclosed by _()'}, - {'pattern': '\WJsonableError\(["\'].+\)', - 'exclude': set(['tools/lint', 'zerver/tests']), - 'description': 'Argument to JsonableError should be a literal string enclosed by _()'}, - {'pattern': '([a-zA-Z0-9_]+)=REQ\([\'"]\\1[\'"]', - 'description': 'REQ\'s first argument already defaults to parameter name'}, - {'pattern': 'self\.client\.(get|post|patch|put|delete)', - 'exclude': set(['zilencer/tests.py']), - 'description': \ - '''Do not call self.client directly for put/patch/post/get. - See WRAPPER_COMMENT in test_helpers.py for details. - '''}, - # Directly fetching Message objects in e.g. views code is often a security bug. - {'pattern': '[^r][M]essage.objects.get', - 'exclude': set(["zerver/tests", "zerver/worker/queue_processors.py"]), - 'description': 'Please use access_message() to fetch Message objects', - }, - {'pattern': '[S]tream.objects.get', - 'include_only': set(["zerver/views/"]), - 'description': 'Please use access_stream_by_*() to fetch Stream objects', - }, - {'pattern': 'get_stream[(]', - 'include_only': set(["zerver/views/", "zerver/lib/actions.py"]), - # messages.py needs to support accessing invite-only streams - # that you are no longer subscribed to, so need get_stream. - 'exclude': set(['zerver/views/messages.py']), - 'exclude_line': set([ - # This is a check for whether a stream rename is invalid because it already exists - ('zerver/lib/actions.py', 'get_stream(new_name, stream.realm)'), - # This one in check_message is kinda terrible, since it's - # how most instances are written, but better to exclude something than nothing - ('zerver/lib/actions.py', 'stream = get_stream(stream_name, realm)'), - ('zerver/lib/actions.py', 'get_stream(signups_stream, admin_realm)'), - ]), - 'description': 'Please use access_stream_by_*() to fetch Stream objects', - }, - {'pattern': '[S]tream.objects.filter', - 'include_only': set(["zerver/views/"]), - 'description': 'Please use access_stream_by_*() to fetch Stream objects', - }, - {'pattern': '^from (zerver|analytics|confirmation)', - 'include_only': set(["/migrations/"]), - 'exclude': set(['zerver/migrations/0032_verify_all_medium_avatar_images.py', - 'zerver/migrations/0041_create_attachments_for_old_messages.py', - 'zerver/migrations/0060_move_avatars_to_be_uid_based.py']), - 'description': "Don't import models or other code in migrations; see docs/schema-migrations.md", - }, - {'pattern': 'datetime[.](now|utcnow)', - 'include_only': set(["zerver/", "analytics/"]), - 'description': "Don't use datetime in backend code.\n" - "See https://zulip.readthedocs.io/en/latest/code-style.html#naive-datetime-objects", - }, - {'pattern': 'render_to_response\(', - 'description': "Use render() instead of render_to_response().", - 'exclude': set(['tools/lint']), - }, - # This rule might give false positives in virtualenv setup files which should be excluded, - # and comments which should be rewritten to avoid use of "python2", "python3", etc. - {'pattern': 'python[23]', - 'exclude': set(['tools/lib/provision.py', - 'tools/setup/setup_venvs.py', - 'scripts/lib/setup_venv.py', - 'tools/lint']), - 'description': 'Explicit python invocations should not include a version'} - ]) + whitespace_rules - bash_rules = [ - {'pattern': '#!.*sh [-xe]', - 'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches' - ' to set -x|set -e'}, - ] + whitespace_rules[0:1] # type: RuleList - css_rules = cast(RuleList, [ - {'pattern': '^[^:]*:\S[^:]*;$', - 'description': "Missing whitespace after : in CSS"}, - {'pattern': '[a-z]{', - 'description': "Missing whitespace before '{' in CSS."}, - {'pattern': 'https://', - 'description': "Zulip CSS should have no dependencies on external resources"}, - {'pattern': '^[ ][ ][a-zA-Z0-9]', - 'description': "Incorrect 2-space indentation in CSS", - 'exclude': set(['static/third/thirdparty-fonts.css']), - 'strip': '\n'}, - {'pattern': '{\w', - 'description': "Missing whitespace after '{' in CSS (should be newline)."}, - {'pattern': ' thin[; ]', - 'description': "thin CSS attribute is under-specified, please use 1px."}, - {'pattern': ' medium[; ]', - 'description': "medium CSS attribute is under-specified, please use pixels."}, - {'pattern': ' thick[; ]', - 'description': "thick CSS attribute is under-specified, please use pixels."}, - ]) + whitespace_rules # type: RuleList - prose_style_rules = [ - {'pattern': '[^\/\#\-\"]([jJ]avascript)', # exclude usage in hrefs/divs - 'description': "javascript should be spelled JavaScript"}, - {'pattern': '[^\/\-\.\"\'\_\=\>]([gG]ithub)[^\.\-\_\"\<]', # exclude usage in hrefs/divs - 'description': "github should be spelled GitHub"}, - {'pattern': '[oO]rganisation', # exclude usage in hrefs/divs - 'description': "Organization is spelled with a z"}, - {'pattern': '!!! warning', - 'description': "!!! warning is invalid; it's spelled '!!! warn'"}, - ] # type: RuleList - html_rules = whitespace_rules + prose_style_rules + [ - {'pattern': 'placeholder="[^{]', - 'description': "`placeholder` value should be translatable.", - 'exclude_line': [('templates/zerver/register.html', 'placeholder="acme"'), - ('templates/zerver/register.html', 'placeholder="Acme"'), - ('static/templates/settings/realm-domains-modal.handlebars', - '')], - 'exclude': set(["static/templates/settings/emoji-settings-admin.handlebars", - "static/templates/settings/realm-filter-settings-admin.handlebars", - "static/templates/settings/bot-settings.handlebars"])}, - {'pattern': "placeholder='[^{]", - 'description': "`placeholder` value should be translatable."}, - {'pattern': 'script src="http', - 'description': "Don't directly load dependencies from CDNs. See docs/front-end-build-process.md"}, - {'pattern': "title='[^{]", - 'description': "`title` value should be translatable."}, - {'pattern': 'title="[^{\:]', - 'exclude_line': set([ - ('templates/zerver/markdown_help.html', - ':heart:') - ]), - 'exclude': set(["templates/zerver/emails"]), - 'description': "`title` value should be translatable."}, - {'pattern': '\Walt=["\'][^{"\']', - 'description': "alt argument should be enclosed by _() or it should be an empty string.", - 'exclude': set(['tools/lint', - 'static/templates/settings/display-settings.handlebars', - 'templates/zerver/keyboard_shortcuts.html', - 'templates/zerver/markdown_help.html']), - }, - {'pattern': '\Walt=["\']{{ ?["\']', - 'description': "alt argument should be enclosed by _().", - 'exclude': set(['tools/lint']), - }, - ] # type: RuleList - handlebars_rules = html_rules + [ - {'pattern': "[<]script", - 'description': "Do not use inline