mirror of https://github.com/zulip/zulip.git
lint: Combine functions in custom_rules into RuleList class.
This makes linting rules in zulint more general. Make necessary changes in tools/lint and tools/custom_check.py to run with the new RuleList class. Modify tests for `RuleList` class. Tests only include minor changes to test with the new class.
This commit is contained in:
Aman Agrawal
Tim Abbott
parent
fd25ced43c
commit
db25c0c2ca
10
tools/lint
10
tools/lint
|
|
@ -10,6 +10,7 @@ import argparse
|
||||||
from lib import sanity_check
|
from lib import sanity_check
|
||||||
sanity_check.check_venv(__file__)
|
sanity_check.check_venv(__file__)
|
||||||
|
|
||||||
|
from linter_lib.custom_check import python_rules, non_py_rules
|
||||||
from zulint import lister
|
from zulint import lister
|
||||||
from zulint.command import add_default_linter_arguments, LinterConfig
|
from zulint.command import add_default_linter_arguments, LinterConfig
|
||||||
from typing import cast, Dict, List
|
from typing import cast, Dict, List
|
||||||
|
|
@ -44,7 +45,6 @@ def run():
|
||||||
root_dir = os.path.dirname(tools_dir)
|
root_dir = os.path.dirname(tools_dir)
|
||||||
sys.path.insert(0, root_dir)
|
sys.path.insert(0, root_dir)
|
||||||
|
|
||||||
from tools.linter_lib.custom_check import build_custom_checkers
|
|
||||||
from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
|
from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
|
||||||
from tools.linter_lib.pyflakes import check_pyflakes
|
from tools.linter_lib.pyflakes import check_pyflakes
|
||||||
from tools.linter_lib.pep8 import check_pep8
|
from tools.linter_lib.pep8 import check_pep8
|
||||||
|
|
@ -85,8 +85,6 @@ def run():
|
||||||
else:
|
else:
|
||||||
logger.setLevel(logging.WARNING)
|
logger.setLevel(logging.WARNING)
|
||||||
|
|
||||||
check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)
|
|
||||||
|
|
||||||
linter_config = LinterConfig(by_lang)
|
linter_config = LinterConfig(by_lang)
|
||||||
linter_config.external_linter('add_class', ['tools/find-add-class'], ['js'])
|
linter_config.external_linter('add_class', ['tools/find-add-class'], ['js'])
|
||||||
linter_config.external_linter('css', ['node', 'node_modules/.bin/stylelint'], ['css', 'scss'])
|
linter_config.external_linter('css', ['node', 'node_modules/.bin/stylelint'], ['css', 'scss'])
|
||||||
|
|
@ -111,13 +109,15 @@ def run():
|
||||||
@linter_config.lint
|
@linter_config.lint
|
||||||
def custom_py():
|
def custom_py():
|
||||||
# type: () -> int
|
# type: () -> int
|
||||||
failed = check_custom_checks_py()
|
failed = python_rules.check(by_lang)
|
||||||
return 1 if failed else 0
|
return 1 if failed else 0
|
||||||
|
|
||||||
@linter_config.lint
|
@linter_config.lint
|
||||||
def custom_nonpy():
|
def custom_nonpy():
|
||||||
# type: () -> int
|
# type: () -> int
|
||||||
failed = check_custom_checks_nonpy()
|
failed = False
|
||||||
|
for rule in non_py_rules:
|
||||||
|
failed = failed or rule.check(by_lang)
|
||||||
return 1 if failed else 0
|
return 1 if failed else 0
|
||||||
|
|
||||||
@linter_config.lint
|
@linter_config.lint
|
||||||
|
|
|
||||||
|
|
@ -3,17 +3,31 @@
|
||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
|
|
||||||
from zulint.printer import colors
|
from zulint.custom_rules import RuleList
|
||||||
from zulint.custom_rules import (
|
|
||||||
custom_check_file,
|
|
||||||
)
|
|
||||||
|
|
||||||
from typing import cast, Any, Callable, Dict, List, Tuple
|
from typing import cast, Any, Dict, List, Tuple
|
||||||
|
|
||||||
Rule = Dict[str, Any]
|
Rule = List[Dict[str, Any]]
|
||||||
RuleList = List[Dict[str, Any]]
|
# Rule help:
|
||||||
|
# By default, a rule applies to all files within the extension for which it is specified (e.g. all .py files)
|
||||||
|
# There are three operators we can use to manually include or exclude files from linting for a rule:
|
||||||
|
# 'exclude': 'set([<path>, ...])' - if <path> is a filename, excludes that file.
|
||||||
|
# if <path> is a directory, excludes all files directly below the directory <path>.
|
||||||
|
# 'exclude_line': 'set([(<path>, <line>), ...])' - excludes all lines matching <line> in the file <path> from linting.
|
||||||
|
# 'include_only': 'set([<path>, ...])' - includes only those files where <path> is a substring of the filepath.
|
||||||
LineTup = Tuple[int, str, str, str]
|
LineTup = Tuple[int, str, str, str]
|
||||||
|
|
||||||
|
PYDELIMS = r'''"'()\[\]{}#\\'''
|
||||||
|
PYREG = r"[^{}]".format(PYDELIMS)
|
||||||
|
PYSQ = r'"(?:[^"\\]|\\.)*"'
|
||||||
|
PYDQ = r"'(?:[^'\\]|\\.)*'"
|
||||||
|
PYLEFT = r"[(\[{]"
|
||||||
|
PYRIGHT = r"[)\]}]"
|
||||||
|
PYCODE = PYREG
|
||||||
|
for depth in range(5):
|
||||||
|
PYGROUP = r"""(?:{}|{}|{}{}*{})""".format(PYSQ, PYDQ, PYLEFT, PYCODE, PYRIGHT)
|
||||||
|
PYCODE = r"""(?:{}|{})""".format(PYREG, PYGROUP)
|
||||||
|
|
||||||
FILES_WITH_LEGACY_SUBJECT = {
|
FILES_WITH_LEGACY_SUBJECT = {
|
||||||
# This basically requires a big DB migration:
|
# This basically requires a big DB migration:
|
||||||
'zerver/lib/topic.py',
|
'zerver/lib/topic.py',
|
||||||
|
|
@ -40,63 +54,47 @@ FILES_WITH_LEGACY_SUBJECT = {
|
||||||
'zerver/tests/test_narrow.py',
|
'zerver/tests/test_narrow.py',
|
||||||
}
|
}
|
||||||
|
|
||||||
PYDELIMS = r'''"'()\[\]{}#\\'''
|
trailing_whitespace_rule = {
|
||||||
PYREG = r"[^{}]".format(PYDELIMS)
|
'pattern': r'\s+$',
|
||||||
PYSQ = r'"(?:[^"\\]|\\.)*"'
|
'strip': '\n',
|
||||||
PYDQ = r"'(?:[^'\\]|\\.)*'"
|
'description': 'Fix trailing whitespace'
|
||||||
PYLEFT = r"[(\[{]"
|
}
|
||||||
PYRIGHT = r"[)\]}]"
|
whitespace_rules = [
|
||||||
PYCODE = PYREG
|
# This linter should be first since bash_rules depends on it.
|
||||||
for depth in range(5):
|
trailing_whitespace_rule,
|
||||||
PYGROUP = r"""(?:{}|{}|{}{}*{})""".format(PYSQ, PYDQ, PYLEFT, PYCODE, PYRIGHT)
|
{'pattern': 'http://zulip.readthedocs.io',
|
||||||
PYCODE = r"""(?:{}|{})""".format(PYREG, PYGROUP)
|
'description': 'Use HTTPS when linking to ReadTheDocs',
|
||||||
|
},
|
||||||
|
{'pattern': '\t',
|
||||||
|
'strip': '\n',
|
||||||
|
'exclude': set(['tools/ci/success-http-headers.txt']),
|
||||||
|
'description': 'Fix tab-based whitespace'},
|
||||||
|
] # type: Rule
|
||||||
|
comma_whitespace_rule = [
|
||||||
|
{'pattern': ', {2,}[^#/ ]',
|
||||||
|
'exclude': set(['zerver/tests', 'frontend_tests/node_tests', 'corporate/tests']),
|
||||||
|
'description': "Remove multiple whitespaces after ','",
|
||||||
|
'good_lines': ['foo(1, 2, 3)', 'foo = bar # some inline comment'],
|
||||||
|
'bad_lines': ['foo(1, 2, 3)', 'foo(1, 2, 3)']},
|
||||||
|
] # type: Rule
|
||||||
|
markdown_whitespace_rules = list([rule for rule in whitespace_rules if rule['pattern'] != r'\s+$']) + [
|
||||||
|
# Two spaces trailing a line with other content is okay--it's a markdown line break.
|
||||||
|
# This rule finds one space trailing a non-space, three or more trailing spaces, and
|
||||||
|
# spaces on an empty line.
|
||||||
|
{'pattern': r'((?<!\s)\s$)|(\s\s\s+$)|(^\s+$)',
|
||||||
|
'strip': '\n',
|
||||||
|
'description': 'Fix trailing whitespace'},
|
||||||
|
{'pattern': '^#+[A-Za-z0-9]',
|
||||||
|
'strip': '\n',
|
||||||
|
'description': 'Missing space after # in heading',
|
||||||
|
'good_lines': ['### some heading', '# another heading'],
|
||||||
|
'bad_lines': ['###some heading', '#another heading']},
|
||||||
|
] # type: Rule
|
||||||
|
|
||||||
def build_custom_checkers(by_lang):
|
|
||||||
# type: (Dict[str, List[str]]) -> Tuple[Callable[[], bool], Callable[[], bool]]
|
|
||||||
|
|
||||||
# By default, a rule applies to all files within the extension for which it is specified (e.g. all .py files)
|
js_rules = RuleList(
|
||||||
# There are three operators we can use to manually include or exclude files from linting for a rule:
|
langs=['js'],
|
||||||
# 'exclude': 'set([<path>, ...])' - if <path> is a filename, excludes that file.
|
rules=cast(Rule, [
|
||||||
# if <path> is a directory, excludes all files directly below the directory <path>.
|
|
||||||
# 'exclude_line': 'set([(<path>, <line>), ...])' - excludes all lines matching <line> in the file <path> from linting.
|
|
||||||
# 'include_only': 'set([<path>, ...])' - includes only those files where <path> is a substring of the filepath.
|
|
||||||
trailing_whitespace_rule = {
|
|
||||||
'pattern': r'\s+$',
|
|
||||||
'strip': '\n',
|
|
||||||
'description': 'Fix trailing whitespace'
|
|
||||||
}
|
|
||||||
whitespace_rules = [
|
|
||||||
# This linter should be first since bash_rules depends on it.
|
|
||||||
trailing_whitespace_rule,
|
|
||||||
{'pattern': 'http://zulip.readthedocs.io',
|
|
||||||
'description': 'Use HTTPS when linking to ReadTheDocs',
|
|
||||||
},
|
|
||||||
{'pattern': '\t',
|
|
||||||
'strip': '\n',
|
|
||||||
'exclude': set(['tools/ci/success-http-headers.txt']),
|
|
||||||
'description': 'Fix tab-based whitespace'},
|
|
||||||
] # type: RuleList
|
|
||||||
comma_whitespace_rule = [
|
|
||||||
{'pattern': ', {2,}[^#/ ]',
|
|
||||||
'exclude': set(['zerver/tests', 'frontend_tests/node_tests', 'corporate/tests']),
|
|
||||||
'description': "Remove multiple whitespaces after ','",
|
|
||||||
'good_lines': ['foo(1, 2, 3)', 'foo = bar # some inline comment'],
|
|
||||||
'bad_lines': ['foo(1, 2, 3)', 'foo(1, 2, 3)']},
|
|
||||||
] # type: RuleList
|
|
||||||
markdown_whitespace_rules = list([rule for rule in whitespace_rules if rule['pattern'] != r'\s+$']) + [
|
|
||||||
# Two spaces trailing a line with other content is okay--it's a markdown line break.
|
|
||||||
# This rule finds one space trailing a non-space, three or more trailing spaces, and
|
|
||||||
# spaces on an empty line.
|
|
||||||
{'pattern': r'((?<!\s)\s$)|(\s\s\s+$)|(^\s+$)',
|
|
||||||
'strip': '\n',
|
|
||||||
'description': 'Fix trailing whitespace'},
|
|
||||||
{'pattern': '^#+[A-Za-z0-9]',
|
|
||||||
'strip': '\n',
|
|
||||||
'description': 'Missing space after # in heading',
|
|
||||||
'good_lines': ['### some heading', '# another heading'],
|
|
||||||
'bad_lines': ['###some heading', '#another heading']},
|
|
||||||
] # type: RuleList
|
|
||||||
js_rules = cast(RuleList, [
|
|
||||||
{'pattern': 'subject|SUBJECT',
|
{'pattern': 'subject|SUBJECT',
|
||||||
'exclude': set(['static/js/util.js',
|
'exclude': set(['static/js/util.js',
|
||||||
'frontend_tests/']),
|
'frontend_tests/']),
|
||||||
|
|
@ -202,8 +200,12 @@ def build_custom_checkers(by_lang):
|
||||||
]),
|
]),
|
||||||
'good_lines': ['#my-style {color: blue;}'],
|
'good_lines': ['#my-style {color: blue;}'],
|
||||||
'bad_lines': ['<p style="color: blue;">Foo</p>', 'style = "color: blue;"']},
|
'bad_lines': ['<p style="color: blue;">Foo</p>', 'style = "color: blue;"']},
|
||||||
]) + whitespace_rules + comma_whitespace_rule
|
]) + whitespace_rules + comma_whitespace_rule,
|
||||||
python_rules = cast(RuleList, [
|
)
|
||||||
|
|
||||||
|
python_rules = RuleList(
|
||||||
|
langs=['py'],
|
||||||
|
rules=cast(Rule, [
|
||||||
{'pattern': 'subject|SUBJECT',
|
{'pattern': 'subject|SUBJECT',
|
||||||
'exclude_pattern': 'subject to the|email|outbox',
|
'exclude_pattern': 'subject to the|email|outbox',
|
||||||
'description': 'avoid subject as a var',
|
'description': 'avoid subject as a var',
|
||||||
|
|
@ -501,8 +503,13 @@ def build_custom_checkers(by_lang):
|
||||||
'include_only': set(["/management/commands/"]),
|
'include_only': set(["/management/commands/"]),
|
||||||
'description': 'Raise CommandError to exit with failure in management commands',
|
'description': 'Raise CommandError to exit with failure in management commands',
|
||||||
},
|
},
|
||||||
]) + whitespace_rules + comma_whitespace_rule
|
]) + whitespace_rules + comma_whitespace_rule,
|
||||||
bash_rules = cast(RuleList, [
|
max_length=110,
|
||||||
|
)
|
||||||
|
|
||||||
|
bash_rules = RuleList(
|
||||||
|
langs=['bash'],
|
||||||
|
rules=cast(Rule, [
|
||||||
{'pattern': '#!.*sh [-xe]',
|
{'pattern': '#!.*sh [-xe]',
|
||||||
'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches'
|
'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches'
|
||||||
' to set -x|set -e'},
|
' to set -x|set -e'},
|
||||||
|
|
@ -513,8 +520,12 @@ def build_custom_checkers(by_lang):
|
||||||
'scripts/lib/install',
|
'scripts/lib/install',
|
||||||
'scripts/setup/configure-rabbitmq'
|
'scripts/setup/configure-rabbitmq'
|
||||||
]), },
|
]), },
|
||||||
]) + whitespace_rules[0:1]
|
]) + whitespace_rules[0:1],
|
||||||
css_rules = cast(RuleList, [
|
)
|
||||||
|
|
||||||
|
css_rules = RuleList(
|
||||||
|
langs=['css', 'scss'],
|
||||||
|
rules=cast(Rule, [
|
||||||
{'pattern': r'calc\([^+]+\+[^+]+\)',
|
{'pattern': r'calc\([^+]+\+[^+]+\)',
|
||||||
'description': "Avoid using calc with '+' operator. See #8403 : in CSS.",
|
'description': "Avoid using calc with '+' operator. See #8403 : in CSS.",
|
||||||
'good_lines': ["width: calc(20% - -14px);"],
|
'good_lines': ["width: calc(20% - -14px);"],
|
||||||
|
|
@ -553,146 +564,150 @@ def build_custom_checkers(by_lang):
|
||||||
'good_lines': ["border-width: 5px;"],
|
'good_lines': ["border-width: 5px;"],
|
||||||
'bad_lines': ["border-width: thick;", "border: thick solid black;"]},
|
'bad_lines': ["border-width: thick;", "border: thick solid black;"]},
|
||||||
]) + whitespace_rules + comma_whitespace_rule
|
]) + whitespace_rules + comma_whitespace_rule
|
||||||
prose_style_rules = cast(RuleList, [
|
)
|
||||||
{'pattern': r'[^\/\#\-"]([jJ]avascript)', # exclude usage in hrefs/divs
|
|
||||||
'description': "javascript should be spelled JavaScript"},
|
|
||||||
{'pattern': r'''[^\/\-\."'\_\=\>]([gG]ithub)[^\.\-\_"\<]''', # exclude usage in hrefs/divs
|
|
||||||
'description': "github should be spelled GitHub"},
|
|
||||||
{'pattern': '[oO]rganisation', # exclude usage in hrefs/divs
|
|
||||||
'description': "Organization is spelled with a z",
|
|
||||||
'exclude_line': [('docs/translating/french.md', '* organization - **organisation**')]},
|
|
||||||
{'pattern': '!!! warning',
|
|
||||||
'description': "!!! warning is invalid; it's spelled '!!! warn'"},
|
|
||||||
{'pattern': 'Terms of service',
|
|
||||||
'description': "The S in Terms of Service is capitalized"},
|
|
||||||
{'pattern': '[^-_]botserver(?!rc)|bot server',
|
|
||||||
'description': "Use Botserver instead of botserver or bot server."},
|
|
||||||
]) + comma_whitespace_rule
|
|
||||||
html_rules = whitespace_rules + prose_style_rules + [
|
|
||||||
{'pattern': 'subject|SUBJECT',
|
|
||||||
'exclude': set(['templates/zerver/email.html']),
|
|
||||||
'exclude_pattern': 'email subject',
|
|
||||||
'description': 'avoid subject in templates',
|
|
||||||
'good_lines': ['topic_name'],
|
|
||||||
'bad_lines': ['subject="foo"', ' MAX_SUBJECT_LEN']},
|
|
||||||
{'pattern': r'placeholder="[^{#](?:(?!\.com).)+$',
|
|
||||||
'description': "`placeholder` value should be translatable.",
|
|
||||||
'exclude_line': [('templates/zerver/register.html', 'placeholder="acme"'),
|
|
||||||
('templates/zerver/register.html', 'placeholder="Acme or Aκμή"')],
|
|
||||||
'exclude': set(["templates/analytics/support.html"]),
|
|
||||||
'good_lines': ['<input class="stream-list-filter" type="text" placeholder="{{ _(\'Search streams\') }}" />'],
|
|
||||||
'bad_lines': ['<input placeholder="foo">']},
|
|
||||||
{'pattern': "placeholder='[^{]",
|
|
||||||
'description': "`placeholder` value should be translatable.",
|
|
||||||
'good_lines': ['<input class="stream-list-filter" type="text" placeholder="{{ _(\'Search streams\') }}" />'],
|
|
||||||
'bad_lines': ["<input placeholder='foo'>"]},
|
|
||||||
{'pattern': "aria-label='[^{]",
|
|
||||||
'description': "`aria-label` value should be translatable.",
|
|
||||||
'good_lines': ['<button type="button" class="close close-alert-word-status" aria-label="{{t \'Close\' }}">'],
|
|
||||||
'bad_lines': ["<button aria-label='foo'></button>"]},
|
|
||||||
{'pattern': 'aria-label="[^{]',
|
|
||||||
'description': "`aria-label` value should be translatable.",
|
|
||||||
'good_lines': ['<button type="button" class="close close-alert-word-status" aria-label="{{t \'Close\' }}">'],
|
|
||||||
'bad_lines': ['<button aria-label="foo"></button>']},
|
|
||||||
{'pattern': 'script src="http',
|
|
||||||
'description': "Don't directly load dependencies from CDNs. See docs/subsystems/front-end-build-process.md",
|
|
||||||
'exclude': set(["templates/corporate/billing.html", "templates/zerver/hello.html",
|
|
||||||
"templates/corporate/upgrade.html"]),
|
|
||||||
'good_lines': ["{{ render_bundle('landing-page') }}"],
|
|
||||||
'bad_lines': ['<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>']},
|
|
||||||
{'pattern': "title='[^{]",
|
|
||||||
'description': "`title` value should be translatable.",
|
|
||||||
'good_lines': ['<link rel="author" title="{{ _(\'About these documents\') }}" />'],
|
|
||||||
'bad_lines': ["<p title='foo'></p>"],
|
|
||||||
},
|
|
||||||
{'pattern': r'title="[^{\:]',
|
|
||||||
'exclude_line': set([
|
|
||||||
('templates/zerver/app/markdown_help.html',
|
|
||||||
'<td class="rendered_markdown"><img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/heart.png" title=":heart:" /></td>')
|
|
||||||
]),
|
|
||||||
'exclude': set(["templates/zerver/emails", "templates/analytics/support.html"]),
|
|
||||||
'description': "`title` value should be translatable."},
|
|
||||||
{'pattern': r'''\Walt=["'][^{"']''',
|
|
||||||
'description': "alt argument should be enclosed by _() or it should be an empty string.",
|
|
||||||
'exclude': set(['static/templates/settings/display-settings.handlebars',
|
|
||||||
'templates/zerver/app/keyboard_shortcuts.html',
|
|
||||||
'templates/zerver/app/markdown_help.html']),
|
|
||||||
'good_lines': ['<img src="{{source_url}}" alt="{{ _(name) }}" />', '<img alg="" />'],
|
|
||||||
'bad_lines': ['<img alt="Foo Image" />']},
|
|
||||||
{'pattern': r'''\Walt=["']{{ ?["']''',
|
|
||||||
'description': "alt argument should be enclosed by _().",
|
|
||||||
'good_lines': ['<img src="{{source_url}}" alt="{{ _(name) }}" />'],
|
|
||||||
'bad_lines': ['<img alt="{{ " />']},
|
|
||||||
{'pattern': r'\bon\w+ ?=',
|
|
||||||
'description': "Don't use inline event handlers (onclick=, etc. attributes) in HTML. Instead,"
|
|
||||||
"attach a jQuery event handler ($('#foo').on('click', function () {...})) when "
|
|
||||||
"the DOM is ready (inside a $(function () {...}) block).",
|
|
||||||
'exclude': set(['templates/zerver/dev_login.html', 'templates/corporate/upgrade.html']),
|
|
||||||
'good_lines': ["($('#foo').on('click', function () {}"],
|
|
||||||
'bad_lines': ["<button id='foo' onclick='myFunction()'>Foo</button>", "<input onchange='myFunction()'>"]},
|
|
||||||
{'pattern': 'style ?=',
|
|
||||||
'description': "Avoid using the `style=` attribute; we prefer styling in CSS files",
|
|
||||||
'exclude_pattern': r'.*style ?=["' + "'" + '](display: ?none|background: {{|color: {{|background-color: {{).*',
|
|
||||||
'exclude': set([
|
|
||||||
# KaTeX output uses style attribute
|
|
||||||
'templates/zerver/app/markdown_help.html',
|
|
||||||
# 5xx page doesn't have external CSS
|
|
||||||
'static/html/5xx.html',
|
|
||||||
# Group PMs color is dynamically calculated
|
|
||||||
'static/templates/group_pms.handlebars',
|
|
||||||
|
|
||||||
# exclude_pattern above handles color, but have other issues:
|
prose_style_rules = cast(Rule, [
|
||||||
'static/templates/draft.handlebars',
|
{'pattern': r'[^\/\#\-"]([jJ]avascript)', # exclude usage in hrefs/divs
|
||||||
'static/templates/subscription.handlebars',
|
'description': "javascript should be spelled JavaScript"},
|
||||||
'static/templates/single_message.handlebars',
|
{'pattern': r'''[^\/\-\."'\_\=\>]([gG]ithub)[^\.\-\_"\<]''', # exclude usage in hrefs/divs
|
||||||
|
'description': "github should be spelled GitHub"},
|
||||||
|
{'pattern': '[oO]rganisation', # exclude usage in hrefs/divs
|
||||||
|
'description': "Organization is spelled with a z",
|
||||||
|
'exclude_line': [('docs/translating/french.md', '* organization - **organisation**')]},
|
||||||
|
{'pattern': '!!! warning',
|
||||||
|
'description': "!!! warning is invalid; it's spelled '!!! warn'"},
|
||||||
|
{'pattern': 'Terms of service',
|
||||||
|
'description': "The S in Terms of Service is capitalized"},
|
||||||
|
{'pattern': '[^-_]botserver(?!rc)|bot server',
|
||||||
|
'description': "Use Botserver instead of botserver or bot server."},
|
||||||
|
]) + comma_whitespace_rule
|
||||||
|
html_rules = whitespace_rules + prose_style_rules + cast(Rule, [
|
||||||
|
{'pattern': 'subject|SUBJECT',
|
||||||
|
'exclude': set(['templates/zerver/email.html']),
|
||||||
|
'exclude_pattern': 'email subject',
|
||||||
|
'description': 'avoid subject in templates',
|
||||||
|
'good_lines': ['topic_name'],
|
||||||
|
'bad_lines': ['subject="foo"', ' MAX_SUBJECT_LEN']},
|
||||||
|
{'pattern': r'placeholder="[^{#](?:(?!\.com).)+$',
|
||||||
|
'description': "`placeholder` value should be translatable.",
|
||||||
|
'exclude_line': [('templates/zerver/register.html', 'placeholder="acme"'),
|
||||||
|
('templates/zerver/register.html', 'placeholder="Acme or Aκμή"')],
|
||||||
|
'exclude': set(["templates/analytics/support.html"]),
|
||||||
|
'good_lines': ['<input class="stream-list-filter" type="text" placeholder="{{ _(\'Search streams\') }}" />'],
|
||||||
|
'bad_lines': ['<input placeholder="foo">']},
|
||||||
|
{'pattern': "placeholder='[^{]",
|
||||||
|
'description': "`placeholder` value should be translatable.",
|
||||||
|
'good_lines': ['<input class="stream-list-filter" type="text" placeholder="{{ _(\'Search streams\') }}" />'],
|
||||||
|
'bad_lines': ["<input placeholder='foo'>"]},
|
||||||
|
{'pattern': "aria-label='[^{]",
|
||||||
|
'description': "`aria-label` value should be translatable.",
|
||||||
|
'good_lines': ['<button type="button" class="close close-alert-word-status" aria-label="{{t \'Close\' }}">'],
|
||||||
|
'bad_lines': ["<button aria-label='foo'></button>"]},
|
||||||
|
{'pattern': 'aria-label="[^{]',
|
||||||
|
'description': "`aria-label` value should be translatable.",
|
||||||
|
'good_lines': ['<button type="button" class="close close-alert-word-status" aria-label="{{t \'Close\' }}">'],
|
||||||
|
'bad_lines': ['<button aria-label="foo"></button>']},
|
||||||
|
{'pattern': 'script src="http',
|
||||||
|
'description': "Don't directly load dependencies from CDNs. See docs/subsystems/front-end-build-process.md",
|
||||||
|
'exclude': set(["templates/corporate/billing.html", "templates/zerver/hello.html",
|
||||||
|
"templates/corporate/upgrade.html"]),
|
||||||
|
'good_lines': ["{{ render_bundle('landing-page') }}"],
|
||||||
|
'bad_lines': ['<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>']},
|
||||||
|
{'pattern': "title='[^{]",
|
||||||
|
'description': "`title` value should be translatable.",
|
||||||
|
'good_lines': ['<link rel="author" title="{{ _(\'About these documents\') }}" />'],
|
||||||
|
'bad_lines': ["<p title='foo'></p>"]},
|
||||||
|
{'pattern': r'title="[^{\:]',
|
||||||
|
'exclude_line': set([
|
||||||
|
('templates/zerver/app/markdown_help.html',
|
||||||
|
'<td class="rendered_markdown"><img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/heart.png" title=":heart:" /></td>')
|
||||||
|
]),
|
||||||
|
'exclude': set(["templates/zerver/emails", "templates/analytics/support.html"]),
|
||||||
|
'description': "`title` value should be translatable."},
|
||||||
|
{'pattern': r'''\Walt=["'][^{"']''',
|
||||||
|
'description': "alt argument should be enclosed by _() or it should be an empty string.",
|
||||||
|
'exclude': set(['static/templates/settings/display-settings.handlebars',
|
||||||
|
'templates/zerver/app/keyboard_shortcuts.html',
|
||||||
|
'templates/zerver/app/markdown_help.html']),
|
||||||
|
'good_lines': ['<img src="{{source_url}}" alt="{{ _(name) }}" />', '<img alg="" />'],
|
||||||
|
'bad_lines': ['<img alt="Foo Image" />']},
|
||||||
|
{'pattern': r'''\Walt=["']{{ ?["']''',
|
||||||
|
'description': "alt argument should be enclosed by _().",
|
||||||
|
'good_lines': ['<img src="{{source_url}}" alt="{{ _(name) }}" />'],
|
||||||
|
'bad_lines': ['<img alt="{{ " />']},
|
||||||
|
{'pattern': r'\bon\w+ ?=',
|
||||||
|
'description': "Don't use inline event handlers (onclick=, etc. attributes) in HTML. Instead,"
|
||||||
|
"attach a jQuery event handler ($('#foo').on('click', function () {...})) when "
|
||||||
|
"the DOM is ready (inside a $(function () {...}) block).",
|
||||||
|
'exclude': set(['templates/zerver/dev_login.html', 'templates/corporate/upgrade.html']),
|
||||||
|
'good_lines': ["($('#foo').on('click', function () {}"],
|
||||||
|
'bad_lines': ["<button id='foo' onclick='myFunction()'>Foo</button>", "<input onchange='myFunction()'>"]},
|
||||||
|
{'pattern': 'style ?=',
|
||||||
|
'description': "Avoid using the `style=` attribute; we prefer styling in CSS files",
|
||||||
|
'exclude_pattern': r'.*style ?=["' + "'" + '](display: ?none|background: {{|color: {{|background-color: {{).*',
|
||||||
|
'exclude': set([
|
||||||
|
# KaTeX output uses style attribute
|
||||||
|
'templates/zerver/app/markdown_help.html',
|
||||||
|
# 5xx page doesn't have external CSS
|
||||||
|
'static/html/5xx.html',
|
||||||
|
# Group PMs color is dynamically calculated
|
||||||
|
'static/templates/group_pms.handlebars',
|
||||||
|
|
||||||
# Old-style email templates need to use inline style
|
# exclude_pattern above handles color, but have other issues:
|
||||||
# attributes; it should be possible to clean these up
|
'static/templates/draft.handlebars',
|
||||||
# when we convert these templates to use premailer.
|
'static/templates/subscription.handlebars',
|
||||||
'templates/zerver/emails/email_base_messages.html',
|
'static/templates/single_message.handlebars',
|
||||||
|
|
||||||
# Email log templates; should clean up.
|
# Old-style email templates need to use inline style
|
||||||
'templates/zerver/email.html',
|
# attributes; it should be possible to clean these up
|
||||||
'templates/zerver/email_log.html',
|
# when we convert these templates to use premailer.
|
||||||
|
'templates/zerver/emails/email_base_messages.html',
|
||||||
|
|
||||||
# Probably just needs to be changed to display: none so the exclude works
|
# Email log templates; should clean up.
|
||||||
'templates/zerver/app/navbar.html',
|
'templates/zerver/email.html',
|
||||||
|
'templates/zerver/email_log.html',
|
||||||
|
|
||||||
# Needs the width cleaned up; display: none is fine
|
# Probably just needs to be changed to display: none so the exclude works
|
||||||
'static/templates/settings/account-settings.handlebars',
|
'templates/zerver/app/navbar.html',
|
||||||
|
|
||||||
# background image property is dynamically generated
|
# Needs the width cleaned up; display: none is fine
|
||||||
'static/templates/user_profile_modal.handlebars',
|
'static/templates/settings/account-settings.handlebars',
|
||||||
'static/templates/sidebar_private_message_list.handlebars',
|
|
||||||
|
|
||||||
# Inline styling for an svg; could be moved to CSS files?
|
# background image property is dynamically generated
|
||||||
'templates/zerver/landing_nav.html',
|
'static/templates/user_profile_modal.handlebars',
|
||||||
'templates/zerver/billing_nav.html',
|
'static/templates/sidebar_private_message_list.handlebars',
|
||||||
'templates/zerver/app/home.html',
|
|
||||||
'templates/zerver/features.html',
|
|
||||||
'templates/zerver/portico-header.html',
|
|
||||||
'templates/corporate/billing.html',
|
|
||||||
'templates/corporate/upgrade.html',
|
|
||||||
|
|
||||||
# Miscellaneous violations to be cleaned up
|
# Inline styling for an svg; could be moved to CSS files?
|
||||||
'static/templates/user_info_popover_title.handlebars',
|
'templates/zerver/landing_nav.html',
|
||||||
'static/templates/subscription_invites_warning_modal.handlebars',
|
'templates/zerver/billing_nav.html',
|
||||||
'templates/zerver/reset_confirm.html',
|
'templates/zerver/app/home.html',
|
||||||
'templates/zerver/config_error.html',
|
'templates/zerver/features.html',
|
||||||
'templates/zerver/dev_env_email_access_details.html',
|
'templates/zerver/portico-header.html',
|
||||||
'templates/zerver/confirm_continue_registration.html',
|
'templates/corporate/billing.html',
|
||||||
'templates/zerver/register.html',
|
'templates/corporate/upgrade.html',
|
||||||
'templates/zerver/accounts_send_confirm.html',
|
|
||||||
'templates/zerver/integrations/index.html',
|
# Miscellaneous violations to be cleaned up
|
||||||
'templates/zerver/documentation_main.html',
|
'static/templates/user_info_popover_title.handlebars',
|
||||||
'templates/analytics/realm_summary_table.html',
|
'static/templates/subscription_invites_warning_modal.handlebars',
|
||||||
'templates/corporate/zephyr.html',
|
'templates/zerver/reset_confirm.html',
|
||||||
'templates/corporate/zephyr-mirror.html',
|
'templates/zerver/config_error.html',
|
||||||
]),
|
'templates/zerver/dev_env_email_access_details.html',
|
||||||
'good_lines': ['#my-style {color: blue;}', 'style="display: none"', "style='display: none"],
|
'templates/zerver/confirm_continue_registration.html',
|
||||||
'bad_lines': ['<p style="color: blue;">Foo</p>', 'style = "color: blue;"']},
|
'templates/zerver/register.html',
|
||||||
] # type: RuleList
|
'templates/zerver/accounts_send_confirm.html',
|
||||||
handlebars_rules = html_rules + [
|
'templates/zerver/integrations/index.html',
|
||||||
|
'templates/zerver/documentation_main.html',
|
||||||
|
'templates/analytics/realm_summary_table.html',
|
||||||
|
'templates/corporate/zephyr.html',
|
||||||
|
'templates/corporate/zephyr-mirror.html',
|
||||||
|
]),
|
||||||
|
'good_lines': ['#my-style {color: blue;}', 'style="display: none"', "style='display: none"],
|
||||||
|
'bad_lines': ['<p style="color: blue;">Foo</p>', 'style = "color: blue;"']},
|
||||||
|
])
|
||||||
|
|
||||||
|
handlebars_rules = RuleList(
|
||||||
|
langs=['handlebars'],
|
||||||
|
rules=html_rules + cast(Rule, [
|
||||||
{'pattern': "[<]script",
|
{'pattern': "[<]script",
|
||||||
'description': "Do not use inline <script> tags here; put JavaScript in static/js instead."},
|
'description': "Do not use inline <script> tags here; put JavaScript in static/js instead."},
|
||||||
{'pattern': '{{ t ("|\')',
|
{'pattern': '{{ t ("|\')',
|
||||||
|
|
@ -709,14 +724,22 @@ def build_custom_checkers(by_lang):
|
||||||
'description': 'Translatable strings should not have trailing spaces.'},
|
'description': 'Translatable strings should not have trailing spaces.'},
|
||||||
{'pattern': '{{t "[^"]+ " }}',
|
{'pattern': '{{t "[^"]+ " }}',
|
||||||
'description': 'Translatable strings should not have trailing spaces.'},
|
'description': 'Translatable strings should not have trailing spaces.'},
|
||||||
]
|
]),
|
||||||
jinja2_rules = html_rules + [
|
)
|
||||||
|
|
||||||
|
jinja2_rules = RuleList(
|
||||||
|
langs=['html'],
|
||||||
|
rules=html_rules + cast(Rule, [
|
||||||
{'pattern': r"{% endtrans %}[\.\?!]",
|
{'pattern': r"{% endtrans %}[\.\?!]",
|
||||||
'description': "Period should be part of the translatable string."},
|
'description': "Period should be part of the translatable string."},
|
||||||
{'pattern': r"{{ _(.+) }}[\.\?!]",
|
{'pattern': r"{{ _(.+) }}[\.\?!]",
|
||||||
'description': "Period should be part of the translatable string."},
|
'description': "Period should be part of the translatable string."},
|
||||||
]
|
]),
|
||||||
json_rules = [
|
)
|
||||||
|
|
||||||
|
json_rules = RuleList(
|
||||||
|
langs=['json'],
|
||||||
|
rules=cast(Rule, [
|
||||||
# Here, we don't use `whitespace_rules`, because the tab-based
|
# Here, we don't use `whitespace_rules`, because the tab-based
|
||||||
# whitespace rule flags a lot of third-party JSON fixtures
|
# whitespace rule flags a lot of third-party JSON fixtures
|
||||||
# under zerver/webhooks that we want preserved verbatim. So
|
# under zerver/webhooks that we want preserved verbatim. So
|
||||||
|
|
@ -732,8 +755,32 @@ def build_custom_checkers(by_lang):
|
||||||
{'pattern': r'":["\[\{]',
|
{'pattern': r'":["\[\{]',
|
||||||
'exclude': set(['zerver/webhooks/', 'zerver/tests/fixtures/']),
|
'exclude': set(['zerver/webhooks/', 'zerver/tests/fixtures/']),
|
||||||
'description': 'Require space after : in JSON'},
|
'description': 'Require space after : in JSON'},
|
||||||
] # type: RuleList
|
])
|
||||||
markdown_rules = markdown_whitespace_rules + prose_style_rules + [
|
)
|
||||||
|
|
||||||
|
markdown_docs_length_exclude = {
|
||||||
|
# Has some example Vagrant output that's very long
|
||||||
|
"docs/development/setup-vagrant.md",
|
||||||
|
# Have wide output in code blocks
|
||||||
|
"docs/subsystems/logging.md",
|
||||||
|
"docs/subsystems/migration-renumbering.md",
|
||||||
|
# Have curl commands with JSON that would be messy to wrap
|
||||||
|
"zerver/webhooks/helloworld/doc.md",
|
||||||
|
"zerver/webhooks/trello/doc.md",
|
||||||
|
# Has a very long configuration line
|
||||||
|
"templates/zerver/integrations/perforce.md",
|
||||||
|
# Has some example code that could perhaps be wrapped
|
||||||
|
"templates/zerver/api/incoming-webhooks-walkthrough.md",
|
||||||
|
# This macro has a long indented URL
|
||||||
|
"templates/zerver/help/include/git-webhook-url-with-branches-indented.md",
|
||||||
|
# These two are the same file and have some too-long lines for GitHub badges
|
||||||
|
"README.md",
|
||||||
|
"docs/overview/readme.md",
|
||||||
|
}
|
||||||
|
|
||||||
|
markdown_rules = RuleList(
|
||||||
|
langs=['md'],
|
||||||
|
rules=markdown_whitespace_rules + prose_style_rules + cast(Rule, [
|
||||||
{'pattern': r'\[(?P<url>[^\]]+)\]\((?P=url)\)',
|
{'pattern': r'\[(?P<url>[^\]]+)\]\((?P=url)\)',
|
||||||
'description': 'Linkified markdown URLs should use cleaner <http://example.com> syntax.'},
|
'description': 'Linkified markdown URLs should use cleaner <http://example.com> syntax.'},
|
||||||
{'pattern': 'https://zulip.readthedocs.io/en/latest/[a-zA-Z0-9]',
|
{'pattern': 'https://zulip.readthedocs.io/en/latest/[a-zA-Z0-9]',
|
||||||
|
|
@ -753,108 +800,40 @@ def build_custom_checkers(by_lang):
|
||||||
'include_only': set(['README.md', 'CONTRIBUTING.md']),
|
'include_only': set(['README.md', 'CONTRIBUTING.md']),
|
||||||
'description': "Use absolute links from docs served by GitHub",
|
'description': "Use absolute links from docs served by GitHub",
|
||||||
},
|
},
|
||||||
]
|
]),
|
||||||
help_markdown_rules = markdown_rules + [
|
max_length=120,
|
||||||
|
length_exclude=markdown_docs_length_exclude,
|
||||||
|
exclude_files_in='templates/zerver/help/'
|
||||||
|
)
|
||||||
|
|
||||||
|
help_markdown_rules = RuleList(
|
||||||
|
langs=['md'],
|
||||||
|
rules=markdown_rules.rules + cast(Rule, [
|
||||||
{'pattern': '[a-z][.][A-Z]',
|
{'pattern': '[a-z][.][A-Z]',
|
||||||
'description': "Likely missing space after end of sentence"},
|
'description': "Likely missing space after end of sentence",
|
||||||
|
'include_only': set(['templates/zerver/help/']),
|
||||||
|
},
|
||||||
{'pattern': r'\b[rR]ealm[s]?\b',
|
{'pattern': r'\b[rR]ealm[s]?\b',
|
||||||
|
'include_only': set(['templates/zerver/help/']),
|
||||||
'good_lines': ['Organization', 'deactivate_realm', 'realm_filter'],
|
'good_lines': ['Organization', 'deactivate_realm', 'realm_filter'],
|
||||||
'bad_lines': ['Users are in a realm', 'Realm is the best model'],
|
'bad_lines': ['Users are in a realm', 'Realm is the best model'],
|
||||||
'description': "Realms are referred to as Organizations in user-facing docs."},
|
'description': "Realms are referred to as Organizations in user-facing docs."},
|
||||||
]
|
]),
|
||||||
txt_rules = whitespace_rules
|
length_exclude=markdown_docs_length_exclude,
|
||||||
|
)
|
||||||
|
|
||||||
def check_custom_checks_py():
|
txt_rules = RuleList(
|
||||||
# type: () -> bool
|
langs=['txt', 'text', 'yaml', 'rst'],
|
||||||
failed = False
|
rules=whitespace_rules,
|
||||||
color = next(colors)
|
)
|
||||||
|
non_py_rules = [
|
||||||
for fn in by_lang['py']:
|
handlebars_rules,
|
||||||
if 'custom_check.py' in fn:
|
jinja2_rules,
|
||||||
continue
|
css_rules,
|
||||||
if custom_check_file(fn, 'py', python_rules, color, max_length=110):
|
js_rules,
|
||||||
failed = True
|
json_rules,
|
||||||
return failed
|
markdown_rules,
|
||||||
|
help_markdown_rules,
|
||||||
def check_custom_checks_nonpy():
|
bash_rules,
|
||||||
# type: () -> bool
|
txt_rules,
|
||||||
failed = False
|
]
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['js']:
|
|
||||||
if custom_check_file(fn, 'js', js_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['sh']:
|
|
||||||
if custom_check_file(fn, 'sh', bash_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['scss']:
|
|
||||||
if custom_check_file(fn, 'scss', css_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['handlebars']:
|
|
||||||
if custom_check_file(fn, 'handlebars', handlebars_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['html']:
|
|
||||||
if custom_check_file(fn, 'html', jinja2_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['json']:
|
|
||||||
if custom_check_file(fn, 'json', json_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
markdown_docs_length_exclude = {
|
|
||||||
# Has some example Vagrant output that's very long
|
|
||||||
"docs/development/setup-vagrant.md",
|
|
||||||
# Have wide output in code blocks
|
|
||||||
"docs/subsystems/logging.md",
|
|
||||||
"docs/subsystems/migration-renumbering.md",
|
|
||||||
# Have curl commands with JSON that would be messy to wrap
|
|
||||||
"zerver/webhooks/helloworld/doc.md",
|
|
||||||
"zerver/webhooks/trello/doc.md",
|
|
||||||
# Has a very long configuration line
|
|
||||||
"templates/zerver/integrations/perforce.md",
|
|
||||||
# Has some example code that could perhaps be wrapped
|
|
||||||
"templates/zerver/api/incoming-webhooks-walkthrough.md",
|
|
||||||
# This macro has a long indented URL
|
|
||||||
"templates/zerver/help/include/git-webhook-url-with-branches-indented.md",
|
|
||||||
# These two are the same file and have some too-long lines for GitHub badges
|
|
||||||
"README.md",
|
|
||||||
"docs/overview/readme.md",
|
|
||||||
}
|
|
||||||
for fn in by_lang['md']:
|
|
||||||
max_length = None
|
|
||||||
if fn not in markdown_docs_length_exclude:
|
|
||||||
max_length = 120
|
|
||||||
rules = markdown_rules
|
|
||||||
if fn.startswith("templates/zerver/help"):
|
|
||||||
rules = help_markdown_rules
|
|
||||||
if custom_check_file(fn, 'md', rules, color, max_length=max_length):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['txt'] + by_lang['text']:
|
|
||||||
if custom_check_file(fn, 'txt', txt_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['rst']:
|
|
||||||
if custom_check_file(fn, 'rst', txt_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
color = next(colors)
|
|
||||||
for fn in by_lang['yaml']:
|
|
||||||
if custom_check_file(fn, 'yaml', txt_rules, color):
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
return failed
|
|
||||||
|
|
||||||
return (check_custom_checks_py, check_custom_checks_nonpy)
|
|
||||||
|
|
|
||||||
|
|
@ -3,28 +3,18 @@ import os
|
||||||
from mock import patch
|
from mock import patch
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
from typing import Any, Dict, List
|
from zulint.custom_rules import RuleList
|
||||||
|
from linter_lib.custom_check import python_rules, non_py_rules
|
||||||
from tools.linter_lib.custom_check import build_custom_checkers
|
|
||||||
from tools.linter_lib.custom_check import custom_check_file
|
|
||||||
|
|
||||||
ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
|
ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
|
||||||
CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`."
|
CHECK_MESSAGE = "Fix the corresponding rule in `tools/zulint/custom_rules.py`."
|
||||||
|
|
||||||
class TestCustomRules(TestCase):
|
class TestRuleList(TestCase):
|
||||||
|
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
self.all_rules = [] # type: List[Dict[str, Any]]
|
self.all_rules = python_rules.rules
|
||||||
with patch('tools.linter_lib.custom_check.custom_check_file', return_value=False) as mock_custom_check_file:
|
for rule in non_py_rules:
|
||||||
by_lang = dict.fromkeys(['py', 'js', 'sh', 'scss', 'handlebars', 'html',
|
self.all_rules.extend(rule.rules)
|
||||||
'json', 'md', 'txt', 'text', 'yaml', 'rst'],
|
|
||||||
['foo/bar.baz'])
|
|
||||||
check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)
|
|
||||||
check_custom_checks_py()
|
|
||||||
check_custom_checks_nonpy()
|
|
||||||
for call_args in mock_custom_check_file.call_args_list:
|
|
||||||
rule_set = call_args[0][2]
|
|
||||||
self.all_rules.extend(rule_set)
|
|
||||||
|
|
||||||
def test_paths_in_rules(self) -> None:
|
def test_paths_in_rules(self) -> None:
|
||||||
"""Verifies that the paths mentioned in linter rules actually exist"""
|
"""Verifies that the paths mentioned in linter rules actually exist"""
|
||||||
|
|
@ -53,7 +43,7 @@ class TestCustomRules(TestCase):
|
||||||
for line in rule.get('good_lines', []):
|
for line in rule.get('good_lines', []):
|
||||||
# create=True is superfluous when mocking built-ins in Python >= 3.5
|
# create=True is superfluous when mocking built-ins in Python >= 3.5
|
||||||
with patch('builtins.open', return_value=iter((line+'\n\n').splitlines()), create=True, autospec=True):
|
with patch('builtins.open', return_value=iter((line+'\n\n').splitlines()), create=True, autospec=True):
|
||||||
self.assertFalse(custom_check_file('foo.bar', 'baz', [rule], ''),
|
self.assertFalse(RuleList([], [rule]).custom_check_file('foo.bar', 'baz', ''),
|
||||||
"The pattern '{}' matched the line '{}' while it shouldn't.".format(pattern, line))
|
"The pattern '{}' matched the line '{}' while it shouldn't.".format(pattern, line))
|
||||||
|
|
||||||
for line in rule.get('bad_lines', []):
|
for line in rule.get('bad_lines', []):
|
||||||
|
|
@ -61,5 +51,5 @@ class TestCustomRules(TestCase):
|
||||||
with patch('builtins.open',
|
with patch('builtins.open',
|
||||||
return_value=iter((line+'\n\n').splitlines()), create=True, autospec=True), patch('builtins.print'):
|
return_value=iter((line+'\n\n').splitlines()), create=True, autospec=True), patch('builtins.print'):
|
||||||
filename = list(rule.get('include_only', {'foo.bar'}))[0]
|
filename = list(rule.get('include_only', {'foo.bar'}))[0]
|
||||||
self.assertTrue(custom_check_file(filename, 'baz', [rule], ''),
|
self.assertTrue(RuleList([], [rule]).custom_check_file(filename, 'baz', ''),
|
||||||
"The pattern '{}' didn't match the line '{}' while it should.".format(pattern, line))
|
"The pattern '{}' didn't match the line '{}' while it should.".format(pattern, line))
|
||||||
|
|
@ -6,192 +6,237 @@ from __future__ import absolute_import
|
||||||
import re
|
import re
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
from zulint.printer import print_err
|
from zulint.printer import print_err, colors
|
||||||
|
|
||||||
if False:
|
if False:
|
||||||
from typing import Any, Dict, List, Optional, Tuple, Iterable
|
from typing import Any, Dict, List, Optional, Tuple, Iterable
|
||||||
|
|
||||||
Rule = Dict[str, Any]
|
Rule = List[Dict[str, Any]]
|
||||||
RuleList = List[Dict[str, Any]]
|
|
||||||
LineTup = Tuple[int, str, str, str]
|
LineTup = Tuple[int, str, str, str]
|
||||||
|
|
||||||
def get_line_info_from_file(fn):
|
|
||||||
# type: (str) -> List[LineTup]
|
|
||||||
line_tups = []
|
|
||||||
for i, line in enumerate(open(fn)):
|
|
||||||
line_newline_stripped = line.strip('\n')
|
|
||||||
line_fully_stripped = line_newline_stripped.strip()
|
|
||||||
if line_fully_stripped.endswith(' # nolint'):
|
|
||||||
continue
|
|
||||||
tup = (i, line, line_newline_stripped, line_fully_stripped)
|
|
||||||
line_tups.append(tup)
|
|
||||||
return line_tups
|
|
||||||
|
|
||||||
def get_rules_applying_to_fn(fn, rules):
|
class RuleList:
|
||||||
# type: (str, RuleList) -> RuleList
|
"""Defines and runs custom linting rules for the specified language."""
|
||||||
rules_to_apply = []
|
|
||||||
for rule in rules:
|
def __init__(self, langs, rules, max_length=None, length_exclude=[], exclude_files_in=None):
|
||||||
excluded = False
|
# type: (List[str], Rule, Optional[int], List[str], Rule, Optional[str]) -> None
|
||||||
for item in rule.get('exclude', set()):
|
self.langs = langs
|
||||||
if fn.startswith(item):
|
self.rules = rules
|
||||||
excluded = True
|
self.max_length = max_length
|
||||||
break
|
self.length_exclude = length_exclude
|
||||||
if excluded:
|
# Exclude the files in this folder from rules
|
||||||
continue
|
self.exclude_files_in = "\\"
|
||||||
if rule.get("include_only"):
|
|
||||||
found = False
|
def get_line_info_from_file(self, fn):
|
||||||
for item in rule.get("include_only", set()):
|
# type: (str) -> List[LineTup]
|
||||||
if item in fn:
|
line_tups = []
|
||||||
found = True
|
for i, line in enumerate(open(fn)):
|
||||||
if not found:
|
line_newline_stripped = line.strip('\n')
|
||||||
|
line_fully_stripped = line_newline_stripped.strip()
|
||||||
|
if line_fully_stripped.endswith(' # nolint'):
|
||||||
continue
|
continue
|
||||||
rules_to_apply.append(rule)
|
tup = (i, line, line_newline_stripped, line_fully_stripped)
|
||||||
|
line_tups.append(tup)
|
||||||
|
return line_tups
|
||||||
|
|
||||||
return rules_to_apply
|
def get_rules_applying_to_fn(self, fn, rules):
|
||||||
|
# type: (str, Rule) -> Rule
|
||||||
|
rules_to_apply = []
|
||||||
|
for rule in rules:
|
||||||
|
excluded = False
|
||||||
|
for item in rule.get('exclude', set()):
|
||||||
|
if fn.startswith(item):
|
||||||
|
excluded = True
|
||||||
|
break
|
||||||
|
if excluded:
|
||||||
|
continue
|
||||||
|
if rule.get("include_only"):
|
||||||
|
found = False
|
||||||
|
for item in rule.get("include_only", set()):
|
||||||
|
if item in fn:
|
||||||
|
found = True
|
||||||
|
if not found:
|
||||||
|
continue
|
||||||
|
rules_to_apply.append(rule)
|
||||||
|
|
||||||
def check_file_for_pattern(fn,
|
return rules_to_apply
|
||||||
line_tups,
|
|
||||||
identifier,
|
|
||||||
color,
|
|
||||||
rule):
|
|
||||||
# type: (str, List[LineTup], str, Optional[Iterable[str]], Rule) -> bool
|
|
||||||
|
|
||||||
'''
|
def check_file_for_pattern(self,
|
||||||
DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING.
|
fn,
|
||||||
|
line_tups,
|
||||||
|
identifier,
|
||||||
|
color,
|
||||||
|
rule):
|
||||||
|
# type: (str, List[LineTup], str, Optional[Iterable[str]], Dict[str, Any]) -> bool
|
||||||
|
|
||||||
This function gets called ~40k times, once per file per regex.
|
'''
|
||||||
|
DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING.
|
||||||
|
|
||||||
Inside it's doing a regex check for every line in the file, so
|
This function gets called ~40k times, once per file per regex.
|
||||||
it's important to do things like pre-compiling regexes.
|
|
||||||
|
|
||||||
DO NOT INLINE THIS FUNCTION.
|
Inside it's doing a regex check for every line in the file, so
|
||||||
|
it's important to do things like pre-compiling regexes.
|
||||||
|
|
||||||
We need to see it show up in profiles, and the function call
|
DO NOT INLINE THIS FUNCTION.
|
||||||
overhead will never be a bottleneck.
|
|
||||||
'''
|
|
||||||
exclude_lines = {
|
|
||||||
line for
|
|
||||||
(exclude_fn, line) in rule.get('exclude_line', set())
|
|
||||||
if exclude_fn == fn
|
|
||||||
}
|
|
||||||
|
|
||||||
pattern = re.compile(rule['pattern'])
|
We need to see it show up in profiles, and the function call
|
||||||
strip_rule = rule.get('strip') # type: Optional[str]
|
overhead will never be a bottleneck.
|
||||||
|
'''
|
||||||
|
exclude_lines = {
|
||||||
|
line for
|
||||||
|
(exclude_fn, line) in rule.get('exclude_line', set())
|
||||||
|
if exclude_fn == fn
|
||||||
|
}
|
||||||
|
|
||||||
ok = True
|
pattern = re.compile(rule['pattern'])
|
||||||
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
|
strip_rule = rule.get('strip') # type: Optional[str]
|
||||||
if line_fully_stripped in exclude_lines:
|
|
||||||
exclude_lines.remove(line_fully_stripped)
|
ok = True
|
||||||
continue
|
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
|
||||||
try:
|
if line_fully_stripped in exclude_lines:
|
||||||
line_to_check = line_fully_stripped
|
exclude_lines.remove(line_fully_stripped)
|
||||||
if strip_rule is not None:
|
continue
|
||||||
if strip_rule == '\n':
|
try:
|
||||||
line_to_check = line_newline_stripped
|
line_to_check = line_fully_stripped
|
||||||
else:
|
if strip_rule is not None:
|
||||||
raise Exception("Invalid strip rule")
|
if strip_rule == '\n':
|
||||||
if pattern.search(line_to_check):
|
line_to_check = line_newline_stripped
|
||||||
if rule.get("exclude_pattern"):
|
else:
|
||||||
if re.search(rule['exclude_pattern'], line_to_check):
|
raise Exception("Invalid strip rule")
|
||||||
continue
|
if pattern.search(line_to_check):
|
||||||
print_err(identifier, color, '{} at {} line {}:'.format(
|
if rule.get("exclude_pattern"):
|
||||||
rule['description'], fn, i+1))
|
if re.search(rule['exclude_pattern'], line_to_check):
|
||||||
print_err(identifier, color, line)
|
continue
|
||||||
|
print_err(identifier, color, '{} at {} line {}:'.format(
|
||||||
|
rule['description'], fn, i+1))
|
||||||
|
print_err(identifier, color, line)
|
||||||
|
ok = False
|
||||||
|
except Exception:
|
||||||
|
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
|
||||||
|
traceback.print_exc()
|
||||||
|
|
||||||
|
if exclude_lines:
|
||||||
|
print('Please remove exclusions for file %s: %s' % (fn, exclude_lines))
|
||||||
|
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def check_file_for_long_lines(self,
|
||||||
|
fn,
|
||||||
|
max_length,
|
||||||
|
line_tups):
|
||||||
|
# type: (str, int, List[LineTup]) -> bool
|
||||||
|
ok = True
|
||||||
|
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
|
||||||
|
if isinstance(line, bytes):
|
||||||
|
line_length = len(line.decode("utf-8"))
|
||||||
|
else:
|
||||||
|
line_length = len(line)
|
||||||
|
if (line_length > max_length and
|
||||||
|
'# type' not in line and 'test' not in fn and 'example' not in fn and
|
||||||
|
# Don't throw errors for markdown format URLs
|
||||||
|
not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and
|
||||||
|
# Don't throw errors for URLs in code comments
|
||||||
|
not re.search(r"[#].*http.*", line) and
|
||||||
|
not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and
|
||||||
|
"# ignorelongline" not in line and 'migrations' not in fn):
|
||||||
|
print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped))
|
||||||
ok = False
|
ok = False
|
||||||
except Exception:
|
return ok
|
||||||
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
|
|
||||||
traceback.print_exc()
|
|
||||||
|
|
||||||
if exclude_lines:
|
def custom_check_file(self,
|
||||||
print('Please remove exclusions for file %s: %s' % (fn, exclude_lines))
|
fn,
|
||||||
|
identifier,
|
||||||
|
color,
|
||||||
|
max_length=None):
|
||||||
|
# type: (str, str, Optional[Iterable[str]], Optional[int]) -> bool
|
||||||
|
failed = False
|
||||||
|
|
||||||
return ok
|
line_tups = self.get_line_info_from_file(fn=fn)
|
||||||
|
|
||||||
def check_file_for_long_lines(fn,
|
rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.rules)
|
||||||
max_length,
|
|
||||||
line_tups):
|
|
||||||
# type: (str, int, List[LineTup]) -> bool
|
|
||||||
ok = True
|
|
||||||
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
|
|
||||||
if isinstance(line, bytes):
|
|
||||||
line_length = len(line.decode("utf-8"))
|
|
||||||
else:
|
|
||||||
line_length = len(line)
|
|
||||||
if (line_length > max_length and
|
|
||||||
'# type' not in line and 'test' not in fn and 'example' not in fn and
|
|
||||||
# Don't throw errors for markdown format URLs
|
|
||||||
not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and
|
|
||||||
# Don't throw errors for URLs in code comments
|
|
||||||
not re.search(r"[#].*http.*", line) and
|
|
||||||
not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and
|
|
||||||
"# ignorelongline" not in line and 'migrations' not in fn):
|
|
||||||
print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped))
|
|
||||||
ok = False
|
|
||||||
return ok
|
|
||||||
|
|
||||||
def custom_check_file(fn,
|
for rule in rules_to_apply:
|
||||||
identifier,
|
ok = self.check_file_for_pattern(
|
||||||
rules,
|
fn=fn,
|
||||||
color,
|
line_tups=line_tups,
|
||||||
max_length=None):
|
identifier=identifier,
|
||||||
# type: (str, str, RuleList, Optional[Iterable[str]], Optional[int]) -> bool
|
color=color,
|
||||||
failed = False
|
rule=rule,
|
||||||
|
)
|
||||||
line_tups = get_line_info_from_file(fn=fn)
|
if not ok:
|
||||||
|
|
||||||
rules_to_apply = get_rules_applying_to_fn(fn=fn, rules=rules)
|
|
||||||
|
|
||||||
for rule in rules_to_apply:
|
|
||||||
ok = check_file_for_pattern(
|
|
||||||
fn=fn,
|
|
||||||
line_tups=line_tups,
|
|
||||||
identifier=identifier,
|
|
||||||
color=color,
|
|
||||||
rule=rule,
|
|
||||||
)
|
|
||||||
if not ok:
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
# TODO: Move the below into more of a framework.
|
|
||||||
firstline = None
|
|
||||||
lastLine = None
|
|
||||||
if line_tups:
|
|
||||||
firstline = line_tups[0][3] # line_fully_stripped for the first line.
|
|
||||||
lastLine = line_tups[-1][1]
|
|
||||||
|
|
||||||
if max_length is not None:
|
|
||||||
ok = check_file_for_long_lines(
|
|
||||||
fn=fn,
|
|
||||||
max_length=max_length,
|
|
||||||
line_tups=line_tups,
|
|
||||||
)
|
|
||||||
if not ok:
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
if firstline:
|
|
||||||
shebang_rules = [
|
|
||||||
{'pattern': '^#!',
|
|
||||||
'description': "zerver library code shouldn't have a shebang line.",
|
|
||||||
'include_only': set(['zerver/'])},
|
|
||||||
# /bin/sh and /usr/bin/env are the only two binaries
|
|
||||||
# that NixOS provides at a fixed path (outside a
|
|
||||||
# buildFHSUserEnv sandbox).
|
|
||||||
{'pattern': '^#!(?! *(?:/usr/bin/env|/bin/sh)(?: |$))',
|
|
||||||
'description': "Use `#!/usr/bin/env foo` instead of `#!/path/foo`"
|
|
||||||
" for interpreters other than sh."},
|
|
||||||
{'pattern': '^#!/usr/bin/env python$',
|
|
||||||
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}
|
|
||||||
] # type: RuleList
|
|
||||||
shebang_rules_to_apply = get_rules_applying_to_fn(fn=fn, rules=shebang_rules)
|
|
||||||
for rule in shebang_rules_to_apply:
|
|
||||||
if re.search(rule['pattern'], firstline):
|
|
||||||
print_err(identifier, color,
|
|
||||||
'{} at {} line 1:'.format(rule['description'], fn))
|
|
||||||
print_err(identifier, color, firstline)
|
|
||||||
failed = True
|
failed = True
|
||||||
|
|
||||||
if lastLine and ('\n' not in lastLine):
|
# TODO: Move the below into more of a framework.
|
||||||
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
|
firstline = None
|
||||||
failed = True
|
lastLine = None
|
||||||
|
if line_tups:
|
||||||
|
# line_fully_stripped for the first line.
|
||||||
|
firstline = line_tups[0][3]
|
||||||
|
lastLine = line_tups[-1][1]
|
||||||
|
|
||||||
return failed
|
if max_length is not None:
|
||||||
|
ok = self.check_file_for_long_lines(
|
||||||
|
fn=fn,
|
||||||
|
max_length=max_length,
|
||||||
|
line_tups=line_tups,
|
||||||
|
)
|
||||||
|
if not ok:
|
||||||
|
failed = True
|
||||||
|
|
||||||
|
if firstline:
|
||||||
|
shebang_rules = [
|
||||||
|
{'pattern': '^#!',
|
||||||
|
'description': "zerver library code shouldn't have a shebang line.",
|
||||||
|
'include_only': set(['zerver/'])},
|
||||||
|
# /bin/sh and /usr/bin/env are the only two binaries
|
||||||
|
# that NixOS provides at a fixed path (outside a
|
||||||
|
# buildFHSUserEnv sandbox).
|
||||||
|
{'pattern': '^#!(?! *(?:/usr/bin/env|/bin/sh)(?: |$))',
|
||||||
|
'description': "Use `#!/usr/bin/env foo` instead of `#!/path/foo`"
|
||||||
|
" for interpreters other than sh."},
|
||||||
|
{'pattern': '^#!/usr/bin/env python$',
|
||||||
|
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}
|
||||||
|
] # type: Rule
|
||||||
|
shebang_rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=shebang_rules)
|
||||||
|
for rule in shebang_rules_to_apply:
|
||||||
|
if re.search(rule['pattern'], firstline):
|
||||||
|
print_err(identifier, color,
|
||||||
|
'{} at {} line 1:'.format(rule['description'], fn))
|
||||||
|
print_err(identifier, color, firstline)
|
||||||
|
failed = True
|
||||||
|
|
||||||
|
if lastLine and ('\n' not in lastLine):
|
||||||
|
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
|
||||||
|
failed = True
|
||||||
|
|
||||||
|
return failed
|
||||||
|
|
||||||
|
def check(self, by_lang):
|
||||||
|
# type: (Dict[str, List[str]]) -> bool
|
||||||
|
# By default, a rule applies to all files within the extension for
|
||||||
|
# which it is specified (e.g. all .py files)
|
||||||
|
# There are three operators we can use to manually include or exclude files from linting for a rule:
|
||||||
|
# 'exclude': 'set([<path>, ...])' - if <path> is a filename, excludes that file.
|
||||||
|
# if <path> is a directory, excludes all files
|
||||||
|
# directly below the directory <path>.
|
||||||
|
# 'exclude_line': 'set([(<path>, <line>), ...])' - excludes all lines matching <line>
|
||||||
|
# in the file <path> from linting.
|
||||||
|
# 'include_only': 'set([<path>, ...])' - includes only those files where <path> is a
|
||||||
|
# substring of the filepath.
|
||||||
|
failed = False
|
||||||
|
for lang in self.langs:
|
||||||
|
color = next(colors)
|
||||||
|
for fn in by_lang[lang]:
|
||||||
|
if fn.startswith(self.exclude_files_in) or ('custom_check.py' in fn):
|
||||||
|
# This is a bit of a hack, but it generally really doesn't
|
||||||
|
# work to check the file that defines all the things to check for.
|
||||||
|
#
|
||||||
|
# TODO: Migrate this to looking at __module__ type attributes.
|
||||||
|
continue
|
||||||
|
max_length = None
|
||||||
|
if fn not in self.length_exclude:
|
||||||
|
max_length = self.max_length
|
||||||
|
if self.custom_check_file(fn, lang, color, max_length=max_length):
|
||||||
|
failed = True
|
||||||
|
|
||||||
|
return failed
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue