linter_lib: Fix mypy errors.

tools/linter_lib/pyflakes.py:35: error: Argument 3 to "run_pyflakes" has incompatible type "List[Tuple[bytes, bytes]]"; expected "List[Tuple[str, str]]"
    tools/linter_lib/custom_check.py:110: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:214: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:214: error: Argument "shebang_rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:502: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:502: error: Argument "shebang_rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:519: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:706: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:728: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:738: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:779: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"
    tools/linter_lib/custom_check.py:779: error: Argument "length_exclude" to "RuleList" has incompatible type "Set[str]"; expected "List[str]"
    tools/linter_lib/custom_check.py:803: error: Argument "length_exclude" to "RuleList" has incompatible type "Set[str]"; expected "List[str]"
    tools/linter_lib/custom_check.py:805: error: Unsupported operand types for + ("List[Rule]" and "List[Dict[str, Any]]")
    tools/linter_lib/custom_check.py:819: error: Argument "rules" to "RuleList" has incompatible type "List[Dict[str, Any]]"; expected "List[Rule]"

These were missed the `zulint` package was missing PEP 561 type
annotation markers, and if it’d had them, mypy daemon mode would’ve
required us to set `follow_imports = skip` for it.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

mypy.ini

@@ -78,6 +78,24 @@ ignore_errors = True
 # haven't told it to.)
 follow_imports = skip
 
+[mypy-zulint]
+follow_imports = skip
+
+[mypy-zulint.command]
+follow_imports = skip
+
+[mypy-zulint.custom_rules]
+follow_imports = skip
+
+[mypy-zulint.linters]
+follow_imports = skip
+
+[mypy-zulint.lister]
+follow_imports = skip
+
+[mypy-zulint.printer]
+follow_imports = skip
+
 
 #
 #

requirements/dev.in

@@ -53,6 +53,6 @@ python-digitalocean==1.14.0
 pip-tools==2.0.2
 
 # zulip's linting framework - zulint
--e git+https://github.com/zulip/zulint@56f9cd518aeb11c0e7b8914eee2a70516e301e10#egg=zulint==0.0.1
+-e git+https://github.com/zulip/zulint@9e0da9f6c9ffc617e0782ab69617f6bcf90a4271#egg=zulint==0.0.1
 
 -r mypy.in

requirements/dev.txt

@@ -13,7 +13,7 @@ git+https://github.com/zulip/libthumbor.git@60ed2431c07686a12f2770b2d852c5650f3c
 git+https://github.com/zulip/line_profiler.git#egg=line_profiler==2.1.2.zulip1
 git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1
 git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git
-git+https://github.com/zulip/zulint@56f9cd518aeb11c0e7b8914eee2a70516e301e10#egg=zulint==0.0.1
+git+https://github.com/zulip/zulint@9e0da9f6c9ffc617e0782ab69617f6bcf90a4271#egg=zulint==0.0.1
 git+https://github.com/zulip/python-zulip-api.git@0.6.1#egg=zulip==0.6.1_git&subdirectory=zulip
 git+https://github.com/zulip/python-zulip-api.git@0.6.1#egg=zulip_bots==0.6.1+git&subdirectory=zulip_bots
 alabaster==0.7.12         # via sphinx

tools/check-templates

@@ -11,7 +11,7 @@ from lib import sanity_check
 sanity_check.check_venv(__file__)
 
 from zulint import lister
-from typing import cast, Dict, Iterable, List
+from typing import Dict, Iterable, List
 
 EXCLUDED_FILES = [
     ## Test data Files for testing modules in tests
@@ -24,13 +24,12 @@ EXCLUDED_FILES = [
 
 def check_our_files(modified_only, all_dups, fix, targets):
     # type: (bool, bool, bool, List[str]) -> None
-    by_lang = cast(
+    by_lang = lister.list_files(
-        Dict[str, List[str]],
-        lister.list_files(
         targets=targets,
         modified_only=args.modified,
         ftypes=['hbs', 'html'],
-            group_by_ftype=True, exclude=EXCLUDED_FILES))
+        group_by_ftype=True, exclude=EXCLUDED_FILES,
+    )
 
     check_handlebar_templates(by_lang['hbs'], fix)
     check_html_templates(by_lang['html'], all_dups, fix)

tools/linter_lib/custom_check.py

@@ -3,11 +3,12 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
+from typing import List, TYPE_CHECKING
+
 from zulint.custom_rules import RuleList
+if TYPE_CHECKING:
+    from zulint.custom_rules import Rule
 
-from typing import cast, Any, Dict, List, Tuple
-
-Rule = List[Dict[str, Any]]
 # Rule help:
 # By default, a rule applies to all files within the extension for which it is specified (e.g. all .py files)
 # There are three operators we can use to manually include or exclude files from linting for a rule:
@@ -15,7 +16,6 @@ Rule = List[Dict[str, Any]]
 #                                   if <path> is a directory, excludes all files directly below the directory <path>.
 # 'exclude_line': 'set([(<path>, <line>), ...])' - excludes all lines matching <line> in the file <path> from linting.
 # 'include_only': 'set([<path>, ...])' - includes only those files where <path> is a substring of the filepath.
-LineTup = Tuple[int, str, str, str]
 
 PYDELIMS = r'''"'()\[\]{}#\\'''
 PYREG = r"[^{}]".format(PYDELIMS)
@@ -67,13 +67,13 @@ shebang_rules = [
      " for interpreters other than sh."},
     {'pattern': '^#!/usr/bin/env python$',
      'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}
-]  # type: Rule
+]  # type: List[Rule]
 
 trailing_whitespace_rule = {
     'pattern': r'\s+$',
     'strip': '\n',
     'description': 'Fix trailing whitespace'
-}
+}  # type: Rule
 whitespace_rules = [
     # This linter should be first since bash_rules depends on it.
     trailing_whitespace_rule,
@@ -84,14 +84,14 @@ whitespace_rules = [
      'strip': '\n',
      'exclude': set(['tools/ci/success-http-headers.txt']),
      'description': 'Fix tab-based whitespace'},
-]  # type: Rule
+]  # type: List[Rule]
 comma_whitespace_rule = [
     {'pattern': ', {2,}[^#/ ]',
      'exclude': set(['zerver/tests', 'frontend_tests/node_tests', 'corporate/tests']),
      'description': "Remove multiple whitespaces after ','",
      'good_lines': ['foo(1, 2, 3)', 'foo = bar  # some inline comment'],
      'bad_lines': ['foo(1,  2, 3)', 'foo(1,    2, 3)']},
-]  # type: Rule
+]  # type: List[Rule]
 markdown_whitespace_rules = list([rule for rule in whitespace_rules if rule['pattern'] != r'\s+$']) + [
     # Two spaces trailing a line with other content is okay--it's a markdown line break.
     # This rule finds one space trailing a non-space, three or more trailing spaces, and
@@ -104,12 +104,12 @@ markdown_whitespace_rules = list([rule for rule in whitespace_rules if rule['pat
      'description': 'Missing space after # in heading',
      'good_lines': ['### some heading', '# another heading'],
      'bad_lines': ['###some heading', '#another heading']},
-]  # type: Rule
+]
 
 
 js_rules = RuleList(
     langs=['js'],
-    rules=cast(Rule, [
+    rules=[
         {'pattern': 'subject|SUBJECT',
          'exclude': set(['static/js/util.js',
                          'frontend_tests/']),
@@ -132,13 +132,13 @@ js_rules = RuleList(
         {'pattern': r'\+.*i18n\.t\(.+\)',
          'description': 'Do not concatenate i18n strings'},
         {'pattern': '[.]includes[(]',
-         'exclude': ['frontend_tests/'],
+         'exclude': {'frontend_tests/'},
          'description': '.includes() is incompatible with Internet Explorer. Use .indexOf() !== -1 instead.'},
         {'pattern': '[.]html[(]',
          'exclude_pattern': r'''[.]html[(]("|'|render_|html|message.content|sub.rendered_description|i18n.t|rendered_|$|[)]|error_text|widget_elem|[$]error|[$][(]"<p>"[)])''',
-         'exclude': ['static/js/portico', 'static/js/lightbox.js', 'static/js/ui_report.js',
+         'exclude': {'static/js/portico', 'static/js/lightbox.js', 'static/js/ui_report.js',
                      'static/js/confirm_dialog.js',
-                     'frontend_tests/'],
+                     'frontend_tests/'},
          'description': 'Setting HTML content with jQuery .html() can lead to XSS security bugs.  Consider .text() or using rendered_foo as a variable name if content comes from handlebars and thus is already sanitized.'},
         {'pattern': '["\']json/',
          'description': 'Relative URL for JSON route not supported by i18n'},
@@ -208,12 +208,14 @@ js_rules = RuleList(
          ]),
          'good_lines': ['#my-style {color: blue;}'],
          'bad_lines': ['<p style="color: blue;">Foo</p>', 'style = "color: blue;"']},
-    ]) + whitespace_rules + comma_whitespace_rule,
+        *whitespace_rules,
+        *comma_whitespace_rule,
+    ],
 )
 
 python_rules = RuleList(
     langs=['py'],
-    rules=cast(Rule, [
+    rules=[
         {'pattern': 'subject|SUBJECT',
          'exclude_pattern': 'subject to the|email|outbox',
          'description': 'avoid subject as a var',
@@ -494,14 +496,16 @@ python_rules = RuleList(
          'include_only': set(["/management/commands/"]),
          'description': 'Raise CommandError to exit with failure in management commands',
          },
-    ]) + whitespace_rules + comma_whitespace_rule,
+        *whitespace_rules,
+        *comma_whitespace_rule,
+    ],
     max_length=110,
     shebang_rules=shebang_rules,
 )
 
 bash_rules = RuleList(
     langs=['bash'],
-    rules=cast(Rule, [
+    rules=[
         {'pattern': '#!.*sh [-xe]',
          'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches'
                         ' to set -x|set -e'},
@@ -512,13 +516,14 @@ bash_rules = RuleList(
              'scripts/lib/install',
              'scripts/setup/configure-rabbitmq'
          ]), },
-    ]) + whitespace_rules[0:1],
+        *whitespace_rules[0:1],
+    ],
     shebang_rules=shebang_rules,
 )
 
 css_rules = RuleList(
     langs=['css', 'scss'],
-    rules=cast(Rule, [
+    rules=[
         {'pattern': r'calc\([^+]+\+[^+]+\)',
          'description': "Avoid using calc with '+' operator. See #8403 : in CSS.",
          'good_lines': ["width: calc(20% - -14px);"],
@@ -560,10 +565,12 @@ css_rules = RuleList(
          'description': 'Use of rgb(a) format is banned, Please use hsl(a) instead',
          'good_lines': ['hsl(0, 0%, 0%)', 'hsla(0, 0%, 100%, 0.1)'],
          'bad_lines': ['rgb(0, 0, 0)', 'rgba(255, 255, 255, 0.1)']},
-    ]) + whitespace_rules + comma_whitespace_rule
+        *whitespace_rules,
+        *comma_whitespace_rule,
+    ],
 )
 
-prose_style_rules = cast(Rule, [
+prose_style_rules = [
     {'pattern': r'[^\/\#\-"]([jJ]avascript)',  # exclude usage in hrefs/divs
      'exclude': set(["docs/documentation/api.md"]),
      'description': "javascript should be spelled JavaScript"},
@@ -571,15 +578,16 @@ prose_style_rules = cast(Rule, [
      'description': "github should be spelled GitHub"},
     {'pattern': '[oO]rganisation',  # exclude usage in hrefs/divs
      'description': "Organization is spelled with a z",
-     'exclude_line': [('docs/translating/french.md', '* organization - **organisation**')]},
+     'exclude_line': {('docs/translating/french.md', '* organization - **organisation**')}},
     {'pattern': '!!! warning',
      'description': "!!! warning is invalid; it's spelled '!!! warn'"},
     {'pattern': 'Terms of service',
      'description': "The S in Terms of Service is capitalized"},
     {'pattern': '[^-_]botserver(?!rc)|bot server',
      'description': "Use Botserver instead of botserver or bot server."},
-]) + comma_whitespace_rule
+    *comma_whitespace_rule,
-html_rules = whitespace_rules + prose_style_rules + cast(Rule, [
+]  # type: List[Rule]
+html_rules = whitespace_rules + prose_style_rules + [
     {'pattern': 'subject|SUBJECT',
      'exclude': set(['templates/zerver/email.html']),
      'exclude_pattern': 'email subject',
@@ -588,8 +596,8 @@ html_rules = whitespace_rules + prose_style_rules + cast(Rule, [
      'bad_lines': ['subject="foo"', ' MAX_SUBJECT_LEN']},
     {'pattern': r'placeholder="[^{#](?:(?!\.com).)+$',
      'description': "`placeholder` value should be translatable.",
-     'exclude_line': [('templates/zerver/register.html', 'placeholder="acme"'),
+     'exclude_line': {('templates/zerver/register.html', 'placeholder="acme"'),
-                      ('templates/zerver/register.html', 'placeholder="Acme or Aκμή"')],
+                      ('templates/zerver/register.html', 'placeholder="Acme or Aκμή"')},
      'exclude': set(["templates/analytics/support.html"]),
      'good_lines': ['<input class="stream-list-filter" type="text" placeholder="{{ _(\'Search streams\') }}" />'],
      'bad_lines': ['<input placeholder="foo">']},
@@ -701,11 +709,11 @@ html_rules = whitespace_rules + prose_style_rules + cast(Rule, [
      ]),
      'good_lines': ['#my-style {color: blue;}', 'style="display: none"', "style='display: none"],
      'bad_lines': ['<p style="color: blue;">Foo</p>', 'style = "color: blue;"']},
-])
+]  # type: List[Rule]
 
 handlebars_rules = RuleList(
     langs=['hbs'],
-    rules=html_rules + cast(Rule, [
+    rules=html_rules + [
         {'pattern': "[<]script",
          'description': "Do not use inline <script> tags here; put JavaScript in static/js instead."},
         {'pattern': '{{ t ("|\')',
@@ -722,22 +730,22 @@ handlebars_rules = RuleList(
          'description': 'Translatable strings should not have trailing spaces.'},
         {'pattern': '{{t "[^"]+ " }}',
          'description': 'Translatable strings should not have trailing spaces.'},
-    ]),
+    ],
 )
 
 jinja2_rules = RuleList(
     langs=['html'],
-    rules=html_rules + cast(Rule, [
+    rules=html_rules + [
         {'pattern': r"{% endtrans %}[\.\?!]",
          'description': "Period should be part of the translatable string."},
         {'pattern': r"{{ _(.+) }}[\.\?!]",
          'description': "Period should be part of the translatable string."},
-    ]),
+    ],
 )
 
 json_rules = RuleList(
     langs=['json'],
-    rules=cast(Rule, [
+    rules=[
         # Here, we don't use `whitespace_rules`, because the tab-based
         # whitespace rule flags a lot of third-party JSON fixtures
         # under zerver/webhooks that we want preserved verbatim.  So
@@ -753,7 +761,7 @@ json_rules = RuleList(
         {'pattern': r'":["\[\{]',
          'exclude': set(['zerver/webhooks/', 'zerver/tests/fixtures/']),
          'description': 'Require space after : in JSON'},
-    ])
+    ]
 )
 
 markdown_docs_length_exclude = {
@@ -778,11 +786,11 @@ markdown_docs_length_exclude = {
 
 markdown_rules = RuleList(
     langs=['md'],
-    rules=markdown_whitespace_rules + prose_style_rules + cast(Rule, [
+    rules=markdown_whitespace_rules + prose_style_rules + [
         {'pattern': r'\[(?P<url>[^\]]+)\]\((?P=url)\)',
          'description': 'Linkified markdown URLs should use cleaner <http://example.com> syntax.'},
         {'pattern': 'https://zulip.readthedocs.io/en/latest/[a-zA-Z0-9]',
-         'exclude': ['docs/overview/contributing.md', 'docs/overview/readme.md', 'docs/README.md'],
+         'exclude': {'docs/overview/contributing.md', 'docs/overview/readme.md', 'docs/README.md'},
          'include_only': set(['docs/']),
          'description': "Use relative links (../foo/bar.html) to other documents in docs/",
          },
@@ -794,7 +802,7 @@ markdown_rules = RuleList(
          'include_only': set(['README.md', 'CONTRIBUTING.md']),
          'description': "Use absolute links from docs served by GitHub",
          },
-    ]),
+    ],
     max_length=120,
     length_exclude=markdown_docs_length_exclude,
     exclude_files_in='templates/zerver/help/'
@@ -802,7 +810,7 @@ markdown_rules = RuleList(
 
 help_markdown_rules = RuleList(
     langs=['md'],
-    rules=markdown_rules.rules + cast(Rule, [
+    rules=markdown_rules.rules + [
         {'pattern': '[a-z][.][A-Z]',
          'description': "Likely missing space after end of sentence",
          'include_only': set(['templates/zerver/help/']),
@@ -812,7 +820,7 @@ help_markdown_rules = RuleList(
          'good_lines': ['Organization', 'deactivate_realm', 'realm_filter'],
          'bad_lines': ['Users are in a realm', 'Realm is the best model'],
          'description': "Realms are referred to as Organizations in user-facing docs."},
-    ]),
+    ],
     length_exclude=markdown_docs_length_exclude,
 )
 

tools/linter_lib/pyflakes.py

@@ -11,24 +11,24 @@ from zulint.linters import run_pyflakes
 def check_pyflakes(files, options):
     # type: (List[str], argparse.Namespace) -> bool
     suppress_patterns = [
-        (b"scripts/lib/pythonrc.py", b"imported but unused"),
+        ("scripts/lib/pythonrc.py", "imported but unused"),
-        (b'', b"'scripts.lib.setup_path_on_import' imported but unused"),
+        ('', "'scripts.lib.setup_path_on_import' imported but unused"),
         # Intentionally imported by zerver/lib/webhooks/common.py
-        (b'', b"'zerver.lib.exceptions.UnexpectedWebhookEventType' imported but unused"),
+        ('', "'zerver.lib.exceptions.UnexpectedWebhookEventType' imported but unused"),
 
 
         # Our ipython startup pythonrc file intentionally imports *
-        (b"scripts/lib/pythonrc.py",
+        ("scripts/lib/pythonrc.py",
-         b" import *' used; unable to detect undefined names"),
+         " import *' used; unable to detect undefined names"),
 
         # Special dev_settings.py import
-        (b'', b"from .prod_settings_template import *"),
+        ('', "from .prod_settings_template import *"),
 
-        (b"settings.py", b"settings import *' used; unable to detect undefined names"),
+        ("settings.py", "settings import *' used; unable to detect undefined names"),
-        (b"settings.py", b"may be undefined, or defined from star imports"),
+        ("settings.py", "may be undefined, or defined from star imports"),
 
         # Sphinx adds `tags` specially to the environment when running conf.py.
-        (b"docs/conf.py", b"undefined name 'tags'"),
+        ("docs/conf.py", "undefined name 'tags'"),
     ]
     if options.full:
         suppress_patterns = []

tools/run-mypy

@@ -7,7 +7,6 @@ import argparse
 import subprocess
 
 from zulint import lister
-from typing import cast, Dict, List
 
 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
 os.chdir(os.path.dirname(TOOLS_DIR))
@@ -58,11 +57,12 @@ if args.all:
     exclude = []
 
 # find all non-excluded files in current directory
-files_dict = cast(Dict[str, List[str]],
+files_dict = lister.list_files(
-                  lister.list_files(targets=args.targets, ftypes=['py', 'pyi'],
+    targets=args.targets, ftypes=['py', 'pyi'],
     use_shebang=True, modified_only=args.modified,
     exclude=exclude, group_by_ftype=True,
-                                    extless_only=args.scripts_only))
+    extless_only=args.scripts_only,
+)
 pyi_files = list(files_dict['pyi'])
 python_files = [fpath for fpath in files_dict['py']
                 if not fpath.endswith('.py') or fpath + 'i' not in pyi_files]

version.py

@@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '46.3'
+PROVISION_VERSION = '47.0'