zulip/tools/run-mypy

#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import print_function

import os
import sys
import lister
import argparse
import subprocess
import six

from typing import cast, Dict, List

TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
os.chdir(os.path.dirname(TOOLS_DIR))

exclude_common = """
api/integrations/asana/zulip_asana_config.py
api/integrations/basecamp/zulip_basecamp_config.py
api/integrations/codebase/zulip_codebase_config.py
api/integrations/git/zulip_git_config.py
api/integrations/hg/zulip-changegroup.py
api/integrations/perforce/git_p4.py
api/integrations/perforce/zulip_change-commit.py
api/integrations/perforce/zulip_perforce_config.py
api/integrations/svn/zulip_svn_config.py
api/integrations/trac/zulip_trac_config.py
api/integrations/trac/zulip_trac.py
api/zulip/__init__.py
bots/jabber_mirror_backend.py
bots/zephyr_mirror_backend.py
tools/deprecated/generate-activity-metrics.py
zproject/settings.py
zproject/test_settings.py
zerver/tests/test_decorators.py
zerver/tests/test_narrow.py
""".split()

# We don't run mypy on contrib_bots, since the code there will
# often be shared with other projects that do not want a mypy
# dependency (at least while it's still kind of beta).
exclude_common  += ['contrib_bots']

exclude_py2 = [] # type: List[str]

exclude_py3 = """
zerver/lib/ccache.py
""".split()

exclude_scripts_common = """
api/integrations/asana/zulip_asana_mirror
api/integrations/basecamp/zulip_basecamp_mirror
api/integrations/codebase/zulip_codebase_mirror
api/integrations/git/post-receive
api/integrations/nagios/nagios-notify-zulip
api/integrations/rss/rss-bot
api/integrations/svn/post-commit
api/integrations/twitter/twitter-bot
api/integrations/twitter/twitter-search-bot
bots/gcal-bot
bots/githook-post-receive
tools/deprecated/inject-messages/inject-messages
puppet/zulip_internal
""".split()

exclude_scripts_py2 = [] # type: List[str]

exclude_scripts_py3 = """
bots/process_ccache
""".split()

parser = argparse.ArgumentParser(description="Run mypy on files tracked by git.")
parser.add_argument('targets', nargs='*', default=[],
                    help="""files and directories to include in the result.
                    If this is not specified, the current directory is used""")
parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files')
parser.add_argument('-a', '--all', dest='all', action='store_true', default=False,
                    help="""run mypy on all python files, ignoring the exclude list.
                    This is useful if you have to find out which files fail mypy check.""")
parser.add_argument('--linecoverage-report', dest='linecoverage_report', action='store_true', default=False,
                    help="""run the linecoverage report to see annotation coverage""")
parser.add_argument('--no-disallow-untyped-defs', dest='disallow_untyped_defs', action='store_false', default=True,
                    help="""Don't throw errors when functions are not annotated""")
parser.add_argument('--scripts-only', dest='scripts_only', action='store_true', default=False,
                    help="""Only type check extensionless python scripts""")

group = parser.add_mutually_exclusive_group()
group.add_argument('--py2', default=False, action='store_true', help="Use Python 2 mode")
group.add_argument('--py3', default=False, action='store_true', help="Use Python 3 mode")
args = parser.parse_args()

if args.py2:
    py_version = 2
elif args.py3:
    py_version = 3
else:
    py_version = 2

if args.all:
    exclude = [] # type: List[str]
elif args.scripts_only:
    if py_version == 2:
        exclude = exclude_scripts_common + exclude_scripts_py2
    else:
        exclude = exclude_scripts_common + exclude_scripts_py3
else:
    if py_version == 2:
        exclude = exclude_common + exclude_py2
    else:
        exclude = exclude_common + exclude_py3

# find all non-excluded files in current directory
files_dict = cast(Dict[str, List[str]],
                  lister.list_files(targets=args.targets, ftypes=['py', 'pyi'],
                                    use_shebang=args.scripts_only, modified_only=args.modified,
                                    exclude = exclude + ['stubs'], group_by_ftype=True,
                                    extless_only=args.scripts_only))
pyi_files = set(files_dict['pyi'])
python_files = [fpath for fpath in files_dict['py']
                if not fpath.endswith('.py') or fpath + 'i' not in pyi_files]

# Use zulip-py3-venv's mypy if it's available and we're on python 2
PY3_VENV_DIR = "/srv/zulip-py3-venv"
MYPY_VENV_PATH = os.path.join(PY3_VENV_DIR, "bin", "mypy")
if six.PY2 and os.path.exists(MYPY_VENV_PATH):
    mypy_command = MYPY_VENV_PATH
    print("Using mypy from", mypy_command)
else:
    mypy_command = "mypy"

extra_args = ["--fast-parser", "--silent-imports", "--check-untyped-defs", "--scripts-are-modules"]
if py_version == 2:
    extra_args.append("--py2")
if args.linecoverage_report:
    extra_args.append("--linecoverage-report")
    extra_args.append("var/linecoverage-report")
if args.disallow_untyped_defs:
    extra_args.append("--disallow-untyped-defs")


# run mypy
if python_files:
    if args.scripts_only:
        rc = subprocess.call([mypy_command] + extra_args + python_files)
    else:
        rc = subprocess.call([mypy_command] + extra_args + python_files)
        if args.linecoverage_report:
            # Move the coverage report to where coveralls will look for it.
            try:
                os.rename('var/linecoverage-report/coverage.txt', 'var/.coverage')
            except OSError:
                # maybe mypy crashed; exit with its error code
                pass
    sys.exit(rc)
else:
    print("There are no files to run mypy on.")