dependencies: Upgrade to Django 3.1.

https://docs.djangoproject.com/en/3.1/releases/3.1/

- django.contrib.postgres.fields.JSONField is deprecated and should be
  replaced with models.JSONField
-  The internals of the implementation in the postgresql backend have
   changed a bit in
   f48f671223
   and thus we need to make an ugly tweak in test_runner.
- app_directories.Loader.get_dirs() now returns a list of PosixPath so
  we need to make a small tweak in TwoFactorLoader for that (PosixPath
  is not iterable)

Fixes #16010.

requirements/common.in

@@ -3,7 +3,7 @@
 # and requirements/prod.txt.
 # See requirements/README.md for more detail.
 # Django itself
-Django==3.0.*
+Django==3.1.*
 
 # needed for Literal, TypedDict
 typing-extensions

requirements/dev.txt

@@ -316,9 +316,9 @@ django-webpack4-loader==0.0.5 \
     --hash=sha256:baa043c4601ed763d161490e2888cf6aa93a2fd9b60681e6b19e35dc7fcb155d \
     --hash=sha256:be90257041170f39c025ff674c9064569c9303b464536488b6a6cedd8e3d28be \
     # via -r requirements/common.in
-django==3.0.11 \
+django==3.1.5 \
-    --hash=sha256:8c334df4160f7c89f6a8a359dd4e95c688ec5ac0db5db75fcc6fec8f590dc8cf \
+    --hash=sha256:2d78425ba74c7a1a74b196058b261b9733a8570782f4e2828974777ccca7edf7 \
-    --hash=sha256:96436d3d2f744d26e193bfb5a1cff3e01b349f835bb0ea16f71743accf9c6fa9 \
+    --hash=sha256:efa2ab96b33b20c2182db93147a0c3cd7769d418926f9e9f140a60dca7c64ca9 \
     # via -r requirements/common.in, django-auth-ldap, django-bitfield, django-formtools, django-otp, django-phonenumber-field, django-sendfile2, django-two-factor-auth
 docker==4.4.0 \
     --hash=sha256:317e95a48c32de8c1aac92a48066a5b73e218ed096e03758bcdd799a7130a1a1 \

requirements/prod.txt

@@ -224,9 +224,9 @@ django-webpack4-loader==0.0.5 \
     --hash=sha256:baa043c4601ed763d161490e2888cf6aa93a2fd9b60681e6b19e35dc7fcb155d \
     --hash=sha256:be90257041170f39c025ff674c9064569c9303b464536488b6a6cedd8e3d28be \
     # via -r requirements/common.in
-django==3.0.11 \
+django==3.1.5 \
-    --hash=sha256:8c334df4160f7c89f6a8a359dd4e95c688ec5ac0db5db75fcc6fec8f590dc8cf \
+    --hash=sha256:2d78425ba74c7a1a74b196058b261b9733a8570782f4e2828974777ccca7edf7 \
-    --hash=sha256:96436d3d2f744d26e193bfb5a1cff3e01b349f835bb0ea16f71743accf9c6fa9 \
+    --hash=sha256:efa2ab96b33b20c2182db93147a0c3cd7769d418926f9e9f140a60dca7c64ca9 \
     # via -r requirements/common.in, django-auth-ldap, django-bitfield, django-formtools, django-otp, django-phonenumber-field, django-sendfile2, django-two-factor-auth
 future==0.18.2 \
     --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d \

version.py

@@ -43,4 +43,4 @@ API_FEATURE_LEVEL = 37
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '121.0'
+PROVISION_VERSION = '122.0'

zerver/lib/test_runner.py

@@ -7,8 +7,9 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union,
 from unittest import TestLoader, TestSuite, runner
 from unittest.result import TestResult
 
+import mock
 from django.conf import settings
-from django.db import ProgrammingError, connections
+from django.db import connections
 from django.test import TestCase
 from django.test import runner as django_runner
 from django.test.runner import DiscoverRunner
@@ -134,7 +135,23 @@ django_runner.multiprocessing = NoDaemonContext()
 def destroy_test_databases(worker_id: Optional[int]=None) -> None:
     for alias in connections:
         connection = connections[alias]
-        try:
+
+        def monkey_patched_destroy_test_db(test_database_name: str, verbosity: Any) -> None:
+            """
+            We need to monkey-patch connection.creation._destroy_test_db to
+            use the IF EXISTS parameter - we don't have a guarantee that the
+            database we're cleaning up actually exists and since Django 3.1 the original implementation
+            throws an ugly `RuntimeError: generator didn't stop after throw()` exception and triggers
+            a confusing warnings.warn inside the postgresql backend implementation in _nodb_cursor()
+            if the database doesn't exist.
+            https://code.djangoproject.com/ticket/32376
+            """
+            with connection.creation._nodb_cursor() as cursor:
+                quoted_name = connection.creation.connection.ops.quote_name(test_database_name)
+                query = f"DROP DATABASE IF EXISTS {quoted_name}"
+                cursor.execute(query)
+
+        with mock.patch.object(connection.creation, '_destroy_test_db', monkey_patched_destroy_test_db):
             # In the parallel mode, the test databases are created
             # through the N=self.parallel child processes, and in the
             # parent process (which calls `destroy_test_databases`),
@@ -156,9 +173,6 @@ def destroy_test_databases(worker_id: Optional[int]=None) -> None:
                 connection.creation.destroy_test_db(suffix=database_id)
             else:
                 connection.creation.destroy_test_db()
-        except ProgrammingError:
-            # DB doesn't exist. No need to do anything.
-            pass
 
 def create_test_databases(worker_id: int) -> None:
     database_id = get_database_id(worker_id)

zerver/migrations/0310_jsonfield.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.5 on 2021-01-10 11:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    """
+    This doesn't actually run any SQL, it's for Django's internal
+    tracking of changes to models only.
+    django.contrib.postgres.fields.JSONField is deprecated as of Django 3.1
+    and should be replaced by models.JSONField which offers the same functionality.
+    """
+
+    dependencies = [
+        ('zerver', '0309_userprofile_can_create_users'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userprofile',
+            name='zoom_token',
+            field=models.JSONField(default=None, null=True),
+        ),
+    ]

zerver/models.py

@@ -26,7 +26,6 @@ from bitfield import BitField
 from bitfield.types import BitHandler
 from django.conf import settings
 from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin, UserManager
-from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import ValidationError
 from django.core.validators import MinLengthValidator, RegexValidator, URLValidator, validate_email
 from django.db import models, transaction
@@ -1121,7 +1120,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin):
     # completed.
     onboarding_steps: str = models.TextField(default='[]')
 
-    zoom_token: Optional[object] = JSONField(default=None, null=True)
+    zoom_token: Optional[object] = models.JSONField(default=None, null=True)
 
     objects: UserManager = UserManager()
 

zproject/computed_settings.py

@@ -2,6 +2,7 @@ import os
 import sys
 import time
 from copy import deepcopy
+from pathlib import PosixPath
 from typing import Any, Dict, List, Tuple, Union
 from urllib.parse import urljoin
 
@@ -163,9 +164,9 @@ ALLOWED_HOSTS += REALM_HOSTS.values()
 
 
 class TwoFactorLoader(app_directories.Loader):
-    def get_dirs(self) -> List[str]:
+    def get_dirs(self) -> List[PosixPath]:
         dirs = super().get_dirs()
-        return [d for d in dirs if 'two_factor' in d]
+        return [d for d in dirs if d.match("two_factor/*")]
 
 MIDDLEWARE = (
     # With the exception of it's dependencies,