From 143206795949d85545d1a76f634d23b5b780898f Mon Sep 17 00:00:00 2001 From: Mateusz Mandera Date: Tue, 12 Jan 2021 18:51:58 +0100 Subject: [PATCH] dependencies: Upgrade to Django 3.1. https://docs.djangoproject.com/en/3.1/releases/3.1/ - django.contrib.postgres.fields.JSONField is deprecated and should be replaced with models.JSONField - The internals of the implementation in the postgresql backend have changed a bit in https://github.com/django/django/commit/f48f671223a20b161ca819cf7d6298e43b8ba5fe and thus we need to make an ugly tweak in test_runner. - app_directories.Loader.get_dirs() now returns a list of PosixPath so we need to make a small tweak in TwoFactorLoader for that (PosixPath is not iterable) Fixes #16010. --- requirements/common.in | 2 +- requirements/dev.txt | 6 +++--- requirements/prod.txt | 6 +++--- version.py | 2 +- zerver/lib/test_runner.py | 24 +++++++++++++++++++----- zerver/migrations/0310_jsonfield.py | 24 ++++++++++++++++++++++++ zerver/models.py | 3 +-- zproject/computed_settings.py | 5 +++-- 8 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 zerver/migrations/0310_jsonfield.py diff --git a/requirements/common.in b/requirements/common.in index 676a89c4c8..419b0ddef4 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -3,7 +3,7 @@ # and requirements/prod.txt. # See requirements/README.md for more detail. # Django itself -Django==3.0.* +Django==3.1.* # needed for Literal, TypedDict typing-extensions diff --git a/requirements/dev.txt b/requirements/dev.txt index 44f8fb4953..a07263b50f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -316,9 +316,9 @@ django-webpack4-loader==0.0.5 \ --hash=sha256:baa043c4601ed763d161490e2888cf6aa93a2fd9b60681e6b19e35dc7fcb155d \ --hash=sha256:be90257041170f39c025ff674c9064569c9303b464536488b6a6cedd8e3d28be \ # via -r requirements/common.in -django==3.0.11 \ - --hash=sha256:8c334df4160f7c89f6a8a359dd4e95c688ec5ac0db5db75fcc6fec8f590dc8cf \ - --hash=sha256:96436d3d2f744d26e193bfb5a1cff3e01b349f835bb0ea16f71743accf9c6fa9 \ +django==3.1.5 \ + --hash=sha256:2d78425ba74c7a1a74b196058b261b9733a8570782f4e2828974777ccca7edf7 \ + --hash=sha256:efa2ab96b33b20c2182db93147a0c3cd7769d418926f9e9f140a60dca7c64ca9 \ # via -r requirements/common.in, django-auth-ldap, django-bitfield, django-formtools, django-otp, django-phonenumber-field, django-sendfile2, django-two-factor-auth docker==4.4.0 \ --hash=sha256:317e95a48c32de8c1aac92a48066a5b73e218ed096e03758bcdd799a7130a1a1 \ diff --git a/requirements/prod.txt b/requirements/prod.txt index 3dba39cd75..8813ab4292 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -224,9 +224,9 @@ django-webpack4-loader==0.0.5 \ --hash=sha256:baa043c4601ed763d161490e2888cf6aa93a2fd9b60681e6b19e35dc7fcb155d \ --hash=sha256:be90257041170f39c025ff674c9064569c9303b464536488b6a6cedd8e3d28be \ # via -r requirements/common.in -django==3.0.11 \ - --hash=sha256:8c334df4160f7c89f6a8a359dd4e95c688ec5ac0db5db75fcc6fec8f590dc8cf \ - --hash=sha256:96436d3d2f744d26e193bfb5a1cff3e01b349f835bb0ea16f71743accf9c6fa9 \ +django==3.1.5 \ + --hash=sha256:2d78425ba74c7a1a74b196058b261b9733a8570782f4e2828974777ccca7edf7 \ + --hash=sha256:efa2ab96b33b20c2182db93147a0c3cd7769d418926f9e9f140a60dca7c64ca9 \ # via -r requirements/common.in, django-auth-ldap, django-bitfield, django-formtools, django-otp, django-phonenumber-field, django-sendfile2, django-two-factor-auth future==0.18.2 \ --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d \ diff --git a/version.py b/version.py index d69b423242..98e4bbd419 100644 --- a/version.py +++ b/version.py @@ -43,4 +43,4 @@ API_FEATURE_LEVEL = 37 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '121.0' +PROVISION_VERSION = '122.0' diff --git a/zerver/lib/test_runner.py b/zerver/lib/test_runner.py index 91828b6320..bd8f6662de 100644 --- a/zerver/lib/test_runner.py +++ b/zerver/lib/test_runner.py @@ -7,8 +7,9 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, from unittest import TestLoader, TestSuite, runner from unittest.result import TestResult +import mock from django.conf import settings -from django.db import ProgrammingError, connections +from django.db import connections from django.test import TestCase from django.test import runner as django_runner from django.test.runner import DiscoverRunner @@ -134,7 +135,23 @@ django_runner.multiprocessing = NoDaemonContext() def destroy_test_databases(worker_id: Optional[int]=None) -> None: for alias in connections: connection = connections[alias] - try: + + def monkey_patched_destroy_test_db(test_database_name: str, verbosity: Any) -> None: + """ + We need to monkey-patch connection.creation._destroy_test_db to + use the IF EXISTS parameter - we don't have a guarantee that the + database we're cleaning up actually exists and since Django 3.1 the original implementation + throws an ugly `RuntimeError: generator didn't stop after throw()` exception and triggers + a confusing warnings.warn inside the postgresql backend implementation in _nodb_cursor() + if the database doesn't exist. + https://code.djangoproject.com/ticket/32376 + """ + with connection.creation._nodb_cursor() as cursor: + quoted_name = connection.creation.connection.ops.quote_name(test_database_name) + query = f"DROP DATABASE IF EXISTS {quoted_name}" + cursor.execute(query) + + with mock.patch.object(connection.creation, '_destroy_test_db', monkey_patched_destroy_test_db): # In the parallel mode, the test databases are created # through the N=self.parallel child processes, and in the # parent process (which calls `destroy_test_databases`), @@ -156,9 +173,6 @@ def destroy_test_databases(worker_id: Optional[int]=None) -> None: connection.creation.destroy_test_db(suffix=database_id) else: connection.creation.destroy_test_db() - except ProgrammingError: - # DB doesn't exist. No need to do anything. - pass def create_test_databases(worker_id: int) -> None: database_id = get_database_id(worker_id) diff --git a/zerver/migrations/0310_jsonfield.py b/zerver/migrations/0310_jsonfield.py new file mode 100644 index 0000000000..0bac90d158 --- /dev/null +++ b/zerver/migrations/0310_jsonfield.py @@ -0,0 +1,24 @@ +# Generated by Django 3.1.5 on 2021-01-10 11:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + """ + This doesn't actually run any SQL, it's for Django's internal + tracking of changes to models only. + django.contrib.postgres.fields.JSONField is deprecated as of Django 3.1 + and should be replaced by models.JSONField which offers the same functionality. + """ + + dependencies = [ + ('zerver', '0309_userprofile_can_create_users'), + ] + + operations = [ + migrations.AlterField( + model_name='userprofile', + name='zoom_token', + field=models.JSONField(default=None, null=True), + ), + ] diff --git a/zerver/models.py b/zerver/models.py index 14768a4c91..761a7edc94 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -26,7 +26,6 @@ from bitfield import BitField from bitfield.types import BitHandler from django.conf import settings from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin, UserManager -from django.contrib.postgres.fields import JSONField from django.core.exceptions import ValidationError from django.core.validators import MinLengthValidator, RegexValidator, URLValidator, validate_email from django.db import models, transaction @@ -1121,7 +1120,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin): # completed. onboarding_steps: str = models.TextField(default='[]') - zoom_token: Optional[object] = JSONField(default=None, null=True) + zoom_token: Optional[object] = models.JSONField(default=None, null=True) objects: UserManager = UserManager() diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 1ca0292ca0..e112d04c0a 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -2,6 +2,7 @@ import os import sys import time from copy import deepcopy +from pathlib import PosixPath from typing import Any, Dict, List, Tuple, Union from urllib.parse import urljoin @@ -163,9 +164,9 @@ ALLOWED_HOSTS += REALM_HOSTS.values() class TwoFactorLoader(app_directories.Loader): - def get_dirs(self) -> List[str]: + def get_dirs(self) -> List[PosixPath]: dirs = super().get_dirs() - return [d for d in dirs if 'two_factor' in d] + return [d for d in dirs if d.match("two_factor/*")] MIDDLEWARE = ( # With the exception of it's dependencies,