Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

models: Migrate ids of all non-Message-related tables to bigint. 

Migrate all `ids` of anything which does not have a foreign key from
the Message or UserMessage table (and would thus require walking
those) to be `bigint`.  This is done by removing explicit
`BigAutoField`s, trading them for explicit `AutoField`s on the tables
to not be migrated, while updating `DEFAULT_AUTO_FIELD` to the new
default.

In general, the tables adjusted in this commit are small tables -- at
least compared to Messages and UserMessages.

Many-to-many tables without their own model class are adjusted by a
custom Operation, since they do not automatically pick up migrations
when `DEFAULT_AUTO_FIELD` changes[^1].

Note that this does multiple scans over tables to update foreign
keys[^2].  Large installs may wish to hand-optimize this using the
output of `./manage.py sqlmigrate` to join multiple `ALTER TABLE`
statements into one, to speed up the migration.  This is unfortunately
not possible to do generically, as constraint names may differ between
installations.

This leaves the following primary keys as non-`bigint`:
- `auth_group.id`
- `auth_group_permissions.id`
- `auth_permission.id`
- `django_content_type.id`
- `django_migrations.id`
- `otp_static_staticdevice.id`
- `otp_static_statictoken.id`
- `otp_totp_totpdevice.id`
- `two_factor_phonedevice.id`
- `zerver_archivedmessage.id`
- `zerver_client.id`
- `zerver_message.id`
- `zerver_realm.id`
- `zerver_recipient.id`
- `zerver_userprofile.id`

[^1]: https://code.djangoproject.com/ticket/32674
[^2]: https://code.djangoproject.com/ticket/24203
This commit is contained in:
Alex Vandiver 2024-05-29 19:41:40 +00:00 committed by Tim Abbott
parent 04450389e1
commit 50c3dd88e6
15 changed files with 586 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
analytics/migrations/0021_alter_fillstate_id.py Normal file
View File

@ -0,0 +1,17 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
]
operations = [
migrations.AlterField(
model_name="fillstate",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
]

3
analytics/models.py
View File

@ -38,9 +38,6 @@ class BaseCount(models.Model):
# Note: When inheriting from BaseCount, you may want to rearrange # Note: When inheriting from BaseCount, you may want to rearrange
# the order of the columns in the migration to make sure they # the order of the columns in the migration to make sure they
# match how you'd like the table to be arranged. # match how you'd like the table to be arranged.
id = models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
)
property = models.CharField(max_length=32) property = models.CharField(max_length=32)
subgroup = models.CharField(max_length=16, null=True) subgroup = models.CharField(max_length=16, null=True)
end_time = models.DateTimeField() end_time = models.DateTimeField()

17
confirmation/migrations/0013_alter_realmcreationkey_id.py Normal file
View File

@ -0,0 +1,17 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("confirmation", "0012_alter_confirmation_id"),
]
operations = [
migrations.AlterField(
model_name="realmcreationkey",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
]

3
confirmation/models.py
View File

@ -207,9 +207,6 @@ def confirmation_url(
class Confirmation(models.Model): class Confirmation(models.Model):
id = models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
)
content_type = models.ForeignKey(ContentType, on_delete=CASCADE) content_type = models.ForeignKey(ContentType, on_delete=CASCADE)
object_id = models.PositiveIntegerField(db_index=True) object_id = models.PositiveIntegerField(db_index=True)
content_object = GenericForeignKey("content_type", "object_id") content_object = GenericForeignKey("content_type", "object_id")

75
corporate/migrations/0044_convert_ids_to_bigints.py Normal file
View File

@ -0,0 +1,75 @@
from django.db import migrations, models
class Migration(migrations.Migration):
atomic = False
dependencies = [
("corporate", "0043_remove_customer_default_discount_and_more"),
]
operations = [
migrations.AlterField(
model_name="customer",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="customerplan",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="customerplanoffer",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="event",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="invoice",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="licenseledger",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="paymentintent",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="session",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="zulipsponsorshiprequest",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
]

3
tools/semgrep-py.yml
View File

@ -151,6 +151,9 @@ rules:
- pattern: django.db.migrations.RunSQL(..., [..., ... + ..., ...], ...) - pattern: django.db.migrations.RunSQL(..., [..., ... + ..., ...], ...)
severity: ERROR severity: ERROR
message: "Do not write a SQL injection vulnerability please" message: "Do not write a SQL injection vulnerability please"
paths:
exclude:
- zerver/migrations/0531_convert_most_ids_to_bigints.py
- id: translated-format-lazy - id: translated-format-lazy
languages: [python] languages: [python]

391
zerver/migrations/0531_convert_most_ids_to_bigints.py Normal file
View File

@ -0,0 +1,391 @@
from typing import Any
from django.db import migrations, models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.operations.base import Operation
from typing_extensions import override
class MigrateIdToBigint(Operation):
"""A helper to migrate the id of a given table to bigint.
Necessary for many-to-many intermediate tables without models, due
to https://code.djangoproject.com/ticket/32674"""
def __init__(self, model_name: str) -> None:
self.model_name = model_name
@override
def state_forwards(self, app_label: str, state: Any) -> None:
pass
@override
def database_forwards(
self,
app_label: str,
schema_editor: BaseDatabaseSchemaEditor,
from_state: Any,
to_state: Any,
) -> None:
model = from_state.apps.get_model(app_label, self.model_name)
table_name = model._meta.db_table
seq_name = table_name + "_id_seq"
schema_editor.execute(
f"ALTER TABLE {schema_editor.quote_name(table_name)} ALTER COLUMN id SET DATA TYPE bigint"
)
schema_editor.execute(f"ALTER SEQUENCE {schema_editor.quote_name(seq_name)} AS bigint")
@override
def database_backwards(
self,
app_label: str,
schema_editor: BaseDatabaseSchemaEditor,
from_state: Any,
to_state: Any,
) -> None:
model = from_state.apps.get_model(app_label, self.model_name)
table_name = model._meta.db_table
seq_name = table_name + "_id_seq"
schema_editor.execute(
f"ALTER TABLE {schema_editor.quote_name(table_name)} ALTER COLUMN id SET DATA TYPE int"
)
schema_editor.execute(f"ALTER SEQUENCE {schema_editor.quote_name(seq_name)} AS int")
@override
def describe(self) -> str:
return f"Migrates the 'id' column of {self.model_name} and its sequence to be a bigint. Requires that the table have no foreign keys."
class Migration(migrations.Migration):
atomic = False
dependencies = [
("zerver", "0530_alter_useractivity_id_alter_useractivityinterval_id"),
]
operations = [
MigrateIdToBigint("archivedattachment_messages"),
MigrateIdToBigint("attachment_messages"),
MigrateIdToBigint("attachment_scheduled_messages"),
MigrateIdToBigint("defaultstreamgroup_streams"),
MigrateIdToBigint("multiuseinvite_streams"),
MigrateIdToBigint("preregistrationuser_streams"),
MigrateIdToBigint("scheduledemail_users"),
MigrateIdToBigint("userprofile_groups"),
MigrateIdToBigint("userprofile_user_permissions"),
migrations.AlterField(
model_name="alertword",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="archivedattachment",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="archivedreaction",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="archivedsubmessage",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="archivetransaction",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="attachment",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="botconfigdata",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="botstoragedata",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="customprofilefield",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="customprofilefieldvalue",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="defaultstream",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="defaultstreamgroup",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="draft",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="emailchangestatus",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="groupgroupmembership",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="huddle",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="missedmessageemailaddress",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="multiuseinvite",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="muteduser",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="onboardingstep",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="preregistrationrealm",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="preregistrationuser",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="presencesequence",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="pushdevicetoken",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="reaction",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmauditlog",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmauthenticationmethod",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmdomain",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmemoji",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmfilter",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmplayground",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmreactivationstatus",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="realmuserdefault",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="scheduledemail",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="scheduledmessage",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="scheduledmessagenotificationemail",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="service",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="stream",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="submessage",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="subscription",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="usergroup",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="usergroupmembership",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="userpresence",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="userstatus",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="usertopic",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
]

1
zerver/models/clients.py
View File

@ -11,6 +11,7 @@ from zerver.lib.cache import cache_with_key
class Client(models.Model): class Client(models.Model):
MAX_NAME_LENGTH = 30 MAX_NAME_LENGTH = 30
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True, unique=True) name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True, unique=True)
@override @override

1
zerver/models/messages.py
View File

@ -26,6 +26,7 @@ from zerver.models.users import UserProfile
class AbstractMessage(models.Model): class AbstractMessage(models.Model):
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
sender = models.ForeignKey(UserProfile, on_delete=CASCADE) sender = models.ForeignKey(UserProfile, on_delete=CASCADE)
# The target of the message is signified by the Recipient object. # The target of the message is signified by the Recipient object.

2
zerver/models/realms.py
View File

@ -192,6 +192,8 @@ class Realm(models.Model): # type: ignore[django-manager-missing] # django-stub
SUBDOMAIN_FOR_ROOT_DOMAIN = "" SUBDOMAIN_FOR_ROOT_DOMAIN = ""
WILDCARD_MENTION_THRESHOLD = 15 WILDCARD_MENTION_THRESHOLD = 15
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
# User-visible display name and description used on e.g. the organization homepage # User-visible display name and description used on e.g. the organization homepage
name = models.CharField(max_length=MAX_REALM_NAME_LENGTH) name = models.CharField(max_length=MAX_REALM_NAME_LENGTH)
description = models.TextField(default="") description = models.TextField(default="")

1
zerver/models/recipients.py
View File

@ -38,6 +38,7 @@ class Recipient(models.Model):
objects are subscribed to which Recipient objects. objects are subscribed to which Recipient objects.
""" """
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
type_id = models.IntegerField(db_index=True) type_id = models.IntegerField(db_index=True)
type = models.PositiveSmallIntegerField(db_index=True) type = models.PositiveSmallIntegerField(db_index=True)
# Valid types are {personal, stream, huddle} # Valid types are {personal, stream, huddle}

6
zerver/models/user_activity.py
View File

@ -19,9 +19,6 @@ class UserActivity(models.Model):
and database migration purposes. and database migration purposes.
""" """
id = models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
)
user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
client = models.ForeignKey(Client, on_delete=CASCADE) client = models.ForeignKey(Client, on_delete=CASCADE)
query = models.CharField(max_length=50, db_index=True) query = models.CharField(max_length=50, db_index=True)
@ -36,9 +33,6 @@ class UserActivity(models.Model):
class UserActivityInterval(models.Model): class UserActivityInterval(models.Model):
MIN_INTERVAL_LENGTH = timedelta(minutes=15) MIN_INTERVAL_LENGTH = timedelta(minutes=15)
id = models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
)
user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
start = models.DateTimeField("start time", db_index=True) start = models.DateTimeField("start time", db_index=True)
end = models.DateTimeField("end time", db_index=True) end = models.DateTimeField("end time", db_index=True)

2
zerver/models/users.py
View File

@ -408,6 +408,8 @@ class UserProfile(AbstractBaseUser, PermissionsMixin, UserBaseSettings):
EMBEDDED_BOT, EMBEDDED_BOT,
] ]
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
# For historical reasons, Zulip has two email fields. The # For historical reasons, Zulip has two email fields. The
# `delivery_email` field is the user's email address, where all # `delivery_email` field is the user's email address, where all
# email notifications will be sent, and is used for all # email notifications will be sent, and is used for all

75
zilencer/migrations/0063_convert_ids_to_bigints.py Normal file
View File

@ -0,0 +1,75 @@
from django.db import migrations, models
class Migration(migrations.Migration):
atomic = False
dependencies = [
("zilencer", "0062_alter_remoteinstallationcount_id_and_more"),
]
operations = [
migrations.AlterField(
model_name="preregistrationremoterealmbillinguser",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="preregistrationremoteserverbillinguser",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remotepushdevicetoken",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remoterealm",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remoterealmauditlog",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remoterealmbillinguser",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remoteserverbillinguser",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remotezulipserver",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="remotezulipserverauditlog",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
]

2
zproject/computed_settings.py
View File

@ -315,7 +315,7 @@ elif (
) )
POSTGRESQL_MISSING_DICTIONARIES = get_config("postgresql", "missing_dictionaries", False) POSTGRESQL_MISSING_DICTIONARIES = get_config("postgresql", "missing_dictionaries", False)
DEFAULT_AUTO_FIELD = "django.db.models.AutoField" DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
######################################################################## ########################################################################
# RABBITMQ CONFIGURATION # RABBITMQ CONFIGURATION