diff --git a/zerver/management/commands/convert_gitter_data.py b/zerver/management/commands/convert_gitter_data.py index e18bd893b9..c6c8d7406b 100644 --- a/zerver/management/commands/convert_gitter_data.py +++ b/zerver/management/commands/convert_gitter_data.py @@ -3,6 +3,7 @@ import os import tempfile from typing import Any +from django.conf import settings from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.gitter import do_convert_data @@ -23,7 +24,7 @@ class Command(BaseCommand): parser.add_argument('--threads', dest='threads', action="store", - default=6, + default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM, help='Threads to download avatars and attachments faster') parser.formatter_class = argparse.RawTextHelpFormatter diff --git a/zerver/management/commands/convert_slack_data.py b/zerver/management/commands/convert_slack_data.py index 5168146d14..dbdb0e0e08 100644 --- a/zerver/management/commands/convert_slack_data.py +++ b/zerver/management/commands/convert_slack_data.py @@ -3,6 +3,7 @@ import os import tempfile from typing import Any +from django.conf import settings from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.slack import do_convert_data @@ -26,7 +27,7 @@ class Command(BaseCommand): parser.add_argument('--threads', dest='threads', action="store", - default=6, + default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM, help='Threads to use in exporting UserMessage objects in parallel') parser.formatter_class = argparse.RawTextHelpFormatter diff --git a/zerver/management/commands/export.py b/zerver/management/commands/export.py index a3a0b5cca9..57ded815ab 100644 --- a/zerver/management/commands/export.py +++ b/zerver/management/commands/export.py @@ -3,6 +3,7 @@ import tempfile from argparse import ArgumentParser from typing import Any +from django.conf import settings from django.core.management.base import CommandError from zerver.lib.export import export_realm_wrapper @@ -87,7 +88,7 @@ class Command(ZulipBaseCommand): parser.add_argument('--threads', dest='threads', action="store", - default=6, + default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM, help='Threads to use in exporting UserMessage objects in parallel') parser.add_argument('--public-only', action="store_true", diff --git a/zerver/management/commands/import.py b/zerver/management/commands/import.py index 92282013c8..fbe68e0f4a 100644 --- a/zerver/management/commands/import.py +++ b/zerver/management/commands/import.py @@ -39,7 +39,7 @@ import a database dump from one or more JSON files.""" parser.add_argument('--processes', dest='processes', action="store", - default=6, + default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM, help='Number of processes to use for uploading Avatars to S3 in parallel') parser.formatter_class = argparse.RawTextHelpFormatter diff --git a/zerver/management/commands/transfer_uploads_to_s3.py b/zerver/management/commands/transfer_uploads_to_s3.py index 77ade9cf48..c646513723 100644 --- a/zerver/management/commands/transfer_uploads_to_s3.py +++ b/zerver/management/commands/transfer_uploads_to_s3.py @@ -13,7 +13,7 @@ class Command(BaseCommand): parser.add_argument('--processes', dest='processes', action="store", - default=6, + default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM, help='Processes to use for exporting uploads in parallel') def handle(self, *args: Any, **options: Any) -> None: diff --git a/zproject/default_settings.py b/zproject/default_settings.py index 104b1426f2..979f3323ef 100644 --- a/zproject/default_settings.py +++ b/zproject/default_settings.py @@ -11,6 +11,8 @@ if PRODUCTION: else: from .dev_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR +import os + DEBUG = DEVELOPMENT # These settings are intended for the server admin to set. We document them in @@ -415,3 +417,6 @@ NAGIOS_BOT_HOST = EXTERNAL_HOST # Automatically deactivate users not found by the AUTH_LDAP_USER_SEARCH query. LDAP_DEACTIVATE_NON_MATCHING_USERS: Optional[bool] = None + +# Use half of the available CPUs for data import purposes. +DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM = (len(os.sched_getaffinity(0)) // 2) or 1