Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

import: Fix import and export parallelism defaults. 

Fixes #14960.
The default of 6 thread may not be appropriate in certain
configurations. Taking half of the numer of CPUs available to the
process will be more flexible.
This commit is contained in:
Mateusz Mandera 2020-06-21 13:18:08 +02:00 committed by Tim Abbott
parent c4ccf0d6b3
commit 60c4896c3a
6 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
zerver/management/commands/convert_gitter_data.py
View File

@ -3,6 +3,7 @@ import os
import tempfile import tempfile
from typing import Any from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError, CommandParser from django.core.management.base import BaseCommand, CommandError, CommandParser
from zerver.data_import.gitter import do_convert_data from zerver.data_import.gitter import do_convert_data
@ -23,7 +24,7 @@ class Command(BaseCommand):
parser.add_argument('--threads', parser.add_argument('--threads',
dest='threads', dest='threads',
action="store", action="store",
default=6, default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Threads to download avatars and attachments faster') help='Threads to download avatars and attachments faster')
parser.formatter_class = argparse.RawTextHelpFormatter parser.formatter_class = argparse.RawTextHelpFormatter

3
zerver/management/commands/convert_slack_data.py
View File

@ -3,6 +3,7 @@ import os
import tempfile import tempfile
from typing import Any from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError, CommandParser from django.core.management.base import BaseCommand, CommandError, CommandParser
from zerver.data_import.slack import do_convert_data from zerver.data_import.slack import do_convert_data
@ -26,7 +27,7 @@ class Command(BaseCommand):
parser.add_argument('--threads', parser.add_argument('--threads',
dest='threads', dest='threads',
action="store", action="store",
default=6, default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Threads to use in exporting UserMessage objects in parallel') help='Threads to use in exporting UserMessage objects in parallel')
parser.formatter_class = argparse.RawTextHelpFormatter parser.formatter_class = argparse.RawTextHelpFormatter

3
zerver/management/commands/export.py
View File

@ -3,6 +3,7 @@ import tempfile
from argparse import ArgumentParser from argparse import ArgumentParser
from typing import Any from typing import Any
from django.conf import settings
from django.core.management.base import CommandError from django.core.management.base import CommandError
from zerver.lib.export import export_realm_wrapper from zerver.lib.export import export_realm_wrapper
@ -87,7 +88,7 @@ class Command(ZulipBaseCommand):
parser.add_argument('--threads', parser.add_argument('--threads',
dest='threads', dest='threads',
action="store", action="store",
default=6, default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Threads to use in exporting UserMessage objects in parallel') help='Threads to use in exporting UserMessage objects in parallel')
parser.add_argument('--public-only', parser.add_argument('--public-only',
action="store_true", action="store_true",

2
zerver/management/commands/import.py
View File

@ -39,7 +39,7 @@ import a database dump from one or more JSON files."""
parser.add_argument('--processes', parser.add_argument('--processes',
dest='processes', dest='processes',
action="store", action="store",
default=6, default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Number of processes to use for uploading Avatars to S3 in parallel') help='Number of processes to use for uploading Avatars to S3 in parallel')
parser.formatter_class = argparse.RawTextHelpFormatter parser.formatter_class = argparse.RawTextHelpFormatter

2
zerver/management/commands/transfer_uploads_to_s3.py
View File

@ -13,7 +13,7 @@ class Command(BaseCommand):
parser.add_argument('--processes', parser.add_argument('--processes',
dest='processes', dest='processes',
action="store", action="store",
default=6, default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Processes to use for exporting uploads in parallel') help='Processes to use for exporting uploads in parallel')
def handle(self, *args: Any, **options: Any) -> None: def handle(self, *args: Any, **options: Any) -> None:

5
zproject/default_settings.py
View File

@ -11,6 +11,8 @@ if PRODUCTION:
else: else:
from .dev_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR from .dev_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR
import os
DEBUG = DEVELOPMENT DEBUG = DEVELOPMENT
# These settings are intended for the server admin to set. We document them in # These settings are intended for the server admin to set. We document them in
@ -415,3 +417,6 @@ NAGIOS_BOT_HOST = EXTERNAL_HOST
# Automatically deactivate users not found by the AUTH_LDAP_USER_SEARCH query. # Automatically deactivate users not found by the AUTH_LDAP_USER_SEARCH query.
LDAP_DEACTIVATE_NON_MATCHING_USERS: Optional[bool] = None LDAP_DEACTIVATE_NON_MATCHING_USERS: Optional[bool] = None
# Use half of the available CPUs for data import purposes.
DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM = (len(os.sched_getaffinity(0)) // 2) or 1