Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

import: Fix import and export parallelism defaults. 

Fixes #14960.
The default of 6 thread may not be appropriate in certain
configurations. Taking half of the numer of CPUs available to the
process will be more flexible.
This commit is contained in:
Mateusz Mandera 2020-06-21 13:18:08 +02:00 committed by Tim Abbott
parent c4ccf0d6b3
commit 60c4896c3a
6 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
zerver/management/commands/convert_gitter_data.py
View File

@ -3,6 +3,7 @@ import os
import tempfile
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError, CommandParser
from zerver.data_import.gitter import do_convert_data
@ -23,7 +24,7 @@ class Command(BaseCommand):
parser.add_argument('--threads',
dest='threads',
action="store",
default=6,
default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Threads to download avatars and attachments faster')
parser.formatter_class = argparse.RawTextHelpFormatter

3
zerver/management/commands/convert_slack_data.py
View File

@ -3,6 +3,7 @@ import os
import tempfile
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError, CommandParser
from zerver.data_import.slack import do_convert_data
@ -26,7 +27,7 @@ class Command(BaseCommand):
parser.add_argument('--threads',
dest='threads',
action="store",
default=6,
default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Threads to use in exporting UserMessage objects in parallel')
parser.formatter_class = argparse.RawTextHelpFormatter

3
zerver/management/commands/export.py
View File

@ -3,6 +3,7 @@ import tempfile
from argparse import ArgumentParser
from typing import Any
from django.conf import settings
from django.core.management.base import CommandError
from zerver.lib.export import export_realm_wrapper
@ -87,7 +88,7 @@ class Command(ZulipBaseCommand):
parser.add_argument('--threads',
dest='threads',
action="store",
default=6,
default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Threads to use in exporting UserMessage objects in parallel')
parser.add_argument('--public-only',
action="store_true",

2
zerver/management/commands/import.py
View File

@ -39,7 +39,7 @@ import a database dump from one or more JSON files."""
parser.add_argument('--processes',
dest='processes',
action="store",
default=6,
default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Number of processes to use for uploading Avatars to S3 in parallel')
parser.formatter_class = argparse.RawTextHelpFormatter

2
zerver/management/commands/transfer_uploads_to_s3.py
View File

@ -13,7 +13,7 @@ class Command(BaseCommand):
parser.add_argument('--processes',
dest='processes',
action="store",
default=6,
default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
help='Processes to use for exporting uploads in parallel')
def handle(self, *args: Any, **options: Any) -> None:

5
zproject/default_settings.py
View File

@ -11,6 +11,8 @@ if PRODUCTION:
else:
from .dev_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR
import os
DEBUG = DEVELOPMENT
# These settings are intended for the server admin to set. We document them in
@ -415,3 +417,6 @@ NAGIOS_BOT_HOST = EXTERNAL_HOST
# Automatically deactivate users not found by the AUTH_LDAP_USER_SEARCH query.
LDAP_DEACTIVATE_NON_MATCHING_USERS: Optional[bool] = None
# Use half of the available CPUs for data import purposes.
DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM = (len(os.sched_getaffinity(0)) // 2) or 1