diff --git a/requirements/common.in b/requirements/common.in index 53688318a4..54ef4a3498 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -75,6 +75,9 @@ html2text==2018.1.9 httplib2==0.12.0 -e git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1 +# Needed for hipchat import +hypchat==0.21 + # Needed for inlining the CSS in emails premailer==3.2.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 63d87e4672..5a365ca127 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -74,6 +74,7 @@ hpack==3.0.0 # via h2 html2text==2018.1.9 httplib2==0.12.0 httpretty==0.9.6 +hypchat==0.21 hyper==0.7.0 # via apns2 hyperframe==3.2.0 # via h2, hyper hyperlink==18.0.0 # via twisted @@ -150,7 +151,7 @@ recommonmark==0.4.0 redis==2.10.6 regex==2018.11.22 requests-oauthlib==1.0.0 -requests[security]==2.21.0 # via aws-xray-sdk, docker, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio +requests[security]==2.21.0 # via aws-xray-sdk, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio responses==0.10.5 # via moto rsa==4.0 s3transfer==0.1.13 # via boto3 diff --git a/requirements/prod.txt b/requirements/prod.txt index da0701980f..81e139ee48 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -55,6 +55,7 @@ h2==2.6.2 # via hyper hpack==3.0.0 # via h2 html2text==2018.1.9 httplib2==0.12.0 +hypchat==0.21 hyper==0.7.0 # via apns2 hyperframe==3.2.0 # via h2, hyper idna==2.8 # via cryptography, requests @@ -107,7 +108,7 @@ qrcode==6.0 # via django-two-factor-auth redis==2.10.6 regex==2018.11.22 requests-oauthlib==1.0.0 -requests[security]==2.21.0 # via matrix-client, premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core, stripe, twilio +requests[security]==2.21.0 # via hypchat, matrix-client, premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core, stripe, twilio rsa==4.0 simplegeneric==0.8.1 # via ipython six==1.12.0 diff --git a/templates/zerver/help/import-from-hipchat.md b/templates/zerver/help/import-from-hipchat.md index c541c9f057..81e168b4d6 100644 --- a/templates/zerver/help/import-from-hipchat.md +++ b/templates/zerver/help/import-from-hipchat.md @@ -69,6 +69,10 @@ Email support@zulipchat.com with exported HipChat archive and your desired subdomain. Your imported organization will be hosted at `.zulipchat.com`. +Also, see the [caveats section notes on room subscribers](#caveats) +and consider whether you want to also send a HipChat API key to +provide a more faithful import. + If you've already created a test organization at `.zulipchat.com`, let us know, and we can rename the old organization first. @@ -113,3 +117,28 @@ root domain. Replace the last line above with the following, after replacing {!import-login.md!} [upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository + +## Caveats + +- While the import tool will correctly import the subscribers of +private rooms precisely, HipChat does not store the subscribers of +public rooms when those users don't have an active client. As a +result, HipChat's data exports don't include subscribers for public +rooms. You can pick one of the following options for handling this: + 1. Subscribe all users to all public streams (the default, which is + good for small organizations), + 2. Subscribe only HipChat room owners to public streams (and plan + for users to subscribe to the imported Zulip streams manually + after the import completes) using the `--slim-mode` option to `manage.py + convert_hipchat_data`, or + 3. Use the [HipChat API][hipchat-api-tokens] to fetch each room's + current room subscribers as of the moment the import is run. + Because HipChat doesn't store subscribers to a room when clients + are not connected, these subscriptons will be incomplete for users + who don't have an actively connected client at the time of the + import. You need to pass the token via `--token=abcd1234` in + `manage.py convert_hipchat_data` (or include it in your request, + if importing into Zulip Cloud). + +[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository +[hipchat-api-tokens]: https://developer.atlassian.com/server/hipchat/hipchat-rest-api-access-tokens/ diff --git a/zerver/data_import/hipchat.py b/zerver/data_import/hipchat.py index 65bf936973..8eba7a5bd2 100755 --- a/zerver/data_import/hipchat.py +++ b/zerver/data_import/hipchat.py @@ -1,6 +1,7 @@ import base64 import dateutil import glob +import hypchat import logging import os import re @@ -207,7 +208,8 @@ def convert_room_data(raw_data: List[ZerverFieldsT], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, - realm_id: int) -> List[ZerverFieldsT]: + realm_id: int, + api_token: Optional[str]=None) -> List[ZerverFieldsT]: flat_data = [ d['Room'] for d in raw_data @@ -249,10 +251,18 @@ def convert_room_data(raw_data: List[ZerverFieldsT], if user_id_mapper.has(in_dict['owner']): owner = user_id_mapper.get(in_dict['owner']) users.add(owner) + else: + users = set() + if api_token is not None: + hc = hypchat.HypChat(api_token) + room_data = hc.fromurl('{0}/v2/room/{1}/member'.format(hc.endpoint, in_dict['id'])) - if not users: - continue + for item in room_data['items']: + hipchat_user_id = item['id'] + zulip_user_id = user_id_mapper.get(hipchat_user_id) + users.add(zulip_user_id) + if users: subscriber_handler.set_info( stream_id=stream_id, users=users, @@ -768,7 +778,9 @@ def make_user_messages(zerver_message: List[ZerverFieldsT], def do_convert_data(input_tar_file: str, output_dir: str, - masking_content: bool) -> None: + masking_content: bool, + api_token: Optional[str]=None, + slim_mode: bool=False) -> None: input_data_dir = untar_input_file(input_tar_file) attachment_handler = AttachmentHandler() @@ -780,8 +792,6 @@ def do_convert_data(input_tar_file: str, realm_id = 0 realm = make_realm(realm_id=realm_id) - slim_mode = False - # users.json -> UserProfile raw_user_data = read_user_data(data_dir=input_data_dir) convert_user_data( @@ -803,6 +813,7 @@ def do_convert_data(input_tar_file: str, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, + api_token=api_token, ) realm['zerver_stream'] = zerver_stream @@ -812,7 +823,7 @@ def do_convert_data(input_tar_file: str, ) realm['zerver_recipient'] = zerver_recipient - if True: + if api_token is None: if slim_mode: public_stream_subscriptions = [] # type: List[ZerverFieldsT] else: @@ -829,6 +840,12 @@ def do_convert_data(input_tar_file: str, if stream_dict['invite_only']], ) stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions + else: + stream_subscriptions = build_stream_subscriptions( + get_users=subscriber_handler.get_users, + zerver_recipient=zerver_recipient, + zerver_stream=zerver_stream, + ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, diff --git a/zerver/management/commands/convert_hipchat_data.py b/zerver/management/commands/convert_hipchat_data.py index 7cb9c09d77..8f2cc56345 100644 --- a/zerver/management/commands/convert_hipchat_data.py +++ b/zerver/management/commands/convert_hipchat_data.py @@ -44,6 +44,14 @@ class Command(BaseCommand): action="store_true", help='Mask the content for privacy during QA.') + parser.add_argument('--slim-mode', dest='slim_mode', + action="store_true", + help='Mask the content for privacy during QA.') + + parser.add_argument('--token', dest='api_token', + action="store", + help='API token for the HipChat API for fetching subscribers.') + parser.formatter_class = argparse.RawTextHelpFormatter def handle(self, *args: Any, **options: Any) -> None: @@ -75,4 +83,6 @@ class Command(BaseCommand): input_tar_file=path, output_dir=output_dir, masking_content=options.get('masking_content', False), + slim_mode=options['slim_mode'], + api_token=options.get("api_token"), )