hipchat: Improve import of public room subscribers.

Now, if you pass an api_key, we'll initialize the public room
subscribers to be whatever they were at the time the import happened.

Also, document the situation on the caveats section.

requirements/common.in

@@ -75,6 +75,9 @@ html2text==2018.1.9
 httplib2==0.12.0
 -e git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1
 
+# Needed for hipchat import
+hypchat==0.21
+
 # Needed for inlining the CSS in emails
 premailer==3.2.0
 

requirements/dev.txt

@@ -74,6 +74,7 @@ hpack==3.0.0              # via h2
 html2text==2018.1.9
 httplib2==0.12.0
 httpretty==0.9.6
+hypchat==0.21
 hyper==0.7.0              # via apns2
 hyperframe==3.2.0         # via h2, hyper
 hyperlink==18.0.0         # via twisted
@@ -150,7 +151,7 @@ recommonmark==0.4.0
 redis==2.10.6
 regex==2018.11.22
 requests-oauthlib==1.0.0
-requests[security]==2.21.0  # via aws-xray-sdk, docker, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio
+requests[security]==2.21.0  # via aws-xray-sdk, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio
 responses==0.10.5         # via moto
 rsa==4.0
 s3transfer==0.1.13        # via boto3

requirements/prod.txt

@@ -55,6 +55,7 @@ h2==2.6.2                 # via hyper
 hpack==3.0.0              # via h2
 html2text==2018.1.9
 httplib2==0.12.0
+hypchat==0.21
 hyper==0.7.0              # via apns2
 hyperframe==3.2.0         # via h2, hyper
 idna==2.8                 # via cryptography, requests
@@ -107,7 +108,7 @@ qrcode==6.0               # via django-two-factor-auth
 redis==2.10.6
 regex==2018.11.22
 requests-oauthlib==1.0.0
-requests[security]==2.21.0  # via matrix-client, premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core, stripe, twilio
+requests[security]==2.21.0  # via hypchat, matrix-client, premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core, stripe, twilio
 rsa==4.0
 simplegeneric==0.8.1      # via ipython
 six==1.12.0

templates/zerver/help/import-from-hipchat.md

@@ -69,6 +69,10 @@ Email support@zulipchat.com with exported HipChat archive and your desired
 subdomain. Your imported organization will be hosted at
 `<subdomain>.zulipchat.com`.
 
+Also, see the [caveats section notes on room subscribers](#caveats)
+and consider whether you want to also send a HipChat API key to
+provide a more faithful import.
+
 If you've already created a test organization at
 `<subdomain>.zulipchat.com`, let us know, and we can rename the old
 organization first.
@@ -113,3 +117,28 @@ root domain. Replace the last line above with the following, after replacing
 {!import-login.md!}
 
 [upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository
+
+## Caveats
+
+- While the import tool will correctly import the subscribers of
+private rooms precisely, HipChat does not store the subscribers of
+public rooms when those users don't have an active client.  As a
+result, HipChat's data exports don't include subscribers for public
+rooms.  You can pick one of the following options for handling this:
+    1. Subscribe all users to all public streams (the default, which is
+    good for small organizations),
+    2. Subscribe only HipChat room owners to public streams (and plan
+    for users to subscribe to the imported Zulip streams manually
+    after the import completes) using the `--slim-mode` option to `manage.py
+    convert_hipchat_data`, or
+    3. Use the [HipChat API][hipchat-api-tokens] to fetch each room's
+    current room subscribers as of the moment the import is run.
+    Because HipChat doesn't store subscribers to a room when clients
+    are not connected, these subscriptons will be incomplete for users
+    who don't have an actively connected client at the time of the
+    import.  You need to pass the token via `--token=abcd1234` in
+    `manage.py convert_hipchat_data` (or include it in your request,
+    if importing into Zulip Cloud).
+
+[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository
+[hipchat-api-tokens]: https://developer.atlassian.com/server/hipchat/hipchat-rest-api-access-tokens/

zerver/data_import/hipchat.py

@@ -1,6 +1,7 @@
 import base64
 import dateutil
 import glob
+import hypchat
 import logging
 import os
 import re
@@ -207,7 +208,8 @@ def convert_room_data(raw_data: List[ZerverFieldsT],
                       subscriber_handler: SubscriberHandler,
                       stream_id_mapper: IdMapper,
                       user_id_mapper: IdMapper,
-                      realm_id: int) -> List[ZerverFieldsT]:
+                      realm_id: int,
+                      api_token: Optional[str]=None) -> List[ZerverFieldsT]:
     flat_data = [
         d['Room']
         for d in raw_data
@@ -249,10 +251,18 @@ def convert_room_data(raw_data: List[ZerverFieldsT],
             if user_id_mapper.has(in_dict['owner']):
                 owner = user_id_mapper.get(in_dict['owner'])
                 users.add(owner)
+        else:
+            users = set()
+            if api_token is not None:
+                hc = hypchat.HypChat(api_token)
+                room_data = hc.fromurl('{0}/v2/room/{1}/member'.format(hc.endpoint, in_dict['id']))
 
-            if not users:
+                for item in room_data['items']:
-                continue
+                    hipchat_user_id = item['id']
+                    zulip_user_id = user_id_mapper.get(hipchat_user_id)
+                    users.add(zulip_user_id)
 
+        if users:
             subscriber_handler.set_info(
                 stream_id=stream_id,
                 users=users,
@@ -768,7 +778,9 @@ def make_user_messages(zerver_message: List[ZerverFieldsT],
 
 def do_convert_data(input_tar_file: str,
                     output_dir: str,
-                    masking_content: bool) -> None:
+                    masking_content: bool,
+                    api_token: Optional[str]=None,
+                    slim_mode: bool=False) -> None:
     input_data_dir = untar_input_file(input_tar_file)
 
     attachment_handler = AttachmentHandler()
@@ -780,8 +792,6 @@ def do_convert_data(input_tar_file: str,
     realm_id = 0
     realm = make_realm(realm_id=realm_id)
 
-    slim_mode = False
-
     # users.json -> UserProfile
     raw_user_data = read_user_data(data_dir=input_data_dir)
     convert_user_data(
@@ -803,6 +813,7 @@ def do_convert_data(input_tar_file: str,
         stream_id_mapper=stream_id_mapper,
         user_id_mapper=user_id_mapper,
         realm_id=realm_id,
+        api_token=api_token,
     )
     realm['zerver_stream'] = zerver_stream
 
@@ -812,7 +823,7 @@ def do_convert_data(input_tar_file: str,
     )
     realm['zerver_recipient'] = zerver_recipient
 
-    if True:
+    if api_token is None:
         if slim_mode:
             public_stream_subscriptions = []  # type: List[ZerverFieldsT]
         else:
@@ -829,6 +840,12 @@ def do_convert_data(input_tar_file: str,
                            if stream_dict['invite_only']],
         )
         stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions
+    else:
+        stream_subscriptions = build_stream_subscriptions(
+            get_users=subscriber_handler.get_users,
+            zerver_recipient=zerver_recipient,
+            zerver_stream=zerver_stream,
+        )
 
     personal_subscriptions = build_personal_subscriptions(
         zerver_recipient=zerver_recipient,

zerver/management/commands/convert_hipchat_data.py

@@ -44,6 +44,14 @@ class Command(BaseCommand):
                             action="store_true",
                             help='Mask the content for privacy during QA.')
 
+        parser.add_argument('--slim-mode', dest='slim_mode',
+                            action="store_true",
+                            help='Mask the content for privacy during QA.')
+
+        parser.add_argument('--token', dest='api_token',
+                            action="store",
+                            help='API token for the HipChat API for fetching subscribers.')
+
         parser.formatter_class = argparse.RawTextHelpFormatter
 
     def handle(self, *args: Any, **options: Any) -> None:
@@ -75,4 +83,6 @@ class Command(BaseCommand):
                 input_tar_file=path,
                 output_dir=output_dir,
                 masking_content=options.get('masking_content', False),
+                slim_mode=options['slim_mode'],
+                api_token=options.get("api_token"),
             )