Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

hipchat: Improve import of public room subscribers. 

Now, if you pass an api_key, we'll initialize the public room
subscribers to be whatever they were at the time the import happened.

Also, document the situation on the caveats section.
This commit is contained in:
Tim Abbott 2019-01-09 15:42:38 -08:00
parent 035138dd98
commit 53436766c1
6 changed files with 70 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
requirements/common.in
View File

@ -75,6 +75,9 @@ html2text==2018.1.9
httplib2==0.12.0
-e git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1
# Needed for hipchat import
hypchat==0.21
# Needed for inlining the CSS in emails
premailer==3.2.0

3
requirements/dev.txt
View File

@ -74,6 +74,7 @@ hpack==3.0.0 # via h2
html2text==2018.1.9
httplib2==0.12.0
httpretty==0.9.6
hypchat==0.21
hyper==0.7.0 # via apns2
hyperframe==3.2.0 # via h2, hyper
hyperlink==18.0.0 # via twisted
@ -150,7 +151,7 @@ recommonmark==0.4.0
redis==2.10.6
regex==2018.11.22
requests-oauthlib==1.0.0
requests[security]==2.21.0 # via aws-xray-sdk, docker, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio
requests[security]==2.21.0 # via aws-xray-sdk, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio
responses==0.10.5 # via moto
rsa==4.0
s3transfer==0.1.13 # via boto3

3
requirements/prod.txt
View File

@ -55,6 +55,7 @@ h2==2.6.2 # via hyper
hpack==3.0.0 # via h2
html2text==2018.1.9
httplib2==0.12.0
hypchat==0.21
hyper==0.7.0 # via apns2
hyperframe==3.2.0 # via h2, hyper
idna==2.8 # via cryptography, requests
@ -107,7 +108,7 @@ qrcode==6.0 # via django-two-factor-auth
redis==2.10.6
regex==2018.11.22
requests-oauthlib==1.0.0
requests[security]==2.21.0 # via matrix-client, premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core, stripe, twilio
requests[security]==2.21.0 # via hypchat, matrix-client, premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core, stripe, twilio
rsa==4.0
simplegeneric==0.8.1 # via ipython
six==1.12.0

29
templates/zerver/help/import-from-hipchat.md
View File

@ -69,6 +69,10 @@ Email support@zulipchat.com with exported HipChat archive and your desired
subdomain. Your imported organization will be hosted at
`<subdomain>.zulipchat.com`.
Also, see the [caveats section notes on room subscribers](#caveats)
and consider whether you want to also send a HipChat API key to
provide a more faithful import.
If you've already created a test organization at
`<subdomain>.zulipchat.com`, let us know, and we can rename the old
organization first.
@ -113,3 +117,28 @@ root domain. Replace the last line above with the following, after replacing
{!import-login.md!}
[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository
## Caveats
- While the import tool will correctly import the subscribers of
private rooms precisely, HipChat does not store the subscribers of
public rooms when those users don't have an active client. As a
result, HipChat's data exports don't include subscribers for public
rooms. You can pick one of the following options for handling this:
1. Subscribe all users to all public streams (the default, which is
good for small organizations),
2. Subscribe only HipChat room owners to public streams (and plan
for users to subscribe to the imported Zulip streams manually
after the import completes) using the `--slim-mode` option to `manage.py
convert_hipchat_data`, or
3. Use the [HipChat API][hipchat-api-tokens] to fetch each room's
current room subscribers as of the moment the import is run.
Because HipChat doesn't store subscribers to a room when clients
are not connected, these subscriptons will be incomplete for users
who don't have an actively connected client at the time of the
import. You need to pass the token via `--token=abcd1234` in
`manage.py convert_hipchat_data` (or include it in your request,
if importing into Zulip Cloud).
[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository
[hipchat-api-tokens]: https://developer.atlassian.com/server/hipchat/hipchat-rest-api-access-tokens/

31
zerver/data_import/hipchat.py
View File

@ -1,6 +1,7 @@
import base64
import dateutil
import glob
import hypchat
import logging
import os
import re
@ -207,7 +208,8 @@ def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int) -> List[ZerverFieldsT]:
realm_id: int,
api_token: Optional[str]=None) -> List[ZerverFieldsT]:
flat_data = [
d['Room']
for d in raw_data
@ -249,10 +251,18 @@ def convert_room_data(raw_data: List[ZerverFieldsT],
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
else:
users = set()
if api_token is not None:
hc = hypchat.HypChat(api_token)
room_data = hc.fromurl('{0}/v2/room/{1}/member'.format(hc.endpoint, in_dict['id']))
if not users:
continue
for item in room_data['items']:
hipchat_user_id = item['id']
zulip_user_id = user_id_mapper.get(hipchat_user_id)
users.add(zulip_user_id)
if users:
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
@ -768,7 +778,9 @@ def make_user_messages(zerver_message: List[ZerverFieldsT],
def do_convert_data(input_tar_file: str,
output_dir: str,
masking_content: bool) -> None:
masking_content: bool,
api_token: Optional[str]=None,
slim_mode: bool=False) -> None:
input_data_dir = untar_input_file(input_tar_file)
attachment_handler = AttachmentHandler()
@ -780,8 +792,6 @@ def do_convert_data(input_tar_file: str,
realm_id = 0
realm = make_realm(realm_id=realm_id)
slim_mode = False
# users.json -> UserProfile
raw_user_data = read_user_data(data_dir=input_data_dir)
convert_user_data(
@ -803,6 +813,7 @@ def do_convert_data(input_tar_file: str,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
api_token=api_token,
)
realm['zerver_stream'] = zerver_stream
@ -812,7 +823,7 @@ def do_convert_data(input_tar_file: str,
)
realm['zerver_recipient'] = zerver_recipient
if True:
if api_token is None:
if slim_mode:
public_stream_subscriptions = [] # type: List[ZerverFieldsT]
else:
@ -829,6 +840,12 @@ def do_convert_data(input_tar_file: str,
if stream_dict['invite_only']],
)
stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions
else:
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient,

10
zerver/management/commands/convert_hipchat_data.py
View File

@ -44,6 +44,14 @@ class Command(BaseCommand):
action="store_true",
help='Mask the content for privacy during QA.')
parser.add_argument('--slim-mode', dest='slim_mode',
action="store_true",
help='Mask the content for privacy during QA.')
parser.add_argument('--token', dest='api_token',
action="store",
help='API token for the HipChat API for fetching subscribers.')
parser.formatter_class = argparse.RawTextHelpFormatter
def handle(self, *args: Any, **options: Any) -> None:
@ -75,4 +83,6 @@ class Command(BaseCommand):
input_tar_file=path,
output_dir=output_dir,
masking_content=options.get('masking_content', False),
slim_mode=options['slim_mode'],
api_token=options.get("api_token"),
)