From dec4b9ed93a1321d4d4fea0d013ae4069fb808ec Mon Sep 17 00:00:00 2001 From: Rishi Gupta Date: Sat, 28 Oct 2017 18:02:58 -0700 Subject: [PATCH] remote dev: Add code and instructions for creating digital ocean droplets. Mostly copied from the zulip/zulip-gci repository, but with some changes to wordings and code cleanup for linters. --- .gitignore | 1 + docs/request-remote-dev.md | 79 +++++++++++ requirements/dev.txt | 3 + requirements/dev_lock.txt | 3 +- tools/droplets/README.md | 144 +++++++++++++++++++ tools/droplets/conf.ini-template | 2 + tools/droplets/create.py | 236 +++++++++++++++++++++++++++++++ tools/linter_lib/custom_check.py | 1 + version.py | 2 +- 9 files changed, 469 insertions(+), 2 deletions(-) create mode 100644 docs/request-remote-dev.md create mode 100644 tools/droplets/README.md create mode 100644 tools/droplets/conf.ini-template create mode 100644 tools/droplets/create.py diff --git a/.gitignore b/.gitignore index 666cd48206..66db277976 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ /zproject/dev-secrets.conf /tools/conf.ini /tools/custom_provision +/tools/droplets/conf.ini ## Byproducts of setting up and using the dev environment *.pyc diff --git a/docs/request-remote-dev.md b/docs/request-remote-dev.md new file mode 100644 index 0000000000..cc488f6066 --- /dev/null +++ b/docs/request-remote-dev.md @@ -0,0 +1,79 @@ +# How to request a remote Zulip development instance + +Under specific circumstances, typically during sprints, hackathons, and +Google Code-in, Zulip can provide you with a virtual machine with the +development environment already set up. + +The machines (droplets) are being generously provided by +[Digital Ocean](https://www.digitalocean.com/). Thank you Digital Ocean! + +## Step 1: Join GitHub and create SSH Keys + +To contribute to Zulip and to use a remote Zulip developer instance, you'll +need a GitHub account. If you don't already have one, sign up +[here][github-join]. + +You'll also need to [create SSH keys and add them to your GitHub +account][github-help-add-ssh-key]. + +## Step 2: Create a fork of zulip/zulip + +Zulip uses a **forked-repo** and **[rebase][gitbook-rebase]-oriented +workflow**. This means that all contributors create a fork of the [Zulip +repository][github-zulip-zulip] they want to contribute to and then submit pull +requests to the upstream repository to have their contributions reviewed and +accepted. + +When we create your Zulip dev instance, we'll connect it to your fork of Zulip, +so that needs to exist before you make your request. + +While you're logged in to GitHub, navigate to [zulip/zulip][github-zulip-zulip] +and click the **Fork** button. (See [GitHub's help article][github-help-fork] +for further details). + +## Step 3: Make request via chat.zulip.org + +Now that you have a GitHub account, have added your SSH keys, and forked +zulip/zulip, you are ready to request your Zulip developer instance. + +If you haven't already, create an account on https://chat.zulip.org/. + +Next, join the [development +help](https://chat.zulip.org/#narrow/stream/development.20help) stream. Create a +new **stream message** with your GitHub username as the **topic** and request +your remote dev instance. **Please make sure you have completed steps 1 and 2 +before doing so**. A core developer should reply letting you know they're +working on creating it as soon as they are available to help. + +Once requested, it will only take a few minutes to create your instance. You +will be contacted when it is complete and available. + +## Next steps + +Once your remote dev instance is ready: + +- Connect to your server by running + `ssh zulipdev@.zulipdev.org` on the command line + (Terminal for macOS and Linux, Bash for Git on Windows). +- There is no password; your account is configured to use your SSH keys. +- Once you log in, you should see `(zulip-venv) ~$`. +- To start the dev server, `cd zulip` and then run `./tools/run-dev.py`. +- While the dev server is running, you can see the Zulip server in your browser + at http://username.zulipdev.org:9991. + +Once you've confirmed you can connect to your remote server, take a look at: + +* [developing remotely](dev-remote.html) for tips on using the remote dev + instance, and +* our [Git & GitHub Guide](git-guide.html) to learn how to use Git with Zulip. + +Next, read the following to learn more about developing for Zulip: + +* [Using the Development Environment](using-dev-environment.html) +* [Testing](testing.html) + +[github-join]: https://github.com/join +[github-help-add-ssh-key]: https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/ +[github-zulip-zulip]: https://github.com/zulip/zulip/ +[github-help-fork]: https://help.github.com/articles/fork-a-repo/ +[gitbook-rebase]: https://git-scm.com/book/en/v2/Git-Branching-Rebasing diff --git a/requirements/dev.txt b/requirements/dev.txt index 40960ff5fd..0441114ea9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -37,6 +37,9 @@ snakeviz==0.4.2 # Needed to sync translations from transifex transifex-client==0.12.4 +# Needed for creating digital ocean droplets +python-digitalocean==1.12 + # Needed for updating the locked pip dependencies pip-tools==1.10.1 diff --git a/requirements/dev_lock.txt b/requirements/dev_lock.txt index 9451278e39..fec220529a 100644 --- a/requirements/dev_lock.txt +++ b/requirements/dev_lock.txt @@ -115,6 +115,7 @@ pyldap==2.4.37 pylibmc==1.5.2 pyopenssl==17.0.0 # via ndg-httpsclient, scrapy, service-identity python-dateutil==2.6.1 +python-digitalocean==1.12 python-gcm==0.4 python-twitter==3.3 python3-openid==3.1.0 # via social-auth-core @@ -125,7 +126,7 @@ recommonmark==0.4.0 redis==2.10.6 regex==2017.7.28 requests-oauthlib==0.8.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, premailer, python-gcm, python-twitter, requests-oauthlib, social-auth-core, sphinx +requests==2.18.4 # via aws-xray-sdk, docker, moto, premailer, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, social-auth-core, sphinx rsa==3.4.2 s3transfer==0.1.11 # via boto3 scrapy==1.4.0 diff --git a/tools/droplets/README.md b/tools/droplets/README.md new file mode 100644 index 0000000000..006da537dd --- /dev/null +++ b/tools/droplets/README.md @@ -0,0 +1,144 @@ +# Create a remote Zulip dev server + +This guide is for mentors who want to help create remote Zulip dev servers +for hackathon, GCI, or sprint participants. + +The machines (droplets) have been generously provided by +[Digital Ocean](https://www.digitalocean.com/) to help Zulip contributors +get up and running as easily as possible. Thank you Digital Ocean! + +The `create.py` create uses the Digital Ocean API to quickly create new virtual +machines (droplets) with the Zulip dev server already configured. + +## Step 1: Join Zulip Digital Ocean team + +We have created a team on Digital Ocean for Zulip mentors. Ask Rishi or Tim +to be added. You need access to the team so you can create your Digital Ocean +API token. + +## Step 2: Create your Digital Ocean API token + +Once you've been added to the Zulip team, +[login](https://cloud.digitalocean.com/droplets) to the Digital Ocean control +panel and [create your personal API token][do-create-api-token]. **Make sure +you create your API token under the Zulip team.** (It should look something +like [this][image-zulip-team]). + +Copy the API token and store it somewhere safe. You'll need it in the next +step. + +## Step 3: Configure create.py + +In `tools/droplets/` there is a sample configuration file `conf.ini-template`. + +Copy this file to `conf.ini`: + +``` +$ cd tools/droplets/ +$ cp conf.ini-template conf.ini +``` + +Now edit the file and replace `APITOKEN` with the personal API token you +generated earlier. + +``` +[digitalocean] +api_token = APITOKEN +``` + +Now you're ready to use the script. + +## Usage + +`create.py` takes two arguments + +* GitHub username +* Tags (Optional argument) + +``` +$ python3 create.py +$ python3 create.py --tags +$ python3 create.py --tags +``` +Assigning tags to droplets like `GCI` can be later useful for +listing all the droplets created during GCI. +[Tags](https://www.digitalocean.com/community/tutorials/how-to-tag-digitalocean-droplets) +may contain letters, numbers, colons, dashes, and underscores. + +You'll need to run this from the Zulip development environment (e.g. in +Vagrant). + +In order for the script to work, the GitHub user must have: + +- forked the [zulip/zulip][zulip-zulip] repository, and +- created an ssh key pair and added it to their GitHub account. + +(Share [this link][how-to-request] with students if they need to do these +steps.) + +The script will stop if it can't find the user's fork or ssh keys. + +The script will also stop if a droplet has already been created for the user. +If you need to re-create a droplet, login to Digital Ocean with your browser +and delete **both** the **droplet** and its **dns entry**. + +Once the droplet is created, you will see something similar to this message: + +``` +Your remote Zulip dev server has been created! + +- Connect to your server by running + `ssh zulipdev@.zulipdev.org` on the command line + (Terminal for macOS and Linux, Bash for Git on Windows). +- There is no password; your account is configured to use your ssh keys. +- Once you log in, you should see `(zulip-venv) ~$`. +- To start the dev server, `cd zulip` and then run `./tools/run-dev.py`. +- While the dev server is running, you can see the Zulip server in your browser + at http://.zulipdev.org:9991. + +See [Developing +remotely](http://zulip.readthedocs.io/en/latest/dev-remote.html) for tips on +using the remote dev instance and [Git & GitHub +Guide](http://zulip.readthedocs.io/en/latest/git-guide.html) to learn how to +use Git with Zulip. +``` + +Copy and paste this message to the user via Zulip chat. Be sure to CC the user +so they are notified. + +[do-create-api-token]: https://www.digitalocean.com/community/tutorials/how-to-use-the-digitalocean-api-v2#how-to-generate-a-personal-access-token +[image-zulip-team]: http://cdn.subfictional.com/dropshare/Screen-Shot-2016-11-28-10-53-24-X86JYrrOzu.png +[zulip-zulip]: https://github.com/zulip/zulip +[python-digitalocean]: https://github.com/koalalorenzo/python-digitalocean +[how-to-request]: https://github.com/zulip/zulip-gci/blob/master/request-remote-dev.md + +## Updating the base image + +Rough steps: + +1. Get the `ssh` key for `base.zulipdev.org` from Christie or Rishi. +1. Power up the `base.zulipdev.org` droplet from the digitalocean UI. You + probably have to be logged in in the Zulip organization view, rather than + via your personal account. +1. `ssh zulipdev@base.zulipdev.org` +1. `git pull upstream master` +1. `tools/provision` +1. `git clean -f`, in case things were added/removed from `.gitignore`. +1. `/srv/zulip-py3-venv/bin/activate` (added after PyCon 2017, I forget why this was needed.) +1. `tools/run-dev.py`, let it run to completion, and then Ctrl-C (to clear + out anything in the Rabbit MQ queue, load messages, etc). +1. `tools/run-dev.py`, and check that `base.zulipdev.org:9991` is up and running. +1. `history -c` to clear any command line history, if you made a typo (to + reduce chance of confusing new contributors). +1. `sudo shutdown -h now` +1. Go to the Images tab on DigitalOcean, and "Take a Snapshot". +1. Wait for several minutes. +1. Make sure to add the appropriate regions via More -> "Add to region" in + the Snapshots section. +1. Do something like `curl -X GET -H "Content-Type: application/json" + -u : "https://api.digitalocean.com/v2/images?page=5" | grep --color=always base.zulipdev.org` + (maybe with a different page number, and replace your API_KEY). +1. Replace `template_id` in `create.py` in this directory with the + appropriate `id`, and region with the appropriate region. +1. Test that everything works. +1. Open a PR with the updated template_id in zulip/zulip! diff --git a/tools/droplets/conf.ini-template b/tools/droplets/conf.ini-template new file mode 100644 index 0000000000..80bc32577c --- /dev/null +++ b/tools/droplets/conf.ini-template @@ -0,0 +1,2 @@ +[digitalocean] +api_token = APITOKEN diff --git a/tools/droplets/create.py b/tools/droplets/create.py new file mode 100644 index 0000000000..4b081992f4 --- /dev/null +++ b/tools/droplets/create.py @@ -0,0 +1,236 @@ +# Creates a Droplet on Digital Ocean for remote Zulip development. +# Particularly useful for sprints/hackathons, interns, and other +# situation where one wants to quickly onboard new contributors. +# +# This script takes one argument: the name of the GitHub user for whom you want +# to create a Zulip developer environment. Requires Python 3. +# +# Requires python-digitalocean library: +# https://github.com/koalalorenzo/python-digitalocean +# +# Also requires Digital Ocean team membership for Zulip and api token: +# https://cloud.digitalocean.com/settings/api/tokens +# +# Copy conf.ini-template to conf.ini and populate with your api token. +# +# usage: python3 create.py + +import sys +import configparser +import urllib.error +import urllib.request +import json +import digitalocean +import time +import argparse +import os + +from typing import Any, Dict, List + +# initiation argument parser +parser = argparse.ArgumentParser(description='Create a Zulip devopment VM Digital Ocean droplet.') +parser.add_argument("username", help="Github username for whom you want to create a Zulip dev droplet") +parser.add_argument('--tags', nargs='+', default=[]) + +def get_config(): + # type: () -> configparser.ConfigParser + config = configparser.ConfigParser() + config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf.ini')) + return config + +def user_exists(username): + # type: (str) -> bool + print("Checking to see if GitHub user {0} exists...".format(username)) + user_api_url = "https://api.github.com/users/{0}".format(username) + try: + response = urllib.request.urlopen(user_api_url) + json.loads(response.read().decode()) + print("...user exists!") + return True + except urllib.error.HTTPError as err: + print(err) + print("Does the github user {0} exist?".format(username)) + sys.exit(1) + +def get_keys(username): + # type: (str) -> List[Dict[str, Any]] + print("Checking to see that GitHub user has available public keys...") + apiurl_keys = "https://api.github.com/users/{0}/keys".format(username) + try: + response = urllib.request.urlopen(apiurl_keys) + userkeys = json.loads(response.read().decode()) + if not userkeys: + print("No keys found. Has user {0} added ssh keys to their github account?".format(username)) + sys.exit(1) + print("...public keys found!") + return userkeys + except urllib.error.HTTPError as err: + print(err) + print("Has user {0} added ssh keys to their github account?".format(username)) + sys.exit(1) + +def fork_exists(username): + # type: (str) -> bool + print("Checking to see GitHub user has forked zulip/zulip...") + apiurl_fork = "https://api.github.com/repos/{0}/zulip".format(username) + try: + response = urllib.request.urlopen(apiurl_fork) + json.loads(response.read().decode()) + print("...fork found!") + return True + except urllib.error.HTTPError as err: + print(err) + print("Has user {0} forked zulip/zulip?".format(username)) + sys.exit(1) + +def exit_if_droplet_exists(my_token, username): + # type: (str, str) -> None + print("Checking to see if droplet for {0} already exists...".format(username)) + manager = digitalocean.Manager(token=my_token) + my_droplets = manager.get_all_droplets() + for droplet in my_droplets: + if droplet.name == "{0}.zulipdev.org".format(username): + print("Droplet for user {0} already exists.".format(username)) + print("Delete droplet AND dns entry via Digital Ocean control panel if you need to re-create.") + sys.exit(1) + print("...No droplet found...proceeding.") + +def set_user_data(username, userkeys): + # type: (str, List[Dict[str, Any]]) -> str + print("Setting cloud-config data, populated with GitHub user's public keys...") + ssh_authorized_keys = "" + + # spaces here are important here - these need to be properly indented under + # ssh_authorized_keys: + for key in userkeys: + ssh_authorized_keys += "\n - {0}".format(key['key']) + # print(ssh_authorized_keys) + + git_add_remote = "git remote add origin" # get around "line too long" lint error + cloudconf = """ + #cloud-config + users: + - name: zulipdev + ssh_authorized_keys:{1} + runcmd: + - su -c 'cd /home/zulipdev/zulip && {2} https://github.com/{0}/zulip.git && git fetch origin' zulipdev + - su -c 'git config --global core.editor nano' zulipdev + power_state: + mode: reboot + condition: True + """.format(username, ssh_authorized_keys, git_add_remote) + + print("...returning cloud-config data.") + return cloudconf + +def create_droplet(my_token, template_id, username, tags, user_data): + # type: (str, str, str, List[str], str) -> str + droplet = digitalocean.Droplet( + token=my_token, + name='{0}.zulipdev.org'.format(username), + region='sfo1', + image=template_id, + size_slug='2gb', + user_data=user_data, + tags=tags, + backups=False) + + print("Initiating droplet creation...") + droplet.create() + + incomplete = True + while incomplete: + actions = droplet.get_actions() + for action in actions: + action.load() + print("...[{0}]: {1}".format(action.type, action.status)) + if action.type == 'create' and action.status == 'completed': + incomplete = False + break + if incomplete: + time.sleep(15) + print("...droplet created!") + droplet.load() + print("...ip address for new droplet is: {0}.".format(droplet.ip_address)) + return droplet.ip_address + +def create_dns_record(my_token, username, ip_address): + # type: (str, str, str) -> None + print("Creating A record for {0}.zulipdev.org that points to {1}.".format(username, ip_address)) + domain = digitalocean.Domain(token=my_token, name='zulipdev.org') + domain.load() + domain.create_new_domain_record(type='A', name=username, data=ip_address) + +def print_completion(username): + # type: (str) -> None + print(""" +COMPLETE! Droplet for GitHub user {0} is available at {0}.zulipdev.org. + +Instructions for use are below. (copy and paste to the user) + +------ +Your remote Zulip dev server has been created! + +- Connect to your server by running + `ssh zulipdev@{0}.zulipdev.org` on the command line + (Terminal for macOS and Linux, Bash for Git on Windows). +- There is no password; your account is configured to use your ssh keys. +- Once you log in, you should see `(zulip-venv) ~$`. +- To start the dev server, `cd zulip` and then run `./tools/run-dev.py`. +- While the dev server is running, you can see the Zulip server in your browser at http://{0}.zulipdev.org:9991. +""".format(username)) + + print("See [Developing remotely](http://zulip.readthedocs.io/en/latest/dev-remote.html) " + "for tips on using the remote dev instance and " + "[Git & GitHub Guide](http://zulip.readthedocs.io/en/latest/git-guide.html) to learn " + "how to use Git with Zulip.\n") + print("Note that this droplet will automatically be deleted after a month of inactivity. " + "If you are leaving Zulip for more than a few weeks, we recommend pushing all of your " + "active branches to GitHub.") + print("------") + +if __name__ == '__main__': + # define id of image to create new droplets from + # You can get this with something like the following. You may need to try other pages. + # Broken in two to satisfy linter (line too long) + # curl -X GET -H "Content-Type: application/json" -u : "https://api.digitaloc + # ean.com/v2/images?page=5" | grep --color=always base.zulipdev.org + template_id = "28792373" + + # get command line arguments + args = parser.parse_args() + print("Creating Zulip developer environment for GitHub user {0}...".format(args.username)) + + # get config details + config = get_config() + + # see if droplet already exists for this user + user_exists(username=args.username) + + # grab user's public keys + public_keys = get_keys(username=args.username) + + # now make sure the user has forked zulip/zulip + fork_exists(username=args.username) + + api_token = config['digitalocean']['api_token'] + # does the droplet already exist? + exit_if_droplet_exists(my_token=api_token, username=args.username) + + # set user_data + user_data = set_user_data(username=args.username, userkeys=public_keys) + + # create droplet + ip_address = create_droplet(my_token=api_token, + template_id=template_id, + username=args.username, + tags=args.tags, + user_data=user_data) + + # create dns entry + create_dns_record(my_token=api_token, username=args.username, ip_address=ip_address) + + # print completion message + print_completion(username=args.username) + + sys.exit(1) diff --git a/tools/linter_lib/custom_check.py b/tools/linter_lib/custom_check.py index 1514aac9fa..cbc345b671 100644 --- a/tools/linter_lib/custom_check.py +++ b/tools/linter_lib/custom_check.py @@ -260,6 +260,7 @@ def build_custom_checkers(by_lang): 'bad_lines': ["'foo':bar", "'foo':1"]}, {'pattern': "^\s+#\w", 'strip': '\n', + 'exclude': set(['tools/droplets/create.py']), 'description': 'Missing whitespace after "#"', 'good_lines': ['a = b # some operation', '1+2 # 3 is the result'], 'bad_lines': [' #some operation', ' #not valid!!!']}, diff --git a/version.py b/version.py index 02f60a7b34..f8e20955ed 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ ZULIP_VERSION = "1.7.0+git" -PROVISION_VERSION = '11.1' +PROVISION_VERSION = '11.2'