# Creates a Zulip remote development environment droplet or # a production droplet in DigitalOcean. # # Particularly useful for sprints/hackathons, interns, and other # situation where one wants to quickly onboard new contributors. # # Requires python-digitalocean library: # https://github.com/koalalorenzo/python-digitalocean # # Also requires DigitalOcean team membership for Zulip and API token: # https://cloud.digitalocean.com/settings/api/tokens # # Copy conf.ini-template to conf.ini and populate with your API token. import argparse import configparser import json import os import sys import time import urllib.error import urllib.request from typing import Any, Dict, List import digitalocean import requests parser = argparse.ArgumentParser(description="Create a Zulip devopment VM DigitalOcean droplet.") parser.add_argument( "username", help="GitHub username for whom you want to create a Zulip dev droplet" ) parser.add_argument("--tags", nargs="+", default=[]) parser.add_argument("-f", "--recreate", action="store_true") parser.add_argument("-p", "--production", action="store_true") def get_config() -> configparser.ConfigParser: config = configparser.ConfigParser() config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), "conf.ini")) return config def assert_github_user_exists(github_username: str) -> bool: print(f"Checking to see if GitHub user {github_username} exists...") user_api_url = f"https://api.github.com/users/{github_username}" try: response = urllib.request.urlopen(user_api_url) json.load(response) print("...user exists!") return True except urllib.error.HTTPError as err: print(err) print(f"Does the GitHub user {github_username} exist?") sys.exit(1) def get_ssh_public_keys_from_github(github_username: str) -> List[Dict[str, Any]]: print("Checking to see that GitHub user has available public keys...") apiurl_keys = f"https://api.github.com/users/{github_username}/keys" try: response = urllib.request.urlopen(apiurl_keys) userkeys = json.load(response) if not userkeys: print( f"No keys found. Has user {github_username} added SSH keys to their GitHub account?" ) sys.exit(1) print("...public keys found!") return userkeys except urllib.error.HTTPError as err: print(err) print(f"Has user {github_username} added SSH keys to their GitHub account?") sys.exit(1) def assert_user_forked_zulip_server_repo(username: str) -> bool: print("Checking to see GitHub user has forked zulip/zulip...") apiurl_fork = f"https://api.github.com/repos/{username}/zulip" try: response = urllib.request.urlopen(apiurl_fork) json.load(response) print("...fork found!") return True except urllib.error.HTTPError as err: print(err) print(f"Has user {username} forked zulip/zulip?") sys.exit(1) def assert_droplet_does_not_exist(my_token: str, droplet_name: str, recreate: bool) -> None: print(f"Checking to see if droplet {droplet_name} already exists...") manager = digitalocean.Manager(token=my_token) my_droplets = manager.get_all_droplets() for droplet in my_droplets: if droplet.name.lower() == droplet_name: if not recreate: print( "Droplet {} already exists. Pass --recreate if you " "need to recreate the droplet.".format(droplet_name) ) sys.exit(1) else: print(f"Deleting existing droplet {droplet_name}.") droplet.destroy() return print("...No droplet found...proceeding.") def get_ssh_keys_string_from_github_ssh_key_dicts(userkey_dicts: List[Dict[str, Any]]) -> str: return "\n".join([userkey_dict["key"] for userkey_dict in userkey_dicts]) def generate_dev_droplet_user_data(username: str, userkey_dicts: List[Dict[str, Any]]) -> str: ssh_keys_string = get_ssh_keys_string_from_github_ssh_key_dicts(userkey_dicts) setup_root_ssh_keys = f"printf '{ssh_keys_string}' > /root/.ssh/authorized_keys" setup_zulipdev_ssh_keys = f"printf '{ssh_keys_string}' > /home/zulipdev/.ssh/authorized_keys" # We pass the hostname as username.zulipdev.org to the DigitalOcean API. # But some droplets (eg on 18.04) are created with with hostname set to just username. # So we fix the hostname using cloud-init. hostname_setup = f"hostnamectl set-hostname {username}.zulipdev.org" setup_repo = ( "cd /home/zulipdev/{1} && " "git remote add origin https://github.com/{0}/{1}.git && " "git fetch origin && " "git clean -f" ) server_repo_setup = setup_repo.format(username, "zulip") python_api_repo_setup = setup_repo.format(username, "python-zulip-api") cloudconf = f"""\ #!/bin/bash {setup_zulipdev_ssh_keys} {setup_root_ssh_keys} sed -i "s/PasswordAuthentication yes/PasswordAuthentication no/g" /etc/ssh/sshd_config service ssh restart {hostname_setup} su -c '{server_repo_setup}' zulipdev su -c '{python_api_repo_setup}' zulipdev su -c 'git config --global core.editor nano' zulipdev su -c 'git config --global pull.rebase true' zulipdev """ print("...returning cloud-config data.") return cloudconf def generate_prod_droplet_user_data(username: str, userkey_dicts: List[Dict[str, Any]]) -> str: ssh_keys_string = get_ssh_keys_string_from_github_ssh_key_dicts(userkey_dicts) setup_root_ssh_keys = f"printf '{ssh_keys_string}' > /root/.ssh/authorized_keys" cloudconf = f"""\ #!/bin/bash {setup_root_ssh_keys} passwd -d root sed -i "s/PasswordAuthentication yes/PasswordAuthentication no/g" /etc/ssh/sshd_config service ssh restart """ print("...returning cloud-config data.") return cloudconf def create_droplet( my_token: str, template_id: str, name: str, tags: List[str], user_data: str ) -> str: droplet = digitalocean.Droplet( token=my_token, name=name, region="nyc3", image=template_id, size_slug="s-1vcpu-2gb", user_data=user_data, tags=tags, backups=False, ) print("Initiating droplet creation...") droplet.create() incomplete = True while incomplete: actions = droplet.get_actions() for action in actions: action.load() print(f"...[{action.type}]: {action.status}") if action.type == "create" and action.status == "completed": incomplete = False break if incomplete: time.sleep(15) print("...droplet created!") droplet.load() print(f"...ip address for new droplet is: {droplet.ip_address}.") return droplet.ip_address def delete_existing_records(records: List[digitalocean.Record], record_name: str) -> None: count = 0 for record in records: if record.name == record_name and record.domain == "zulipdev.org" and record.type == "A": record.destroy() count = count + 1 if count: print(f"Deleted {count} existing A records for {record_name}.zulipdev.org.") def create_dns_record(my_token: str, record_name: str, ip_address: str) -> None: domain = digitalocean.Domain(token=my_token, name="zulipdev.org") domain.load() records = domain.get_records() delete_existing_records(records, record_name) wildcard_name = "*." + record_name delete_existing_records(records, wildcard_name) print(f"Creating new A record for {record_name}.zulipdev.org that points to {ip_address}.") domain.create_new_domain_record(type="A", name=record_name, data=ip_address) print(f"Creating new A record for *.{record_name}.zulipdev.org that points to {ip_address}.") domain.create_new_domain_record(type="A", name=wildcard_name, data=ip_address) def print_dev_droplet_instructions(droplet_domain_name: str) -> None: print( """ COMPLETE! Droplet for GitHub user {0} is available at {0}.zulipdev.org. Instructions for use are below. (copy and paste to the user) ------ Your remote Zulip dev server has been created! - Connect to your server by running `ssh zulipdev@{0}` on the command line (Terminal for macOS and Linux, Bash for Git on Windows). - There is no password; your account is configured to use your SSH keys. - Once you log in, you should see `(zulip-py3-venv) ~$`. - To start the dev server, `cd zulip` and then run `./tools/run-dev.py`. - While the dev server is running, you can see the Zulip server in your browser at http://{0}:9991. """.format( droplet_domain_name ) ) print( "See [Developing remotely](https://zulip.readthedocs.io/en/latest/development/remote.html) " "for tips on using the remote dev instance and " "[Git & GitHub guide](https://zulip.readthedocs.io/en/latest/git/index.html) " "to learn how to use Git with Zulip.\n" ) print( "Note that this droplet will automatically be deleted after a month of inactivity. " "If you are leaving Zulip for more than a few weeks, we recommend pushing all of your " "active branches to GitHub." ) print("------") def print_production_droplet_instructions(droplet_domain_name: str) -> None: print( """ ----- Production droplet created successfully! Connect to the server by running ssh root@{} ----- """.format( droplet_domain_name ) ) def get_zulip_oneclick_app_slug(api_token: str) -> str: response = requests.get( "https://api.digitalocean.com/v2/1-clicks", headers={"Authorization": f"Bearer {api_token}"} ).json() one_clicks = response["1_clicks"] for one_click in one_clicks: if one_click["slug"].startswith("kandralabs"): return one_click["slug"] raise Exception("Unable to find Zulip One-click app slug") if __name__ == "__main__": args = parser.parse_args() username = args.username.lower() if args.production: print(f"Creating production droplet for GitHub user {username}...") else: print(f"Creating Zulip developer environment for GitHub user {username}...") config = get_config() api_token = config["digitalocean"]["api_token"] assert_github_user_exists(github_username=username) public_keys = get_ssh_public_keys_from_github(github_username=username) if args.production: subdomain = f"{username}-prod" droplet_domain_name = f"{subdomain}.zulipdev.org" template_id = get_zulip_oneclick_app_slug(api_token) user_data = generate_prod_droplet_user_data(username=username, userkey_dicts=public_keys) else: assert_user_forked_zulip_server_repo(username=username) subdomain = username droplet_domain_name = f"{subdomain}.zulipdev.org" user_data = generate_dev_droplet_user_data(username=username, userkey_dicts=public_keys) # define id of image to create new droplets from # You can get this with something like the following. You may need to try other pages. # Broken in two to satisfy linter (line too long) # curl -X GET -H "Content-Type: application/json" -u : "https://api.digitaloc # ean.com/v2/images?page=5" | grep --color=always base.zulipdev.org template_id = "63219191" assert_droplet_does_not_exist( my_token=api_token, droplet_name=droplet_domain_name, recreate=args.recreate ) ip_address = create_droplet( my_token=api_token, template_id=template_id, name=droplet_domain_name, tags=args.tags, user_data=user_data, ) create_dns_record(my_token=api_token, record_name=subdomain, ip_address=ip_address) if args.production: print_production_droplet_instructions(droplet_domain_name=droplet_domain_name) else: print_dev_droplet_instructions(droplet_domain_name=droplet_domain_name) sys.exit(1)