mirror of https://github.com/zulip/zulip.git
ci: Remove unused circleci config file and update codebase.
We have disabled CircleCI and are using GitHub Actions for automated testing. docs: Changed context from CircleCI to Github Actions and wrote some documentation specific to GH Actions. tools: Replaced env checks for CIRCLECI with GITHUB_ACTION. README: Use GitHub Actions build status badge.
This commit is contained in:
parent
59c8f3ed92
commit
2b23609f9a
|
@ -1,322 +0,0 @@
|
||||||
# See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
|
|
||||||
# high-level documentation on our CircleCI setup.
|
|
||||||
# See CircleCI upstream's docs on this config format:
|
|
||||||
# https://circleci.com/docs/2.0/language-python/
|
|
||||||
#
|
|
||||||
version: 2.0
|
|
||||||
aliases:
|
|
||||||
- &create_cache_directories
|
|
||||||
run:
|
|
||||||
name: create cache directories
|
|
||||||
command: |
|
|
||||||
dirs=(/srv/zulip-{npm,venv,emoji}-cache)
|
|
||||||
sudo mkdir -p "${dirs[@]}"
|
|
||||||
sudo chown -R circleci "${dirs[@]}"
|
|
||||||
|
|
||||||
- &restore_cache_package_json
|
|
||||||
restore_cache:
|
|
||||||
keys:
|
|
||||||
- v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
|
|
||||||
|
|
||||||
- &restore_cache_requirements
|
|
||||||
restore_cache:
|
|
||||||
keys:
|
|
||||||
- v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor-dev.txt" }}-{{ checksum "requirements/dev.txt" }}
|
|
||||||
|
|
||||||
- &restore_emoji_cache
|
|
||||||
restore_cache:
|
|
||||||
keys:
|
|
||||||
- v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "tools/setup/emoji/emoji_map.json" }}-{{ checksum "tools/setup/emoji/build_emoji" }}-{{checksum "tools/setup/emoji/emoji_setup_utils.py" }}-{{ checksum "tools/setup/emoji/emoji_names.py" }}-{{ checksum "package.json" }}
|
|
||||||
|
|
||||||
- &install_dependencies
|
|
||||||
run:
|
|
||||||
name: install dependencies
|
|
||||||
command: |
|
|
||||||
sudo apt-get update
|
|
||||||
# Install moreutils so we can use `ts` and `mispipe` in the following.
|
|
||||||
sudo apt-get install -y moreutils
|
|
||||||
|
|
||||||
# This is the main setup job for the test suite
|
|
||||||
mispipe "tools/ci/setup-backend --skip-dev-db-build" ts
|
|
||||||
|
|
||||||
# Cleaning caches is mostly unnecessary in Circle, because
|
|
||||||
# most builds don't get to write to the cache.
|
|
||||||
# mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0 2>&1" ts
|
|
||||||
|
|
||||||
- &save_cache_package_json
|
|
||||||
save_cache:
|
|
||||||
paths:
|
|
||||||
- /srv/zulip-npm-cache
|
|
||||||
key: v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
|
|
||||||
|
|
||||||
- &save_cache_requirements
|
|
||||||
save_cache:
|
|
||||||
paths:
|
|
||||||
- /srv/zulip-venv-cache
|
|
||||||
key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor-dev.txt" }}-{{ checksum "requirements/dev.txt" }}
|
|
||||||
|
|
||||||
- &save_emoji_cache
|
|
||||||
save_cache:
|
|
||||||
paths:
|
|
||||||
- /srv/zulip-emoji-cache
|
|
||||||
key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "tools/setup/emoji/emoji_map.json" }}-{{ checksum "tools/setup/emoji/build_emoji" }}-{{checksum "tools/setup/emoji/emoji_setup_utils.py" }}-{{ checksum "tools/setup/emoji/emoji_names.py" }}-{{ checksum "package.json" }}
|
|
||||||
|
|
||||||
- &do_bionic_hack
|
|
||||||
run:
|
|
||||||
name: do Bionic hack
|
|
||||||
command: |
|
|
||||||
# Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
|
|
||||||
# https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
|
|
||||||
sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
|
|
||||||
|
|
||||||
- &run_backend_tests
|
|
||||||
run:
|
|
||||||
name: run backend tests
|
|
||||||
command: |
|
|
||||||
. /srv/zulip-py3-venv/bin/activate
|
|
||||||
mispipe "./tools/ci/backend 2>&1" ts
|
|
||||||
|
|
||||||
- &run_frontend_tests
|
|
||||||
run:
|
|
||||||
name: run frontend tests
|
|
||||||
command: |
|
|
||||||
. /srv/zulip-py3-venv/bin/activate
|
|
||||||
mispipe "./tools/ci/frontend 2>&1" ts
|
|
||||||
|
|
||||||
- &upload_coverage_report
|
|
||||||
run:
|
|
||||||
name: upload coverage report
|
|
||||||
command: |
|
|
||||||
# codecov requires `.coverage` file to be stored in pwd for
|
|
||||||
# uploading coverage results.
|
|
||||||
mv /home/circleci/zulip/var/.coverage /home/circleci/zulip/.coverage
|
|
||||||
|
|
||||||
. /srv/zulip-py3-venv/bin/activate
|
|
||||||
|
|
||||||
pip install codecov && codecov
|
|
||||||
|
|
||||||
- &build_production
|
|
||||||
run:
|
|
||||||
name: build production
|
|
||||||
command: |
|
|
||||||
sudo apt-get update
|
|
||||||
# Install moreutils so we can use `ts` and `mispipe` in the following.
|
|
||||||
sudo apt-get install -y moreutils
|
|
||||||
|
|
||||||
mispipe "./tools/ci/production-build 2>&1" ts
|
|
||||||
|
|
||||||
- &production_extract_tarball
|
|
||||||
run:
|
|
||||||
name: production extract tarball
|
|
||||||
command: |
|
|
||||||
sudo apt-get update
|
|
||||||
# Install moreutils so we can use `ts` and `mispipe` in the following.
|
|
||||||
sudo apt-get install -y moreutils
|
|
||||||
|
|
||||||
mispipe "/tmp/production-extract-tarball 2>&1" ts
|
|
||||||
|
|
||||||
- &install_production
|
|
||||||
run:
|
|
||||||
name: install production
|
|
||||||
command: |
|
|
||||||
sudo service rabbitmq-server restart
|
|
||||||
sudo --preserve-env=CIRCLECI mispipe "/tmp/production-install 2>&1" ts
|
|
||||||
|
|
||||||
- &verify_production
|
|
||||||
run:
|
|
||||||
name: verify install
|
|
||||||
command: |
|
|
||||||
sudo --preserve-env=CIRCLECI mispipe "/tmp/production-verify 2>&1" ts
|
|
||||||
|
|
||||||
- &upgrade_postgresql
|
|
||||||
run:
|
|
||||||
name: upgrade postgresql
|
|
||||||
command: |
|
|
||||||
sudo --preserve-env=CIRCLECI mispipe "/tmp/production-upgrade-pg 2>&1" ts
|
|
||||||
|
|
||||||
- ¬ify_failure_status
|
|
||||||
run:
|
|
||||||
name: On fail
|
|
||||||
when: on_fail
|
|
||||||
branches:
|
|
||||||
only: master
|
|
||||||
command: |
|
|
||||||
if [[ "$CIRCLE_REPOSITORY_URL" == "git@github.com:zulip/zulip.git" && "$ZULIP_BOT_KEY" != "" ]]; then
|
|
||||||
URI_ESCAPED_TOPIC="$(python3 -c 'import sys; import urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$CIRCLE_BRANCH failing")"
|
|
||||||
curl -H "Content-Type: application/json" \
|
|
||||||
-X POST -i 'https://chat.zulip.org/api/v1/external/circleci?api_key='"$ZULIP_BOT_KEY"'&stream=automated%20testing&topic='"$URI_ESCAPED_TOPIC" \
|
|
||||||
-d '{"payload": { "branch": "'"$CIRCLE_BRANCH"'", "reponame": "'"$CIRCLE_PROJECT_REPONAME"'", "status": "failed", "build_url": "'"$CIRCLE_BUILD_URL"'", "username": "'"$CIRCLE_USERNAME"'"}}'
|
|
||||||
fi
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
"bionic-backend-frontend":
|
|
||||||
docker:
|
|
||||||
# This is built from tools/ci/images/bionic/Dockerfile .
|
|
||||||
# Bionic ships with Python 3.6.
|
|
||||||
- image: arpit551/circleci:bionic-python-test
|
|
||||||
|
|
||||||
working_directory: ~/zulip
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
|
|
||||||
- *create_cache_directories
|
|
||||||
- *do_bionic_hack
|
|
||||||
- *restore_cache_package_json
|
|
||||||
- *restore_cache_requirements
|
|
||||||
- *restore_emoji_cache
|
|
||||||
- *install_dependencies
|
|
||||||
- *save_cache_package_json
|
|
||||||
- *save_cache_requirements
|
|
||||||
- *save_emoji_cache
|
|
||||||
- *run_backend_tests
|
|
||||||
|
|
||||||
- run:
|
|
||||||
name: test locked requirements
|
|
||||||
command: |
|
|
||||||
. /srv/zulip-py3-venv/bin/activate
|
|
||||||
mispipe "./tools/test-locked-requirements 2>&1" ts
|
|
||||||
|
|
||||||
- *run_frontend_tests
|
|
||||||
# We only need to upload coverage reports on whichever platform
|
|
||||||
# runs the frontend tests.
|
|
||||||
- *upload_coverage_report
|
|
||||||
|
|
||||||
- store_artifacts:
|
|
||||||
path: ./var/puppeteer/
|
|
||||||
destination: puppeteer
|
|
||||||
|
|
||||||
- *notify_failure_status
|
|
||||||
|
|
||||||
"focal-backend":
|
|
||||||
docker:
|
|
||||||
# This is built from tools/ci/images/focal/Dockerfile.
|
|
||||||
# Focal ships with Python 3.8.2.
|
|
||||||
- image: arpit551/circleci:focal-python-test
|
|
||||||
|
|
||||||
working_directory: ~/zulip
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
|
|
||||||
- *create_cache_directories
|
|
||||||
- *restore_cache_package_json
|
|
||||||
- *restore_cache_requirements
|
|
||||||
- *restore_emoji_cache
|
|
||||||
- *install_dependencies
|
|
||||||
- *save_cache_package_json
|
|
||||||
- *save_cache_requirements
|
|
||||||
- *save_emoji_cache
|
|
||||||
- *run_backend_tests
|
|
||||||
- run:
|
|
||||||
name: Check development database build
|
|
||||||
command: mispipe "tools/ci/setup-backend" ts
|
|
||||||
- *notify_failure_status
|
|
||||||
|
|
||||||
"bionic-production-build":
|
|
||||||
docker:
|
|
||||||
# This is built from tools/ci/images/bionic/Dockerfile .
|
|
||||||
# Bionic ships with Python 3.6.
|
|
||||||
- image: arpit551/circleci:bionic-python-test
|
|
||||||
|
|
||||||
working_directory: ~/zulip
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
|
|
||||||
- *create_cache_directories
|
|
||||||
- *do_bionic_hack
|
|
||||||
- *restore_cache_package_json
|
|
||||||
- *restore_cache_requirements
|
|
||||||
- *restore_emoji_cache
|
|
||||||
- *build_production
|
|
||||||
- *save_cache_package_json
|
|
||||||
- *save_cache_requirements
|
|
||||||
- *save_emoji_cache
|
|
||||||
|
|
||||||
# Persist the built tarball to be used in downstream job
|
|
||||||
# for installation of production server.
|
|
||||||
# See https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
|
|
||||||
- persist_to_workspace:
|
|
||||||
# Must be an absolute path,
|
|
||||||
# or relative path from working_directory.
|
|
||||||
# This is a directory on the container which is
|
|
||||||
# taken to be the root directory of the workspace.
|
|
||||||
root: /tmp/production-build
|
|
||||||
# Must be relative path from root
|
|
||||||
paths:
|
|
||||||
- "*"
|
|
||||||
- *notify_failure_status
|
|
||||||
|
|
||||||
"bionic-production-install":
|
|
||||||
docker:
|
|
||||||
# This is built from tools/ci/images/bionic/Dockerfile .
|
|
||||||
# Bionic ships with Python 3.6.
|
|
||||||
- image: arpit551/circleci:bionic-python-test
|
|
||||||
|
|
||||||
working_directory: ~/zulip
|
|
||||||
|
|
||||||
steps:
|
|
||||||
# Contains the built tarball from bionic-production-build job
|
|
||||||
- attach_workspace:
|
|
||||||
# Must be absolute path or relative path from working_directory
|
|
||||||
at: /tmp
|
|
||||||
|
|
||||||
- *create_cache_directories
|
|
||||||
- *do_bionic_hack
|
|
||||||
- *production_extract_tarball
|
|
||||||
- *restore_cache_package_json
|
|
||||||
- *install_production
|
|
||||||
- *verify_production
|
|
||||||
- *upgrade_postgresql
|
|
||||||
- *verify_production
|
|
||||||
- *save_cache_package_json
|
|
||||||
- *notify_failure_status
|
|
||||||
|
|
||||||
"focal-production-install":
|
|
||||||
docker:
|
|
||||||
# This is built from tools/ci/images/focal/Dockerfile.
|
|
||||||
# Focal ships with Python 3.8.2.
|
|
||||||
- image: arpit551/circleci:focal-python-test
|
|
||||||
|
|
||||||
working_directory: ~/zulip
|
|
||||||
|
|
||||||
steps:
|
|
||||||
# Contains the built tarball from bionic-production-build job
|
|
||||||
- attach_workspace:
|
|
||||||
# Must be absolute path or relative path from working_directory
|
|
||||||
at: /tmp
|
|
||||||
|
|
||||||
- *create_cache_directories
|
|
||||||
|
|
||||||
- run:
|
|
||||||
name: do memcached hack
|
|
||||||
command: |
|
|
||||||
# Temporary hack till memcached upstream is updated in Focal.
|
|
||||||
# https://bugs.launchpad.net/ubuntu/+source/memcached/+bug/1878721
|
|
||||||
echo "export SASL_CONF_PATH=/etc/sasl2" | sudo tee - a /etc/default/memcached
|
|
||||||
|
|
||||||
- *production_extract_tarball
|
|
||||||
- *restore_cache_package_json
|
|
||||||
- *install_production
|
|
||||||
- *verify_production
|
|
||||||
- *save_cache_package_json
|
|
||||||
- *notify_failure_status
|
|
||||||
|
|
||||||
workflows:
|
|
||||||
version: 2
|
|
||||||
"Ubuntu 18.04 Bionic (Python 3.6, backend+frontend)":
|
|
||||||
jobs:
|
|
||||||
- "bionic-backend-frontend"
|
|
||||||
"Ubuntu 20.04 Focal (Python 3.8, backend)":
|
|
||||||
jobs:
|
|
||||||
- "focal-backend"
|
|
||||||
"Production":
|
|
||||||
jobs:
|
|
||||||
- "bionic-production-build"
|
|
||||||
- "bionic-production-install":
|
|
||||||
requires:
|
|
||||||
- "bionic-production-build"
|
|
||||||
- "focal-production-install":
|
|
||||||
requires:
|
|
||||||
- "bionic-production-build"
|
|
|
@ -8,7 +8,7 @@ allows users to easily process hundreds or thousands of messages a day. With
|
||||||
over 700 contributors merging over 500 commits a month, Zulip is also the
|
over 700 contributors merging over 500 commits a month, Zulip is also the
|
||||||
largest and fastest growing open source group chat project.
|
largest and fastest growing open source group chat project.
|
||||||
|
|
||||||
[![CircleCI branch](https://img.shields.io/circleci/project/github/zulip/zulip/master.svg)](https://circleci.com/gh/zulip/zulip/tree/master)
|
[![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg?branch=master)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amaster)
|
||||||
[![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/master.svg)](https://codecov.io/gh/zulip/zulip/branch/master)
|
[![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/master.svg)](https://codecov.io/gh/zulip/zulip/branch/master)
|
||||||
[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
|
[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
|
||||||
[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
|
[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
|
||||||
|
|
|
@ -109,15 +109,15 @@ sooner is better.
|
||||||
* *The CI build.* The tests need to pass. One can investigate
|
* *The CI build.* The tests need to pass. One can investigate
|
||||||
any failures and figure out what to fix by clicking on a red X next
|
any failures and figure out what to fix by clicking on a red X next
|
||||||
to the commit hash or the Detail links on a pull request. (Example:
|
to the commit hash or the Detail links on a pull request. (Example:
|
||||||
in [#10618](https://github.com/zulip/zulip/pull/10618), browse to
|
in [#17584](https://github.com/zulip/zulip/pull/17584),
|
||||||
bottom and click the red X next to `c6044ee` to see the build jobs
|
click the red X before `49b10a3` to see the build jobs
|
||||||
for that commit. You can see that there are 5 build jobs in total.
|
for that commit. You can see that there are 7 build jobs in total.
|
||||||
All the 5 jobs run in CircleCI. You can see what caused
|
All the 7 jobs run in GitHub Actions. You can see what caused
|
||||||
the job to fail by clicking on the failed job. This will open
|
the job to fail by clicking on the failed job. This will open
|
||||||
up a page in the CI that has more details on why the job failed.
|
up a page in the CI that has more details on why the job failed.
|
||||||
For example [this](https://circleci.com/gh/zulip/zulip/16617)
|
For example [this](https://github.com/zulip/zulip/runs/2092955762)
|
||||||
is the page of the `bionic-python-3.6` job. See our docs on
|
is the page of the `Ubuntu 18.04 Bionic (Python 3.6, backend + frontend)` job.
|
||||||
[continuous integration](../testing/continuous-integration.md)
|
See our docs on [continuous integration](../testing/continuous-integration.md)
|
||||||
to learn more.
|
to learn more.
|
||||||
|
|
||||||
* *Technical design.* There are a lot of considerations here:
|
* *Technical design.* There are a lot of considerations here:
|
||||||
|
|
|
@ -103,13 +103,13 @@ first-time contributors][zulip-rtd-dev-first-time].
|
||||||
|
|
||||||
This step is optional, but recommended.
|
This step is optional, but recommended.
|
||||||
|
|
||||||
The Zulip Server project is configured to use [CircleCI][circle-ci]
|
The Zulip Server project is configured to use [GitHub Actions][github-actions]
|
||||||
to test and create builds upon each new commit and pull request.
|
to test and create builds upon each new commit and pull request.
|
||||||
CircleCI is the primary CI that runs frontend and backend
|
GitHub Actions is the primary CI that runs frontend and backend
|
||||||
tests across a wide range of Ubuntu distributions.
|
tests across a wide range of Ubuntu distributions.
|
||||||
|
|
||||||
CircleCI is free for open source projects and it's easy to
|
GitHub Actions is free for open source projects and it's easy to
|
||||||
configure for your own fork of Zulip. After doing so, CircleCI
|
configure for your own fork of Zulip. After doing so, GitHub Actions
|
||||||
will run tests for new refs you push to GitHub and email you the outcome
|
will run tests for new refs you push to GitHub and email you the outcome
|
||||||
(you can also view the results in the web interface).
|
(you can also view the results in the web interface).
|
||||||
|
|
||||||
|
@ -119,21 +119,13 @@ submitting a pull request. We generally recommend a workflow where as
|
||||||
you make changes, you use a fast edit-refresh cycle running individual
|
you make changes, you use a fast edit-refresh cycle running individual
|
||||||
tests locally until your changes work. But then once you've gotten
|
tests locally until your changes work. But then once you've gotten
|
||||||
the tests you'd expect to be relevant to your changes working, push a
|
the tests you'd expect to be relevant to your changes working, push a
|
||||||
branch to run the full test suite in CircleCI before
|
branch to run the full test suite in GitHub Actions before
|
||||||
you create a pull request. While you wait for CircleCI jobs
|
you create a pull request. While you wait for GitHub Actions jobs
|
||||||
to run, you can start working on your next task. When the tests finish,
|
to run, you can start working on your next task. When the tests finish,
|
||||||
you can create a pull request that you already know passes the tests.
|
you can create a pull request that you already know passes the tests.
|
||||||
|
|
||||||
### Set up CircleCI
|
GitHub Actions will run all the jobs by default on your forked repository.
|
||||||
|
You can check the `Actions` tab of your repository to see the builds.
|
||||||
First, sign in to [CircleCI][circle-ci] with your GitHub account and authorize
|
|
||||||
CircleCI to access your GitHub account and repositories. Once you've logged
|
|
||||||
in you'll be in the **Projects** section which will list all your GitHub
|
|
||||||
repositories. Now goto the row of Zulip and click on **Set Up Project**.
|
|
||||||
You'll then see a sample hello world config. As your forked repository from Zulip
|
|
||||||
will already have the config file, so click on **Start Building** and then choose
|
|
||||||
the **Add Manually** option. After that click on **Start Building** to run the build.
|
|
||||||
![Screencast of CircleCI setup](../images/zulip-circleci.gif)
|
|
||||||
|
|
||||||
[gitbook-rebase]: https://git-scm.com/book/en/v2/Git-Branching-Rebasing
|
[gitbook-rebase]: https://git-scm.com/book/en/v2/Git-Branching-Rebasing
|
||||||
[github-help-add-ssh-key]: https://help.github.com/en/articles/adding-a-new-ssh-key-to-your-github-account
|
[github-help-add-ssh-key]: https://help.github.com/en/articles/adding-a-new-ssh-key-to-your-github-account
|
||||||
|
@ -142,7 +134,7 @@ the **Add Manually** option. After that click on **Start Building** to run the b
|
||||||
[github-help-sync-fork]: https://help.github.com/en/articles/syncing-a-fork
|
[github-help-sync-fork]: https://help.github.com/en/articles/syncing-a-fork
|
||||||
[github-zulip]: https://github.com/zulip/
|
[github-zulip]: https://github.com/zulip/
|
||||||
[github-zulip-zulip]: https://github.com/zulip/zulip/
|
[github-zulip-zulip]: https://github.com/zulip/zulip/
|
||||||
[circle-ci]:https://circleci.com/
|
[github-actions]: https://docs.github.com/en/actions
|
||||||
[zulip-rtd-dev-first-time]: ../development/setup-vagrant.md
|
[zulip-rtd-dev-first-time]: ../development/setup-vagrant.md
|
||||||
[zulip-rtd-dev-overview]: ../development/overview.md
|
[zulip-rtd-dev-overview]: ../development/overview.md
|
||||||
[zulip-rtd-tools-setup]: ../git/zulip-tools.html#set-up-git-repo-script
|
[zulip-rtd-tools-setup]: ../git/zulip-tools.html#set-up-git-repo-script
|
||||||
|
|
|
@ -125,7 +125,7 @@ Now you're ready to work on the issue or feature.
|
||||||
|
|
||||||
## Run linters and tests locally
|
## Run linters and tests locally
|
||||||
|
|
||||||
In addition to having CircleCI run tests and linters each time you
|
In addition to having GitHub Actions run tests and linters each time you
|
||||||
push a new commit, you can also run them locally. See
|
push a new commit, you can also run them locally. See
|
||||||
[testing](../testing/testing.md) for details.
|
[testing](../testing/testing.md) for details.
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
# Continuous integration (CI)
|
# Continuous integration (CI)
|
||||||
|
|
||||||
The Zulip server uses [CircleCI](https://circleci.com/) for continuous
|
The Zulip server uses [GitHub Actions](https://docs.github.com/en/actions) for continuous
|
||||||
integration. CircleCI runs frontend, backend and end-to-end production
|
integration. GitHub Actions runs frontend, backend and end-to-end production
|
||||||
installer tests. This page documents useful tools and tips when using
|
installer tests. This page documents useful tools and tips when using
|
||||||
CircleCI and debugging issues with it.
|
GitHub Actions and debugging issues with it.
|
||||||
|
|
||||||
## Goals
|
## Goals
|
||||||
|
|
||||||
|
@ -22,65 +22,51 @@ in development. Except when working on the CI configuration itself, a
|
||||||
developer should never have to repeatedly wait 10 minutes for a full CI
|
developer should never have to repeatedly wait 10 minutes for a full CI
|
||||||
run to iteratively debug something.
|
run to iteratively debug something.
|
||||||
|
|
||||||
## CircleCI
|
## GitHub Actions
|
||||||
|
|
||||||
### Useful debugging tips and tools
|
### Useful debugging tips and tools
|
||||||
|
|
||||||
* Zulip uses the `ts` tool to log the current time on every line of
|
* Zulip uses the `ts` tool to log the current time on every line of
|
||||||
the output in our CircleCI scripts. You can use this output to
|
the output in our GitHub Action scripts. You can use this output to
|
||||||
determine which steps are actually consuming a lot of time.
|
determine which steps are actually consuming a lot of time.
|
||||||
|
|
||||||
* You can [sign up your personal repo for CircleCI][circleci-setup] so
|
* GitHub Actions runs on every branch you push on your Zulip fork.
|
||||||
that every remote branch you push will be tested, which can be helpful
|
This is helpful when debugging something complicated.
|
||||||
when debugging something complicated.
|
|
||||||
|
|
||||||
* With your personal repo signed up, CircleCI
|
* You can also ssh into a container to debug failures. SSHing into
|
||||||
[allows you to SSH][circleci-ssh] into the job container if a job
|
the containers can be helpful, especially in rare cases where the
|
||||||
fails. SSHing into the containers can be helpful, especially in rare
|
tests are passing in your computer but failing in the CI. There are
|
||||||
cases where the tests are passing in your computer but failing in the
|
various
|
||||||
CI. Make sure that you have uploaded your SSH keys to GitHub: CircleCI
|
[Actions](https://github.com/marketplace?type=actions&query=debug+ssh)
|
||||||
uses those SSH keys for authentication.
|
available on GitHub Marketplace to help you SSH into a container. Use
|
||||||
|
whichever you find easiest to set up.
|
||||||
[docker-hub]: https://hub.docker.com/
|
|
||||||
[circleci-setup]: ../git/cloning.html#step-3-configure-continuous-integration-for-your-fork
|
|
||||||
[circleci-ssh]: https://circleci.com/docs/2.0/ssh-access-jobs/
|
|
||||||
|
|
||||||
### Suites
|
### Suites
|
||||||
|
|
||||||
The main CircleCI configuration file defining how the tests are run is
|
We run multiple jobs during a GitHub Actions build to efficiently run
|
||||||
[./circleci/config.yml][circleci-config]. Our code for running the
|
Zulip's various test suites, some of them multiple times because we
|
||||||
tests in CI lives under `tools/ci`; but they are mostly thin wrappers
|
support multiple versions of the base OS. See the [Actions
|
||||||
around [Zulip's test suites](../testing/testing.md) or production
|
tabs](https://github.com/zulip/zulip/actions) for full list of Actions
|
||||||
installer tooling.
|
that we run.
|
||||||
|
|
||||||
[circleci-config]: https://github.com/zulip/zulip/blob/master/.circleci/config.yml
|
Files which define GitHub workflows live in `.github/workflows` directory.
|
||||||
|
`zulip-ci.yml` is the main file where most of the tests are run.
|
||||||
|
`production-suite.yml` builds a Zulip release tarball, which is
|
||||||
|
then installed in a fresh container. Various Nagios and other
|
||||||
|
checks are run to confirm the installation worked.
|
||||||
|
|
||||||
We run multiple jobs during a CircleCI build to run Zulip's test
|
`zulip-ci.yml` is designed to run our main test suites on all of our
|
||||||
suites on our supported production platforms. They are currently:
|
supported platforms. Out of them, only one of them runs the frontend
|
||||||
|
tests, since `puppeteer` is slow and unlikely to catch issues that
|
||||||
|
depend on the version of the base OS and/or Python.
|
||||||
|
|
||||||
* bionic-backend-frontend
|
Our code for running the tests in CI lives under `tools/ci`; but that
|
||||||
* focal-backend
|
logic is mostly thin wrappers around [Zulip's test
|
||||||
|
suites](../testing/testing.md) or production installer.
|
||||||
|
|
||||||
Each runs the Zulip backend test suites, using the indicated
|
The `Legacy OS` tests are designed to ensure we give good error
|
||||||
platform/OS. As suggested by the names, only one
|
messages when trying to upgrade Zulip servers running on very old base
|
||||||
suite runs the frontend test suites, since those are not
|
OS versions with EOL Python versions that Zulip no longer supports.
|
||||||
platform-dependent.
|
|
||||||
|
|
||||||
Additionally, there a couple jobs designed to do an end-to-end test on
|
|
||||||
Zulip's production installer:
|
|
||||||
|
|
||||||
* bionic-production-build
|
|
||||||
* bionic-production-install
|
|
||||||
* xenial-legacy
|
|
||||||
|
|
||||||
The `production-build` job builds a Zulip release tarball, which is
|
|
||||||
then installed in a fresh container in the `production-install` job;
|
|
||||||
various Nagios and other checks are run to confirm the installation
|
|
||||||
worked.
|
|
||||||
|
|
||||||
The xenial-legacy tests are just designed to ensure we give the right
|
|
||||||
error messages when trying to install or upgrade a Xenial system to
|
|
||||||
master.
|
|
||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
|
@ -88,12 +74,12 @@ The remaining details in this section are primarily relevant for doing
|
||||||
development on our CI system and/or provisioning process.
|
development on our CI system and/or provisioning process.
|
||||||
|
|
||||||
The first key of the job section is `docker`. The docker key specifies
|
The first key of the job section is `docker`. The docker key specifies
|
||||||
the image CircleCI should get from [Docker Hub][docker-hub] for running
|
the image GitHub Action should get from [Docker Hub][docker-hub] for running
|
||||||
the job. Once CircleCI fetches the image from Docker Hub, it will spin
|
the job. Once GitHub Action fetches the image from Docker Hub, it will spin
|
||||||
up a docker container. See [images](#images) section to know more about
|
up a docker container. See [images](#images) section to know more about
|
||||||
the images we use in CircleCI for testing.
|
the images we use in GitHub Action for testing.
|
||||||
|
|
||||||
After booting the container from the configured image, CircleCI will
|
After booting the container from the configured image, GitHub Action will
|
||||||
create the directory mentioned in `working_directory` and all the
|
create the directory mentioned in `working_directory` and all the
|
||||||
steps are be run from here.
|
steps are be run from here.
|
||||||
|
|
||||||
|
@ -104,7 +90,7 @@ are defined in the `aliases` section at the top of the file.
|
||||||
|
|
||||||
### Images
|
### Images
|
||||||
|
|
||||||
CircleCI tests are run in containers that are spun off from the images
|
GitHub Action tests are run in containers that are spun off from the images
|
||||||
maintained by Zulip team. The Dockerfiles for the various images can be
|
maintained by Zulip team. The Dockerfiles for the various images can be
|
||||||
generated by running `./tools/ci/generate-dockerfiles`. This command
|
generated by running `./tools/ci/generate-dockerfiles`. This command
|
||||||
will generate the Dockerfiles of the three Ubuntu releases in
|
will generate the Dockerfiles of the three Ubuntu releases in
|
||||||
|
@ -118,7 +104,7 @@ generated Dockerfiles.
|
||||||
|
|
||||||
#### Caching
|
#### Caching
|
||||||
|
|
||||||
An important element of making CircleCI perform effectively is caching
|
An important element of making GitHub Action perform effectively is caching
|
||||||
between jobs the various caches that live under `/srv/` in a Zulip
|
between jobs the various caches that live under `/srv/` in a Zulip
|
||||||
development or production environment. In particular, we cache the
|
development or production environment. In particular, we cache the
|
||||||
following:
|
following:
|
||||||
|
@ -126,7 +112,7 @@ following:
|
||||||
* Python virtualenvs
|
* Python virtualenvs
|
||||||
* node_modules directories
|
* node_modules directories
|
||||||
|
|
||||||
This has a huge impact on the performance of running tests in CircleCI
|
This has a huge impact on the performance of running tests in GitHub Action
|
||||||
CI; without these caches, the average test time would be several times
|
CI; without these caches, the average test time would be several times
|
||||||
longer.
|
longer.
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,7 @@ set -x
|
||||||
|
|
||||||
./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
|
./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
|
||||||
./tools/test-tools
|
./tools/test-tools
|
||||||
# We need to pass a parallel level to test-backend because CircleCI's
|
# We need to pass a parallel level to test-backend because GitHub Actions'
|
||||||
# docker setup means the auto-detection logic sees the ~36 processes
|
# docker setup means the auto-detection logic sees the ~36 processes
|
||||||
# the Docker host has, not the ~2 processes of resources we're
|
# the Docker host has, not the ~2 processes of resources we're
|
||||||
# allocated.
|
# allocated.
|
||||||
|
|
|
@ -4,8 +4,4 @@ set -e
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
ZULIP_PATH=/home/github/zulip
|
ZULIP_PATH=/home/github/zulip
|
||||||
if [ "$CIRCLECI" = true ]; then
|
|
||||||
ZULIP_PATH=/home/circleci/zulip
|
|
||||||
fi
|
|
||||||
|
|
||||||
tar -xf /tmp/zulip-server-test.tar.gz -C "$ZULIP_PATH" --strip-components=1
|
tar -xf /tmp/zulip-server-test.tar.gz -C "$ZULIP_PATH" --strip-components=1
|
||||||
|
|
|
@ -5,9 +5,6 @@ set -e
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
ZULIP_PATH=/home/github/zulip
|
ZULIP_PATH=/home/github/zulip
|
||||||
if [ "$CIRCLECI" = true ]; then
|
|
||||||
ZULIP_PATH=/home/circleci/zulip
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Do an apt upgrade to start with an up-to-date machine
|
# Do an apt upgrade to start with an up-to-date machine
|
||||||
APT_OPTIONS=(-o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold')
|
APT_OPTIONS=(-o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold')
|
||||||
|
|
|
@ -4,9 +4,6 @@ set -e
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
ZULIP_PATH=/home/github/zulip
|
ZULIP_PATH=/home/github/zulip
|
||||||
if [ "$CIRCLECI" = true ]; then
|
|
||||||
ZULIP_PATH=/home/circleci/zulip
|
|
||||||
fi
|
|
||||||
|
|
||||||
supervisorctl stop all
|
supervisorctl stop all
|
||||||
"$ZULIP_PATH"/scripts/setup/upgrade-postgresql
|
"$ZULIP_PATH"/scripts/setup/upgrade-postgresql
|
||||||
|
|
|
@ -37,7 +37,7 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
VAR_DIR_PATH = os.path.join(ZULIP_PATH, "var")
|
VAR_DIR_PATH = os.path.join(ZULIP_PATH, "var")
|
||||||
|
|
||||||
CONTINUOUS_INTEGRATION = "GITHUB_ACTIONS" in os.environ or "CIRCLECI" in os.environ
|
CONTINUOUS_INTEGRATION = "GITHUB_ACTIONS" in os.environ
|
||||||
|
|
||||||
if not os.path.exists(os.path.join(ZULIP_PATH, ".git")):
|
if not os.path.exists(os.path.join(ZULIP_PATH, ".git")):
|
||||||
print(FAIL + "Error: No Zulip Git repository present!" + ENDC)
|
print(FAIL + "Error: No Zulip Git repository present!" + ENDC)
|
||||||
|
|
|
@ -62,7 +62,7 @@ if [ "$failed" -ne 0 ]; then
|
||||||
echo "* Logs are here: zulip/var/log/provision.log"
|
echo "* Logs are here: zulip/var/log/provision.log"
|
||||||
echo -e "$ENDC"
|
echo -e "$ENDC"
|
||||||
exit "$failed"
|
exit "$failed"
|
||||||
elif [ "$VIRTUAL_ENV" != "/srv/zulip-py3-venv" ] && [ -z "${CIRCLECI}${SKIP_VENV_SHELL_WARNING}" ]; then
|
elif [ "$VIRTUAL_ENV" != "/srv/zulip-py3-venv" ] && [ -z "${SKIP_VENV_SHELL_WARNING}" ]; then
|
||||||
echo -e "$WARNING"
|
echo -e "$WARNING"
|
||||||
echo "WARNING: This shell does not have the Zulip Python 3 virtualenv activated."
|
echo "WARNING: This shell does not have the Zulip Python 3 virtualenv activated."
|
||||||
echo "Zulip commands will fail until you activate the virtualenv."
|
echo "Zulip commands will fail until you activate the virtualenv."
|
||||||
|
|
|
@ -112,10 +112,14 @@ For help debugging, read:
|
||||||
or report and ask for help in chat.zulip.org""",
|
or report and ask for help in chat.zulip.org""",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
if os.environ.get("CIRCLECI"):
|
if os.environ.get("GITHUB_ACTIONS"):
|
||||||
print("", file=sys.stderr)
|
print("", file=sys.stderr)
|
||||||
print(
|
print(
|
||||||
"In CircleCI, the Artifacts tab contains screenshots of the failure.",
|
"""
|
||||||
|
See https://docs.github.com/en/rest/reference/actions#artifacts for information
|
||||||
|
on how to view the generated screenshots, which are uploaded as Artifacts.
|
||||||
|
Screenshots are extremely helpful for understanding puppeteer test failures.
|
||||||
|
""",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
print("", file=sys.stderr)
|
print("", file=sys.stderr)
|
||||||
|
|
Loading…
Reference in New Issue