From 3109d40b21e40fb6be6cd09edefa0290b40ff1af Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Sat, 28 Jan 2023 01:00:15 +0000 Subject: [PATCH] puppet: Add a sentry release class. This installs the Sentry CLI, and uses it to send API events to Sentry when a release is started and completed. --- docs/production/deployment.md | 63 ++++++++++++++++++- docs/production/upgrade.md | 3 + docs/subsystems/logging.md | 4 ++ .../files/hooks/post-deploy.d/sentry.hook | 50 +++++++++++++++ .../files/hooks/pre-deploy.d/sentry.hook | 59 +++++++++++++++++ puppet/zulip/manifests/common.pp | 9 +++ puppet/zulip/manifests/hooks/sentry.pp | 39 ++++++++++++ zproject/sentry.py | 9 ++- 8 files changed, 234 insertions(+), 2 deletions(-) create mode 100755 puppet/zulip/files/hooks/post-deploy.d/sentry.hook create mode 100755 puppet/zulip/files/hooks/pre-deploy.d/sentry.hook create mode 100644 puppet/zulip/manifests/hooks/sentry.pp diff --git a/docs/production/deployment.md b/docs/production/deployment.md index 885199b471..529692bc25 100644 --- a/docs/production/deployment.md +++ b/docs/production/deployment.md @@ -89,7 +89,58 @@ process](install-existing-server.md). Zulip's upgrades have a hook system which allows for arbitrary user-configured actions to run before and after an upgrade; see the -[upgrading documentation](upgrade.md#deployment-hooks) for details. +[upgrading documentation](upgrade.md#deployment-hooks) for details on +how to write your own. + +Zulip also provides and optional deploy hook for Sentry. + +### Sentry deploy hook + +Zulip can use its deploy hooks to create [Sentry +releases][sentry-release], which can help associate Sentry [error +logging][sentry-error] with specific releases. If you are deploying +Zulip from Git, it can be aware of which Zulip commits are associated +with the release, and help identify which commits might be relevant to +an error. + +To do so: + +1. Enable [Sentry error logging][sentry-error]. +2. Add a new [internal Sentry integration][sentry-internal] named + "Release annotator". +3. Grant the internal integration the [permissions][sentry-perms] of + "Admin" on "Release". +4. Add `, zulip::hooks::sentry` to the `puppet_classes` line in `/etc/zulip/zulip.conf` +5. Add a `[sentry]` section to `/etc/zulip/zulip.conf`: + ```ini + [sentry] + organization = your-organization-name + project = your-project-name + ``` +6. Add the [authentication token] for your internal Sentry integration + to your `/etc/zulip/zulip-secrets.conf`: + ```ini + # Replace with your own token, found in Sentry + sentry_release_auth_token = 6c12f890c1c864666e64ee9c959c4552b3de473a076815e7669f53793fa16afc + ``` +7. As root, run `/home/zulip/deployments/current/scripts/zulip-puppet-apply`. + +If you are deploying Zulip from Git, you will also need to: + +1. In your Zulip project, add the [GitHub integration][sentry-github]. +2. Configure the `zulip/zulip` GitHub project for your Sentry project. + You should do this even if you are deploying a private fork of + Zulip. +3. Additionally grant the internal integration "Read & Write" on + "Organization"; this is necessary to associate the commits with the + release. + +[sentry-release]: https://docs.sentry.io/product/releases/ +[sentry-error]: ../subsystems/logging.md#sentry-error-logging +[sentry-github]: https://docs.sentry.io/product/integrations/source-code-mgmt/github/ +[sentry-internal]: https://docs.sentry.io/product/integrations/integration-platform/internal-integration/ +[sentry-perms]: https://docs.sentry.io/product/integrations/integration-platform/#permissions +[sentry-tokens]: https://docs.sentry.io/product/integrations/integration-platform/internal-integration#auth-tokens ## Running Zulip's service dependencies on different machines @@ -844,3 +895,13 @@ Because Camo includes logic to deny access to private subnets, routing its requests through Smokescreen is generally not necessary. Set to true or false to override the default, which uses the proxy only if it is not the default of Smokescreen on a local host. + +### `[sentry]` + +#### `organization` + +The Sentry organization used for the [Sentry deploy hook](#sentry-deploy-hook). + +#### `project` + +The Sentry project used for the [Sentry deploy hook](#sentry-deploy-hook). diff --git a/docs/production/upgrade.md b/docs/production/upgrade.md index 1ff44e7e15..01a0ee6a33 100644 --- a/docs/production/upgrade.md +++ b/docs/production/upgrade.md @@ -223,6 +223,9 @@ is called, sorted in alphabetical order, from the working directory of the new version, with arguments of the old and new Zulip versions. If they exit with non-0 exit code, the upgrade will abort. +See the [deploy documentation](deployment.md#deployment-hooks) for +hooks included with Zulip. + ## Preserving local changes to service configuration files :::{warning} diff --git a/docs/subsystems/logging.md b/docs/subsystems/logging.md index 6cabbc5426..cb34d88f39 100644 --- a/docs/subsystems/logging.md +++ b/docs/subsystems/logging.md @@ -61,8 +61,12 @@ You can enable it by: /home/zulip/deployments/current/scripts/restart-server ``` +You may also want to enable Zulip's [Sentry deploy +hook][sentry-deploy-hook]. + [sentry-project]: https://docs.sentry.io/product/projects/ [sentry-dsn]: https://docs.sentry.io/product/sentry-basics/dsn-explainer/ +[sentry-relase-hook]: ../production/deployment.md#sentry-deploy-hook ### Backend logging diff --git a/puppet/zulip/files/hooks/post-deploy.d/sentry.hook b/puppet/zulip/files/hooks/post-deploy.d/sentry.hook new file mode 100755 index 0000000000..ae3c25c332 --- /dev/null +++ b/puppet/zulip/files/hooks/post-deploy.d/sentry.hook @@ -0,0 +1,50 @@ +#!/usr/bin/bash +# Arguments: OLD_COMMIT NEW_COMMIT ...where both are `git describe` +# output or tag names. The CWD will be the new deploy directory. + +set -e +set -u + +if ! grep -q 'SENTRY_DSN' /etc/zulip/settings.py; then + echo "sentry: No DSN configured! Set SENTRY_DSN in /etc/zulip/settings.py" + exit 0 +fi + +if ! SENTRY_AUTH_TOKEN=$(crudini --get /etc/zulip/zulip-secrets.conf secrets sentry_release_auth_token); then + echo "sentry: No release auth token set! Set sentry_release_auth_token in /etc/zulip/zulip-secrets.conf" + exit 0 +fi + +if ! SENTRY_ORG=$(crudini --get /etc/zulip/zulip.conf sentry organization); then + echo "sentry: No organization set! Set sentry.organization in /etc/zulip/zulip.conf" + exit 0 +fi + +if ! SENTRY_PROJECT=$(crudini --get /etc/zulip/zulip.conf sentry project); then + echo "sentry: No project set! Set setry.project in /etc/zulip/zulip.conf" + exit 0 +fi + +if ! which sentry-cli >/dev/null; then + echo "sentry: No sentry-cli installed!" + exit 0 +fi + +if ! [ -f ./sentry-release ]; then + echo "sentry: No Sentry sentry-release file found!" + exit 0 +fi + +SENTRY_RELEASE=$(cat ./sentry-release) + +ENVIRONMENT=$(crudini --get /etc/zulip/zulip.conf machine deploy_type || echo "development") + +echo "sentry: Adding deploy of '$ENVIRONMENT' and finalizing release" + +export SENTRY_AUTH_TOKEN +sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" deploys "$SENTRY_RELEASE" new \ + --env "$ENVIRONMENT" \ + --started "$(stat -c %Y ./sentry-release)" \ + --finished "$(date +%s)" \ + --name "$(hostname --fqdn)" +sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" finalize "$SENTRY_RELEASE" diff --git a/puppet/zulip/files/hooks/pre-deploy.d/sentry.hook b/puppet/zulip/files/hooks/pre-deploy.d/sentry.hook new file mode 100755 index 0000000000..6d1d18ef95 --- /dev/null +++ b/puppet/zulip/files/hooks/pre-deploy.d/sentry.hook @@ -0,0 +1,59 @@ +#!/usr/bin/bash +# Arguments: OLD_COMMIT NEW_COMMIT ...where both are `git describe` +# output or tag names. The CWD will be the new deploy directory. + +set -e +set -u + +if ! grep -q 'SENTRY_DSN' /etc/zulip/settings.py; then + echo "sentry: No DSN configured! Set SENTRY_DSN in /etc/zulip/settings.py" + exit 0 +fi + +if ! SENTRY_AUTH_TOKEN=$(crudini --get /etc/zulip/zulip-secrets.conf secrets sentry_release_auth_token); then + echo "sentry: No release auth token set! Set sentry_release_auth_token in /etc/zulip/zulip-secrets.conf" + exit 0 +fi + +if ! SENTRY_ORG=$(crudini --get /etc/zulip/zulip.conf sentry organization); then + echo "sentry: No organization set! Set sentry.organization in /etc/zulip/zulip.conf" + exit 0 +fi + +if ! SENTRY_PROJECT=$(crudini --get /etc/zulip/zulip.conf sentry project); then + echo "sentry: No project set! Set setry.project in /etc/zulip/zulip.conf" + exit 0 +fi + +if ! which sentry-cli >/dev/null; then + echo "sentry: No sentry-cli installed!" + exit 0 +fi + +NEW_VERSION="$2" + +MERGE_BASE="" +if [ "$(git rev-parse --is-inside-work-tree 2>/dev/null || true)" = "true" ]; then + # Extract the merge-base that tools/cache-zulip-git-version + # encoded into ./zulip-git-version, and turn it from a `git + # describe` into a commit hash + MERGE_BASE_DESCRIBED=$(head -n2 ./zulip-git-version | tail -1) + if [[ "$MERGE_BASE_DESCRIBED" =~ ^.*-g([0-9a-f]{7,})$ ]]; then + MERGE_BASE=$(git rev-parse "${BASH_REMATCH[1]}") + else + MERGE_BASE=$(git rev-parse "$MERGE_BASE_DESCRIBED") + fi +fi + +SENTRY_RELEASE="zulip-server@$NEW_VERSION" +echo "$SENTRY_RELEASE" >./sentry-release + +echo "sentry: Creating release $SENTRY_RELEASE" + +export SENTRY_AUTH_TOKEN +sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" new "$SENTRY_RELEASE" + +if [ -n "$MERGE_BASE" ]; then + echo "sentry: Setting commit range based on merge-base to upstream of $MERGE_BASE" + sudo -u zulip --preserve-env=SENTRY_AUTH_TOKEN sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" set-commits "$SENTRY_RELEASE" --commit="zulip/zulip@$MERGE_BASE" +fi diff --git a/puppet/zulip/manifests/common.pp b/puppet/zulip/manifests/common.pp index 69f260c9ea..59da1f8e25 100644 --- a/puppet/zulip/manifests/common.pp +++ b/puppet/zulip/manifests/common.pp @@ -85,6 +85,15 @@ class zulip::common { ### zulip_ops packages + # https://release-registry.services.sentry.io/apps/sentry-cli/latest + 'sentry-cli' => { + 'version' => '2.11.0', + 'sha256' => { + 'amd64' => 'bc8f5f223fa688b3ad963c60a729f02aa8f5b17525de66fb3abf86800977ff6e', + 'aarch64' => 'c62c5c1259307611e78af4f24a4c30162cff8adb0f021d363b307c42cded5c70', + }, + }, + # https://grafana.com/grafana/download?edition=oss 'grafana' => { 'version' => '9.3.4', diff --git a/puppet/zulip/manifests/hooks/sentry.pp b/puppet/zulip/manifests/hooks/sentry.pp new file mode 100644 index 0000000000..61e25c00db --- /dev/null +++ b/puppet/zulip/manifests/hooks/sentry.pp @@ -0,0 +1,39 @@ +# @summary Install sentry-cli binary and pre/post deploy hooks +# +class zulip::hooks::sentry { + include zulip::hooks::base + $version = $zulip::common::versions['sentry-cli']['version'] + $bin = "/srv/zulip-sentry-cli-${version}" + + $arch = $::os['architecture'] ? { + 'amd64' => 'x86_64', + 'aarch64' => 'aarch64', + } + + zulip::external_dep { 'sentry-cli': + version => $version, + url => "https://downloads.sentry-cdn.com/sentry-cli/${version}/sentry-cli-Linux-${arch}", + } + + file { '/usr/local/bin/sentry-cli': + ensure => link, + target => $bin, + } + + file { '/etc/zulip/hooks/pre-deploy.d/sentry.hook': + ensure => file, + mode => '0755', + owner => 'zulip', + group => 'zulip', + source => 'puppet:///modules/zulip/hooks/pre-deploy.d/sentry.hook', + tag => ['hooks'], + } + file { '/etc/zulip/hooks/post-deploy.d/sentry.hook': + ensure => file, + mode => '0755', + owner => 'zulip', + group => 'zulip', + source => 'puppet:///modules/zulip/hooks/post-deploy.d/sentry.hook', + tag => ['hooks'], + } +} diff --git a/zproject/sentry.py b/zproject/sentry.py index 0d4283db7b..e889710e3c 100644 --- a/zproject/sentry.py +++ b/zproject/sentry.py @@ -1,3 +1,4 @@ +import os from typing import TYPE_CHECKING, Optional import sentry_sdk @@ -9,6 +10,7 @@ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.utils import capture_internal_exceptions from version import ZULIP_VERSION +from zproject.config import DEPLOY_ROOT if TYPE_CHECKING: from sentry_sdk._types import Event, Hint @@ -58,10 +60,15 @@ def add_context(event: "Event", hint: "Hint") -> Optional["Event"]: def setup_sentry(dsn: Optional[str], environment: str) -> None: if not dsn: return + + sentry_release = ZULIP_VERSION + if os.path.exists(os.path.join(DEPLOY_ROOT, "sentry-release")): + with open(os.path.join(DEPLOY_ROOT, "sentry-release")) as sentry_release_file: + sentry_release = sentry_release_file.readline().strip() sentry_sdk.init( dsn=dsn, environment=environment, - release=ZULIP_VERSION, + release=sentry_release, integrations=[ DjangoIntegration(), RedisIntegration(),