puppet: Add a sentry release class.

This installs the Sentry CLI, and uses it to send API events to Sentry
when a release is started and completed.

docs/production/deployment.md

@@ -89,7 +89,58 @@ process](install-existing-server.md).
 
 Zulip's upgrades have a hook system which allows for arbitrary
 user-configured actions to run before and after an upgrade; see the
-[upgrading documentation](upgrade.md#deployment-hooks) for details.
+[upgrading documentation](upgrade.md#deployment-hooks) for details on
+how to write your own.
+
+Zulip also provides and optional deploy hook for Sentry.
+
+### Sentry deploy hook
+
+Zulip can use its deploy hooks to create [Sentry
+releases][sentry-release], which can help associate Sentry [error
+logging][sentry-error] with specific releases. If you are deploying
+Zulip from Git, it can be aware of which Zulip commits are associated
+with the release, and help identify which commits might be relevant to
+an error.
+
+To do so:
+
+1. Enable [Sentry error logging][sentry-error].
+2. Add a new [internal Sentry integration][sentry-internal] named
+   "Release annotator".
+3. Grant the internal integration the [permissions][sentry-perms] of
+   "Admin" on "Release".
+4. Add `, zulip::hooks::sentry` to the `puppet_classes` line in `/etc/zulip/zulip.conf`
+5. Add a `[sentry]` section to `/etc/zulip/zulip.conf`:
+   ```ini
+   [sentry]
+   organization = your-organization-name
+   project = your-project-name
+   ```
+6. Add the [authentication token] for your internal Sentry integration
+   to your `/etc/zulip/zulip-secrets.conf`:
+   ```ini
+   # Replace with your own token, found in Sentry
+   sentry_release_auth_token = 6c12f890c1c864666e64ee9c959c4552b3de473a076815e7669f53793fa16afc
+   ```
+7. As root, run `/home/zulip/deployments/current/scripts/zulip-puppet-apply`.
+
+If you are deploying Zulip from Git, you will also need to:
+
+1. In your Zulip project, add the [GitHub integration][sentry-github].
+2. Configure the `zulip/zulip` GitHub project for your Sentry project.
+   You should do this even if you are deploying a private fork of
+   Zulip.
+3. Additionally grant the internal integration "Read & Write" on
+   "Organization"; this is necessary to associate the commits with the
+   release.
+
+[sentry-release]: https://docs.sentry.io/product/releases/
+[sentry-error]: ../subsystems/logging.md#sentry-error-logging
+[sentry-github]: https://docs.sentry.io/product/integrations/source-code-mgmt/github/
+[sentry-internal]: https://docs.sentry.io/product/integrations/integration-platform/internal-integration/
+[sentry-perms]: https://docs.sentry.io/product/integrations/integration-platform/#permissions
+[sentry-tokens]: https://docs.sentry.io/product/integrations/integration-platform/internal-integration#auth-tokens
 
 ## Running Zulip's service dependencies on different machines
 
@@ -844,3 +895,13 @@ Because Camo includes logic to deny access to private subnets, routing
 its requests through Smokescreen is generally not necessary. Set to
 true or false to override the default, which uses the proxy only if
 it is not the default of Smokescreen on a local host.
+
+### `[sentry]`
+
+#### `organization`
+
+The Sentry organization used for the [Sentry deploy hook](#sentry-deploy-hook).
+
+#### `project`
+
+The Sentry project used for the [Sentry deploy hook](#sentry-deploy-hook).

docs/production/upgrade.md

@@ -223,6 +223,9 @@ is called, sorted in alphabetical order, from the working directory of
 the new version, with arguments of the old and new Zulip versions. If
 they exit with non-0 exit code, the upgrade will abort.
 
+See the [deploy documentation](deployment.md#deployment-hooks) for
+hooks included with Zulip.
+
 ## Preserving local changes to service configuration files
 
 :::{warning}

docs/subsystems/logging.md

@@ -61,8 +61,12 @@ You can enable it by:
     /home/zulip/deployments/current/scripts/restart-server
     ```
 
+You may also want to enable Zulip's [Sentry deploy
+hook][sentry-deploy-hook].
+
 [sentry-project]: https://docs.sentry.io/product/projects/
 [sentry-dsn]: https://docs.sentry.io/product/sentry-basics/dsn-explainer/
+[sentry-relase-hook]: ../production/deployment.md#sentry-deploy-hook
 
 ### Backend logging
 

puppet/zulip/files/hooks/post-deploy.d/sentry.hook

@@ -0,0 +1,50 @@
+#!/usr/bin/bash
+# Arguments: OLD_COMMIT NEW_COMMIT ...where both are `git describe`
+# output or tag names.  The CWD will be the new deploy directory.
+
+set -e
+set -u
+
+if ! grep -q 'SENTRY_DSN' /etc/zulip/settings.py; then
+    echo "sentry: No DSN configured!  Set SENTRY_DSN in /etc/zulip/settings.py"
+    exit 0
+fi
+
+if ! SENTRY_AUTH_TOKEN=$(crudini --get /etc/zulip/zulip-secrets.conf secrets sentry_release_auth_token); then
+    echo "sentry: No release auth token set!  Set sentry_release_auth_token in /etc/zulip/zulip-secrets.conf"
+    exit 0
+fi
+
+if ! SENTRY_ORG=$(crudini --get /etc/zulip/zulip.conf sentry organization); then
+    echo "sentry: No organization set!  Set sentry.organization in /etc/zulip/zulip.conf"
+    exit 0
+fi
+
+if ! SENTRY_PROJECT=$(crudini --get /etc/zulip/zulip.conf sentry project); then
+    echo "sentry: No project set!  Set setry.project in /etc/zulip/zulip.conf"
+    exit 0
+fi
+
+if ! which sentry-cli >/dev/null; then
+    echo "sentry: No sentry-cli installed!"
+    exit 0
+fi
+
+if ! [ -f ./sentry-release ]; then
+    echo "sentry: No Sentry sentry-release file found!"
+    exit 0
+fi
+
+SENTRY_RELEASE=$(cat ./sentry-release)
+
+ENVIRONMENT=$(crudini --get /etc/zulip/zulip.conf machine deploy_type || echo "development")
+
+echo "sentry: Adding deploy of '$ENVIRONMENT' and finalizing release"
+
+export SENTRY_AUTH_TOKEN
+sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" deploys "$SENTRY_RELEASE" new \
+    --env "$ENVIRONMENT" \
+    --started "$(stat -c %Y ./sentry-release)" \
+    --finished "$(date +%s)" \
+    --name "$(hostname --fqdn)"
+sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" finalize "$SENTRY_RELEASE"

puppet/zulip/files/hooks/pre-deploy.d/sentry.hook

@@ -0,0 +1,59 @@
+#!/usr/bin/bash
+# Arguments: OLD_COMMIT NEW_COMMIT ...where both are `git describe`
+# output or tag names.  The CWD will be the new deploy directory.
+
+set -e
+set -u
+
+if ! grep -q 'SENTRY_DSN' /etc/zulip/settings.py; then
+    echo "sentry: No DSN configured!  Set SENTRY_DSN in /etc/zulip/settings.py"
+    exit 0
+fi
+
+if ! SENTRY_AUTH_TOKEN=$(crudini --get /etc/zulip/zulip-secrets.conf secrets sentry_release_auth_token); then
+    echo "sentry: No release auth token set!  Set sentry_release_auth_token in /etc/zulip/zulip-secrets.conf"
+    exit 0
+fi
+
+if ! SENTRY_ORG=$(crudini --get /etc/zulip/zulip.conf sentry organization); then
+    echo "sentry: No organization set!  Set sentry.organization in /etc/zulip/zulip.conf"
+    exit 0
+fi
+
+if ! SENTRY_PROJECT=$(crudini --get /etc/zulip/zulip.conf sentry project); then
+    echo "sentry: No project set!  Set setry.project in /etc/zulip/zulip.conf"
+    exit 0
+fi
+
+if ! which sentry-cli >/dev/null; then
+    echo "sentry: No sentry-cli installed!"
+    exit 0
+fi
+
+NEW_VERSION="$2"
+
+MERGE_BASE=""
+if [ "$(git rev-parse --is-inside-work-tree 2>/dev/null || true)" = "true" ]; then
+    # Extract the merge-base that tools/cache-zulip-git-version
+    # encoded into ./zulip-git-version, and turn it from a `git
+    # describe` into a commit hash
+    MERGE_BASE_DESCRIBED=$(head -n2 ./zulip-git-version | tail -1)
+    if [[ "$MERGE_BASE_DESCRIBED" =~ ^.*-g([0-9a-f]{7,})$ ]]; then
+        MERGE_BASE=$(git rev-parse "${BASH_REMATCH[1]}")
+    else
+        MERGE_BASE=$(git rev-parse "$MERGE_BASE_DESCRIBED")
+    fi
+fi
+
+SENTRY_RELEASE="zulip-server@$NEW_VERSION"
+echo "$SENTRY_RELEASE" >./sentry-release
+
+echo "sentry: Creating release $SENTRY_RELEASE"
+
+export SENTRY_AUTH_TOKEN
+sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" new "$SENTRY_RELEASE"
+
+if [ -n "$MERGE_BASE" ]; then
+    echo "sentry: Setting commit range based on merge-base to upstream of $MERGE_BASE"
+    sudo -u zulip --preserve-env=SENTRY_AUTH_TOKEN sentry-cli releases --org="$SENTRY_ORG" --project="$SENTRY_PROJECT" set-commits "$SENTRY_RELEASE" --commit="zulip/zulip@$MERGE_BASE"
+fi

puppet/zulip/manifests/common.pp

@@ -85,6 +85,15 @@ class zulip::common {
 
     ### zulip_ops packages
 
+    # https://release-registry.services.sentry.io/apps/sentry-cli/latest
+    'sentry-cli' => {
+      'version' => '2.11.0',
+      'sha256'  => {
+        'amd64'   => 'bc8f5f223fa688b3ad963c60a729f02aa8f5b17525de66fb3abf86800977ff6e',
+        'aarch64' => 'c62c5c1259307611e78af4f24a4c30162cff8adb0f021d363b307c42cded5c70',
+      },
+    },
+
     # https://grafana.com/grafana/download?edition=oss
     'grafana' => {
       'version' => '9.3.4',

puppet/zulip/manifests/hooks/sentry.pp

@@ -0,0 +1,39 @@
+# @summary Install sentry-cli binary and pre/post deploy hooks
+#
+class zulip::hooks::sentry {
+  include zulip::hooks::base
+  $version = $zulip::common::versions['sentry-cli']['version']
+  $bin = "/srv/zulip-sentry-cli-${version}"
+
+  $arch = $::os['architecture'] ? {
+    'amd64'   => 'x86_64',
+    'aarch64' => 'aarch64',
+  }
+
+  zulip::external_dep { 'sentry-cli':
+    version => $version,
+    url     => "https://downloads.sentry-cdn.com/sentry-cli/${version}/sentry-cli-Linux-${arch}",
+  }
+
+  file { '/usr/local/bin/sentry-cli':
+    ensure => link,
+    target => $bin,
+  }
+
+  file { '/etc/zulip/hooks/pre-deploy.d/sentry.hook':
+    ensure => file,
+    mode   => '0755',
+    owner  => 'zulip',
+    group  => 'zulip',
+    source => 'puppet:///modules/zulip/hooks/pre-deploy.d/sentry.hook',
+    tag    => ['hooks'],
+  }
+  file { '/etc/zulip/hooks/post-deploy.d/sentry.hook':
+    ensure => file,
+    mode   => '0755',
+    owner  => 'zulip',
+    group  => 'zulip',
+    source => 'puppet:///modules/zulip/hooks/post-deploy.d/sentry.hook',
+    tag    => ['hooks'],
+  }
+}

zproject/sentry.py

@@ -1,3 +1,4 @@
+import os
 from typing import TYPE_CHECKING, Optional
 
 import sentry_sdk
@@ -9,6 +10,7 @@ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 from sentry_sdk.utils import capture_internal_exceptions
 
 from version import ZULIP_VERSION
+from zproject.config import DEPLOY_ROOT
 
 if TYPE_CHECKING:
     from sentry_sdk._types import Event, Hint
@@ -58,10 +60,15 @@ def add_context(event: "Event", hint: "Hint") -> Optional["Event"]:
 def setup_sentry(dsn: Optional[str], environment: str) -> None:
     if not dsn:
         return
+
+    sentry_release = ZULIP_VERSION
+    if os.path.exists(os.path.join(DEPLOY_ROOT, "sentry-release")):
+        with open(os.path.join(DEPLOY_ROOT, "sentry-release")) as sentry_release_file:
+            sentry_release = sentry_release_file.readline().strip()
     sentry_sdk.init(
         dsn=dsn,
         environment=environment,
-        release=ZULIP_VERSION,
+        release=sentry_release,
         integrations=[
             DjangoIntegration(),
             RedisIntegration(),