Merge branch 'main' into #19888-chevron-for-stream-selection

.codespellignore

@@ -0,0 +1,18 @@
+te
+ans
+pullrequest
+ist
+cros
+wit
+nwe
+circularly
+ned
+ba
+ressemble
+ser
+sur
+hel
+fpr
+alls
+nd
+ot

.eslintrc.json

@@ -38,6 +38,7 @@
         "import/extensions": "error",
         "import/first": "error",
         "import/newline-after-import": "error",
+        "import/no-self-import": "error",
         "import/no-useless-path-segments": "error",
         "import/order": [
             "error",

.github/workflows/production-suite.yml

@@ -7,6 +7,7 @@ on:
       - .github/workflows/production-suite.yml
       - "**/migrations/**"
       - babel.config.js
+      - manage.py
       - postcss.config.js
       - puppet/**
       - requirements/**
@@ -16,6 +17,9 @@ on:
       - tools/**
       - webpack.config.ts
       - yarn.lock
+      - zerver/worker/queue_processors.py
+      - zerver/lib/push_notifications.py
+      - zerver/decorator.py
       - zproject/**
 
 defaults:
@@ -29,8 +33,8 @@ jobs:
     name: Bionic production build
     runs-on: ubuntu-latest
 
-    # This docker image was created by a generated Dockerfile at:
-    #   tools/ci/images/bionic/Dockerfile
+    # Docker images are built from 'tools/ci/Dockerfile'; the comments at
+    # the top explain how to build and upload these images.
     # Bionic ships with Python 3.6.
     container: zulip/ci:bionic
     steps:
@@ -159,6 +163,7 @@ jobs:
           # Since actions/download-artifact@v2 loses all the permissions
           # of the tarball uploaded by the upload artifact fix those.
           chmod +x /tmp/production-upgrade-pg
+          chmod +x /tmp/production-pgroonga
           chmod +x /tmp/production-install
           chmod +x /tmp/production-verify
           chmod +x /tmp/send-failure-message
@@ -191,6 +196,14 @@ jobs:
       - name: Verify install
         run: sudo /tmp/production-verify
 
+      - name: Install pgroonga
+        if: ${{ matrix.is_bionic }}
+        run: sudo /tmp/production-pgroonga
+
+      - name: Verify install after installing pgroonga
+        if: ${{ matrix.is_bionic }}
+        run: sudo /tmp/production-verify
+
       - name: Upgrade postgresql
         if: ${{ matrix.is_bionic }}
         run: sudo /tmp/production-upgrade-pg
@@ -216,13 +229,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Base images are built using `tools/ci/Dockerfile.prod.template`.
-          # The comments at the top explain how to build and upload these images.
+          # Docker images are built from 'tools/ci/Dockerfile'; the comments at
+          # the top explain how to build and upload these images.
           - docker_image: zulip/ci:buster-3.4
             name: 3.4 Version Upgrade
-            is_focal: true
             os: buster
 
+          - docker_image: zulip/ci:bullseye-4.7
+            name: 4.7 Version Upgrade
+            os: bullseye
+
     name: ${{ matrix.name  }}
     container:
       image: ${{ matrix.docker_image }}

.github/workflows/zulip-ci.yml

@@ -100,6 +100,11 @@ jobs:
           source tools/ci/activate-venv
           ./tools/test-tools
 
+      - name: Run Codespell lint
+        run: |
+          source tools/ci/activate-venv
+          ./tools/run-codespell
+
       - name: Run backend lint
         run: |
           source tools/ci/activate-venv

.gitignore

@@ -70,9 +70,12 @@ zulip.kdev4
 *.kate-swp
 *.sublime-project
 *.sublime-workspace
-.vscode/
 *.DS_Store
-# .cache/ is generated by Visual Studio Code's test runner
+# VS Code. Avoid checking in .vscode in general, while still specifying
+# recommended extensions for working with this repository.
+/.vscode/**/*
+!/.vscode/extensions.json
+# .cache/ is generated by VS Code test runner
 .cache/
 .eslintcache
 

.vscode/extensions.json

@@ -0,0 +1,23 @@
+{
+    // Recommended VS Code extensions for zulip/zulip.
+    //
+    // VS Code prompts a user to install the recommended extensions
+    // when a workspace is opened for the first time.  The user can
+    // also review the list with the 'Extensions: Show Recommended
+    // Extensions' command.  See
+    // https://code.visualstudio.com/docs/editor/extension-marketplace#_workspace-recommended-extensions
+    // for more information.
+    //
+    // Extension identifier format: ${publisher}.${name}.
+    // Example: vscode.csharp
+
+    "recommendations": [
+        "42crunch.vscode-openapi",
+        "dbaeumer.vscode-eslint",
+        "esbenp.prettier-vscode",
+        "ms-vscode-remote.vscode-remote-extensionpack"
+    ],
+
+    // Extensions recommended by VS Code which are not recommended for users of zulip/zulip.
+    "unwantedRecommendations": []
+}

CONTRIBUTING.md

@@ -8,18 +8,11 @@ The
 [Zulip community server](https://zulip.com/developer-community/)
 is the primary communication forum for the Zulip community. It is a good
 place to start whether you have a question, are a new contributor, are a new
-user, or anything else. Make sure to read the
+user, or anything else. Please review our
 [community norms](https://zulip.com/developer-community/#community-norms)
 before posting. The Zulip community is also governed by a
 [code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
 
-You can subscribe to
-[zulip-devel-announce@googlegroups.com](https://groups.google.com/g/zulip-devel-announce)
-or our [Twitter](https://twitter.com/zulip) account for a very low
-traffic (<1 email/month) way to hear about things like mentorship
-opportunities with Google Summer of Code, in-person sprints at
-conferences, and other opportunities to contribute.
-
 ## Ways to contribute
 
 To make a code or documentation contribution, read our
@@ -41,18 +34,18 @@ needs doing:
   and manually testing pull requests.
 
 **Non-code contributions**: Some of the most valuable ways to contribute
-don't require touching the codebase at all. We list a few of them below:
+don't require touching the codebase at all. For example, you can:
 
-- [Reporting issues](#reporting-issues), including both feature requests and
+- [Report issues](#reporting-issues), including both feature requests and
   bug reports.
-- [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
+- [Give feedback](#user-feedback) if you are evaluating or using Zulip.
 - [Sponsor Zulip](https://github.com/sponsors/zulip) through the GitHub sponsors program.
-- [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip.
-- [Outreach](#zulip-outreach): Star us on GitHub, upvote us
-  on product comparison sites, or write for [the Zulip blog](https://blog.zulip.org/).
+- [Translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
+  Zulip into your language.
+- [Stay connected](#stay-connected) with Zulip, and [help others
+  find us](#help-others-find-zulip).
 
-## Your first (codebase) contribution
+## Your first codebase contribution
 
 This section has a step by step guide to starting as a Zulip codebase
 contributor. It's long, but don't worry about doing all the steps perfectly;
@@ -70,43 +63,66 @@ to help.
 - Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
 - [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
   getting help in
-  [#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
+  [#provision help](https://chat.zulip.org/#narrow/stream/21-provision-help)
   if you run into any troubles.
-- Read the
-  [Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
-  and do the Git tutorial (coming soon) if you are unfamiliar with
-  Git, getting help in
-  [#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if
-  you run into any troubles. Be sure to check out the
-  [extremely useful Zulip-specific tools page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
+- Familiarize yourself with [using the development environment](https://zulip.readthedocs.io/en/latest/development/using.html).
+- Go through the [new application feature
+  tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html) to get familiar with
+  how the Zulip codebase is organized and how to find code in it.
+- Read the [Zulip guide to
+  Git](https://zulip.readthedocs.io/en/latest/git/index.html) if you
+  are unfamiliar with Git or Zulip's rebase-based Git workflow,
+  getting help in [#git
+  help](https://chat.zulip.org/#narrow/stream/44-git-help) if you run
+  into any troubles. Even Git experts should read the [Zulip-specific
+  Git tools
+  page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
 
-### Picking an issue
+### Where to look for an issue
 
-Now, you're ready to pick your first issue! There are hundreds of open issues
-in the main codebase alone. This section will help you find an issue to work
-on.
+Now you're ready to pick your first issue! Zulip has several repositories you
+can check out, depending on your interests. There are hundreds of open issues in
+the [main Zulip server and web app
+repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+alone.
 
-- If you're interested in
-  [mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
-  [desktop](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue),
-  or
-  [bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
-  development, check the respective links for open issues, or post in
-  [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
-  [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
-  [#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
-- For the main server and web repository, we recommend browsing
-  recently opened issues to look for issues you are confident you can
-  fix correctly in a way that clearly communicates why your changes
-  are the correct fix. Our GitHub workflow bot, zulipbot, limits
-  users who have 0 commits merged to claiming a single issue labeled
-  with "good first issue" or "help wanted".
-- We also partition all of our issues in the main repo into areas like
-  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
-  through our [list of labels](https://github.com/zulip/zulip/labels), and
-  click on some of the `area:` labels to see all the issues related to your
-  areas of interest.
-- If the lists of issues are overwhelming, post in
+Any issue with the "good first issue"
+label is a good candidate when you are getting started. In addition, many of the
+issues with the "help wanted" label may be approachable as well.
+
+- [Server and web app](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+- [Mobile apps](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+- [Desktop app](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+- [Terminal app](https://github.com/zulip/zulip-terminal/issues?q=is%3Aopen+is%3Aissue+label%3A"help+wanted")
+- [Python API bindings and bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+
+### Picking an issue to work on
+
+There's a lot to learn while making your first pull request, so start small!
+Many first contributions have fewer than 10 lines of changes (not counting
+changes to tests).
+
+We recommend the following process for finding an issue to work on:
+
+1. Read the description of an issue and make sure you understand it.
+2. If it seems promising, poke around the product
+   (on [chat.zulip.org](https://chat.zulip.org) or in the development
+   environment) until you know how the piece being
+   described fits into the bigger picture. If after some exploration the
+   description seems confusing or ambiguous, post a question on the GitHub
+   issue, as others may benefit from the clarification as well.
+3. When you find an issue you like, try to get started working on it. See if you
+   can find the part of the code you'll need to modify (`git grep` is your
+   friend!) and get some idea of how you'll approach the problem.
+4. If you feel lost, that's OK! Go through these steps again with another issue.
+   There's plenty to work on, and the exploration you do will help you learn
+   more about the project.
+
+Note that you are _not_ claiming an issue while you are iterating through steps
+1-4. _Before you claim an issue_, you should be confident that you will be able to
+tackle it effectively.
+
+If the lists of issues are overwhelming, you can post in
 [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
 bit about your background and interests, and we'll help you out. The most
 important thing to say is whether you're looking for a backend (Python),
@@ -114,49 +130,56 @@ on.
 documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
 bit about your programming experience and available time.
 
-We also welcome suggestions of features that you feel would be valuable or
-changes that you feel would make Zulip a better open source project. If you
-have a new feature you'd like to add, we recommend you start by posting in
-[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
-feature idea and the problem that you're hoping to solve.
+Additional tips for the [main server and web app
+repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
 
-Other notes:
+- We especially recommend browsing recently opened issues, as there are more
+  likely to be easy ones for you to find.
+- All issues are partitioned into areas like
+  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
+  through our [list of labels](https://github.com/zulip/zulip/labels), and
+  click on some of the `area:` labels to see all the issues related to your
+  areas of interest.
+- Avoid issues with the "difficult" label unless you
+  understand why it is difficult and are highly confident you can resolve the
+  issue correctly and completely.
 
-- For a first pull request, it's better to aim for a smaller contribution
-  than a bigger one. Many first contributions have fewer than 10 lines of
-  changes (not counting changes to tests).
-- The full list of issues explicitly looking for a contributor can be
-  found with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  and
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels. Avoid issues with the "difficult" label unless you
-  understand why it is difficult and are confident you can resolve the
-  issue correctly and completely. Issues without one of these labels
-  are fair game if Tim has written a clear technical design proposal
-  in the issue, or it is a bug that you can reproduce and you are
-  confident you can fix the issue correctly.
-- For most new contributors, there's a lot to learn while making your first
-  pull request. It's OK if it takes you a while; that's normal! You'll be
-  able to work a lot faster as you build experience.
+### Claiming an issue
 
-### Working on an issue
+#### In the main server and web app repository
 
-To work on an issue, claim it by adding a comment with `@zulipbot claim` to
+Post a comment with `@zulipbot claim` to
 the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
 workflow bot; it will assign you to the issue and label the issue as "in
-progress". Some additional notes:
-
-- You can only claim issues with the
+progress". You can only claim issues with the
 [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
 or
 [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 labels. Zulipbot will give you an error if you try to claim an issue
 without one of those labels.
+
+New contributors can only claim one issue until their first pull request is
+merged. This is to encourage folks to finish ongoing work before starting
+something new. If you would like to pick up a new issue while waiting for review
+on an almost-ready pull request, you can post a comment to this effect on the
+issue you're interested in.
+
+#### In other Zulip repositories
+
+There is no bot for other repositories, so you can simply post a comment saying
+that you'd like to work on the issue.
+
+Please follow the same guidelines as described above: find an issue labeled
+"good first issue" or "help wanted", and only pick up one issue at a time to
+start with.
+
+### Working on an issue
+
 - You're encouraged to ask questions on how to best implement or debug your
   changes -- the Zulip maintainers are excited to answer questions to help
-  you stay unblocked and working efficiently. You can ask questions on
-  chat.zulip.org, or on the GitHub issue or pull request.
+  you stay unblocked and working efficiently. You can ask questions in the
+  [Zulip development community](https://zulip.com/developer-community/),
+  or on the GitHub issue or pull request.
 - We encourage early pull requests for work in progress. Prefix the title of
   work in progress pull requests with `[WIP]`, and remove the prefix when
   you think it might be mergeable and want it to be reviewed.
@@ -165,22 +188,55 @@ progress". Some additional notes:
   changes when you post a comment, so if you don't, your PR will likely be
   neglected by accident!
 
-### And beyond
+It's OK if your first issue takes you a while; that's normal! You'll be
+able to work a lot faster as you build experience.
 
-A great place to look for a second issue is to look for issues with the same
+For more advice, see [What makes a great Zulip
+contributor?](https://zulip.readthedocs.io/en/latest/overview/contributing.html#what-makes-a-great-zulip-contributor)
+below.
+
+### Beyond the first issue
+
+To find a second issue to work on, we recommend looking through issues with the same
 `area:` label as the last issue you resolved. You'll be able to reuse the
 work you did learning how that part of the codebase works. Also, the path to
 becoming a core developer often involves taking ownership of one of these area
 labels.
 
+### Common questions
+
+- **What if somebody is already working on the issue I want do claim?** There
+  are lots of issue to work on! If somebody else is actively working on the
+  issue, you can find a different one, or help with
+  reviewing their work.
+- **What if somebody else claims an issue while I'm figuring out whether or not to
+  work on it?** No worries! You can contribute by providing feedback on
+  their pull request. If you've made good progress in understanding part of the
+  codebase, you can also find another "help wanted" issue in the same area to
+  work on.
+- **What if there is already a pull request for the issue I want to work on?**
+  Start by reviewing the existing work. If you agree with the approach, you can
+  use the existing pull request (PR) as a starting point for your contribution. If
+  you think a different approach is needed, you can post a new PR, with a comment that clearly
+  explains _why_ you decided to start from scratch.
+- **Can I come up with my own feature idea and work on it?** We welcome
+  suggestions of features or other improvements that you feel would be valuable. If you
+  have a new feature you'd like to add, you can start a conversation [in our
+  development community](https://zulip.com/developer-community/#where-do-i-send-my-message)
+  explaining the feature idea and the problem that you're hoping to solve.
+
 ## What makes a great Zulip contributor?
 
 Zulip has a lot of experience working with new contributors. In our
 experience, these are the best predictors of success:
 
-- Posting good questions. This generally means explaining your current
-  understanding, saying what you've done or tried so far, and including
-  tracebacks or other error messages if appropriate.
+- Posting good questions. It's very hard to answer a general question like, "How
+  do I do this issue?" When asking for help, explain
+  your current understanding, including what you've done or tried so far and where
+  you got stuck. Post tracebacks or other error messages if appropriate. For
+  more information, check out the ["Getting help" section of our community
+  guidelines](https://zulip.com/developer-community/#getting-help) and
+  [this essay][good-questions-blog] for some good advice.
 - Learning and practicing
   [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
 - Submitting carefully tested code. This generally means checking your work
@@ -191,11 +247,17 @@ experience, these are the best predictors of success:
 - Posting
   [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
   for frontend changes.
+- Clearly describing what you have implemented and why. For example, if your
+  implementation differs from the issue description in some way or is a partial
+  step towards the requirements described in the issue, be sure to call
+  out those differences.
 - Being responsive to feedback on pull requests. This means incorporating or
   responding to all suggested changes, and leaving a note if you won't be
   able to address things within a few days.
 - Being helpful and friendly on chat.zulip.org.
 
+[good-questions-blog]: https://jvns.ca/blog/good-questions/
+
 These are also the main criteria we use to select candidates for all
 of our outreach programs.
 
@@ -219,8 +281,8 @@ if appropriate.
 
 **Reporting security issues**. Please do not report security issues
 publicly, including on public streams on chat.zulip.org. You can
-email security@zulip.com. We create a CVE for every security
-issue in our released software.
+email [security@zulip.com](mailto:security@zulip.com). We create a CVE for every
+security issue in our released software.
 
 ## User feedback
 
@@ -243,6 +305,10 @@ to:
 - Organization: What does your organization do? How big is the organization?
   A link to your organization's website?
 
+You can contact us in the [#feedback stream of the Zulip development
+community](https://chat.zulip.org/#narrow/stream/137-feedback) or
+by emailing [support@zulip.com](mailto:support@zulip.com).
+
 ## Outreach programs
 
 Zulip participates in [Google Summer of Code
@@ -278,70 +344,62 @@ important parts of the project. We hope you apply!
 ### Google Summer of Code
 
 The largest outreach program Zulip participates in is GSoC (14
-students in 2017; 11 in 2018; 17 in 2019; 18 in 2020). While we don't control how
+students in 2017; 11 in 2018; 17 in 2019; 18 in 2020; 18 in 2021). While we
+don't control how
 many slots Google allocates to Zulip, we hope to mentor a similar
-number of students in future summers.
+number of students in future summers. Check out our [blog
+post](https://blog.zulip.com/2021/09/30/google-summer-of-code-2021/) to learn
+about the GSoC 2021 experience and our participants' accomplishments.
 
 If you're reading this well before the application deadline and want
 to make your application strong, we recommend getting involved in the
 community and fixing issues in Zulip now. Having good contributions
 and building a reputation for doing good work is the best way to have
-a strong application. About half of Zulip's GSoC students for Summer
-2017 had made significant contributions to the project by February
-2017, and about half had not. Our
-[GSoC project ideas page][gsoc-guide] has lots more details on how
-Zulip does GSoC, as well as project ideas (though the project idea
+a strong application.
+
+Our [GSoC project ideas page][gsoc-guide] has lots more details on how
+Zulip does GSoC, as well as project ideas. Note, however, that the project idea
 list is maintained only during the GSoC application period, so if
 you're looking at some other time of year, the project list is likely
-out-of-date).
+out-of-date.
 
-We also have in some past years run a Zulip Summer of Code (ZSoC)
-program for students who we didn't have enough slots to accept for
-GSoC but were able to find funding for. Student expectations are the
-same as with GSoC, and it has no separate application process; your
+In some years, we have also run a Zulip Summer of Code (ZSoC)
+program for students who we wanted to accept into GSoC but did not have an
+official slot for. Student expectations are the
+same as with GSoC, and ZSoC has no separate application process; your
 GSoC application is your ZSoC application. If we'd like to select you
 for ZSoC, we'll contact you when the GSoC results are announced.
 
 [gsoc-guide]: https://zulip.readthedocs.io/en/latest/contributing/gsoc-ideas.html
 [gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
 
-## Zulip outreach
+## Stay connected
 
-**Upvoting Zulip**. Upvotes and reviews make a big difference in the public
-perception of projects like Zulip. We've collected a few sites below
-where we know Zulip has been discussed. Doing everything in the following
-list typically takes about 15 minutes.
+Even if you are not logging into the development community on a regular basis,
+you can still stay connected with the project.
+
+- Follow us [on Twitter](https://twitter.com/zulip).
+- Subscribe to [our blog](https://blog.zulip.org/).
+- Join or follow the project [on LinkedIn](https://www.linkedin.com/company/zulip-project/).
+
+## Help others find Zulip
+
+Here are some ways you can help others find Zulip:
 
 - Star us on GitHub. There are four main repositories:
   [server/web](https://github.com/zulip/zulip),
   [mobile](https://github.com/zulip/zulip-mobile),
   [desktop](https://github.com/zulip/zulip-desktop), and
   [Python API](https://github.com/zulip/python-zulip-api).
-- [Follow us](https://twitter.com/zulip) on Twitter.
 
-For both of the following, you'll need to make an account on the site if you
-don't already have one.
+- "Like" and retweet [our tweets](https://twitter.com/zulip).
 
-- [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
-  AlternativeTo. We recommend upvoting a couple of other products you like
-  as well, both to give back to their community, and since single-upvote
-  accounts are generally given less weight. You can also
+- Upvote and post feedback on Zulip on comparison websites. A couple specific
+  ones to highlight:
+
+  - [AlternativeTo](https://alternativeto.net/software/zulip-chat-server/). You can also
     [upvote Zulip](https://alternativeto.net/software/slack/) on their page
     for Slack.
   - [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
     it, and upvote the reasons why people like Zulip that you find most
-  compelling. Again, we recommend adding a few other products that you like
-  as well.
-
-We have a doc with more detailed instructions and a few other sites, if you
-have been using Zulip for a while and want to contribute more.
-
-**Blog posts**. Writing a blog post about your experiences with Zulip, or
-about a technical aspect of Zulip can be a great way to spread the word
-about Zulip.
-
-We also occasionally [publish](https://blog.zulip.org/) long-form
-articles related to Zulip. Our posts typically get tens of thousands
-of views, and we always have good ideas for blog posts that we can
-outline but don't have time to write. If you are an experienced writer
-or copyeditor, send us a portfolio; we'd love to talk!
+    compelling.

analytics/tests/test_support_views.py

@@ -466,7 +466,7 @@ class TestSupportEndpoint(ZulipTestCase):
         )
         self.assert_in_success_response(["Sponsorship approved for lear"], result)
         lear_realm.refresh_from_db()
-        self.assertEqual(lear_realm.plan_type, Realm.STANDARD_FREE)
+        self.assertEqual(lear_realm.plan_type, Realm.PLAN_TYPE_STANDARD_FREE)
         customer = get_customer_by_realm(lear_realm)
         assert customer is not None
         self.assertFalse(customer.sponsorship_pending)

analytics/views/installation_activity.py

@@ -223,11 +223,14 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
             if string_id in estimated_arrs:
                 row["arr"] = estimated_arrs[string_id]
 
-            if row["plan_type"] in [Realm.STANDARD, Realm.PLUS]:
+            if row["plan_type"] in [Realm.PLAN_TYPE_STANDARD, Realm.PLAN_TYPE_PLUS]:
                 row["effective_rate"] = 100 - int(realms_to_default_discount.get(string_id, 0))
-            elif row["plan_type"] == Realm.STANDARD_FREE:
+            elif row["plan_type"] == Realm.PLAN_TYPE_STANDARD_FREE:
                 row["effective_rate"] = 0
-            elif row["plan_type"] == Realm.LIMITED and string_id in realms_to_default_discount:
+            elif (
+                row["plan_type"] == Realm.PLAN_TYPE_LIMITED
+                and string_id in realms_to_default_discount
+            ):
                 row["effective_rate"] = 100 - int(realms_to_default_discount[string_id])
             else:
                 row["effective_rate"] = ""

analytics/views/support.py

@@ -59,11 +59,11 @@ if settings.BILLING_ENABLED:
 
 def get_plan_name(plan_type: int) -> str:
     return {
-        Realm.SELF_HOSTED: "self hosted",
-        Realm.LIMITED: "limited",
-        Realm.STANDARD: "standard",
-        Realm.STANDARD_FREE: "open source",
-        Realm.PLUS: "plus",
+        Realm.PLAN_TYPE_SELF_HOSTED: "self hosted",
+        Realm.PLAN_TYPE_LIMITED: "limited",
+        Realm.PLAN_TYPE_STANDARD: "standard",
+        Realm.PLAN_TYPE_STANDARD_FREE: "open source",
+        Realm.PLAN_TYPE_PLUS: "plus",
     }[plan_type]
 
 

confirmation/models.py

@@ -43,10 +43,10 @@ def render_confirmation_key_error(
     request: HttpRequest, exception: ConfirmationKeyException
 ) -> HttpResponse:
     if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
-        return render(request, "confirmation/link_malformed.html")
+        return render(request, "confirmation/link_malformed.html", status=404)
     if exception.error_type == ConfirmationKeyException.EXPIRED:
-        return render(request, "confirmation/link_expired.html")
-    return render(request, "confirmation/link_does_not_exist.html")
+        return render(request, "confirmation/link_expired.html", status=404)
+    return render(request, "confirmation/link_does_not_exist.html", status=404)
 
 
 def generate_key() -> str:
@@ -170,9 +170,9 @@ class ConfirmationType:
 
 
 _properties = {
-    Confirmation.USER_REGISTRATION: ConfirmationType("check_prereg_key_and_redirect"),
+    Confirmation.USER_REGISTRATION: ConfirmationType("get_prereg_key_and_redirect"),
     Confirmation.INVITATION: ConfirmationType(
-        "check_prereg_key_and_redirect", validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS
+        "get_prereg_key_and_redirect", validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS
     ),
     Confirmation.EMAIL_CHANGE: ConfirmationType("confirm_email_change"),
     Confirmation.UNSUBSCRIBE: ConfirmationType(
@@ -182,7 +182,7 @@ _properties = {
     Confirmation.MULTIUSE_INVITE: ConfirmationType(
         "join", validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS
     ),
-    Confirmation.REALM_CREATION: ConfirmationType("check_prereg_key_and_redirect"),
+    Confirmation.REALM_CREATION: ConfirmationType("get_prereg_key_and_redirect"),
     Confirmation.REALM_REACTIVATION: ConfirmationType("realm_reactivation"),
 }
 

corporate/lib/stripe.py

@@ -34,7 +34,6 @@ from zerver.lib.utils import assert_is_not_none
 from zerver.models import Realm, RealmAuditLog, UserProfile, get_system_bot
 from zproject.config import get_secret
 
-STRIPE_PUBLISHABLE_KEY = get_secret("stripe_publishable_key")
 stripe.api_key = get_secret("stripe_secret_key")
 
 BILLING_LOG_PATH = os.path.join(
@@ -51,6 +50,9 @@ MIN_INVOICED_LICENSES = 30
 MAX_INVOICED_LICENSES = 1000
 DEFAULT_INVOICE_DAYS_UNTIL_DUE = 30
 
+# The version of Stripe API the billing system supports.
+STRIPE_API_VERSION = "2020-08-27"
+
 
 def get_latest_seat_count(realm: Realm) -> int:
     non_guests = (
@@ -220,6 +222,14 @@ class StripeConnectionError(BillingError):
     pass
 
 
+class UpgradeWithExistingPlanError(BillingError):
+    def __init__(self) -> None:
+        super().__init__(
+            "subscribing with existing subscription",
+            "The organization is already subscribed to a plan. Please reload the billing page.",
+        )
+
+
 class InvalidBillingSchedule(Exception):
     def __init__(self, billing_schedule: int) -> None:
         self.message = f"Unknown billing_schedule: {billing_schedule}"
@@ -235,13 +245,6 @@ class InvalidTier(Exception):
 def catch_stripe_errors(func: CallableT) -> CallableT:
     @wraps(func)
     def wrapped(*args: object, **kwargs: object) -> object:
-        if settings.DEVELOPMENT and not settings.TEST_SUITE:  # nocoverage
-            if STRIPE_PUBLISHABLE_KEY is None:
-                raise BillingError(
-                    "missing stripe config",
-                    "Missing Stripe config. "
-                    "See https://zulip.readthedocs.io/en/latest/subsystems/billing.html.",
-                )
         try:
             return func(*args, **kwargs)
         # See https://stripe.com/docs/api/python#error_handling, though
@@ -280,11 +283,13 @@ def catch_stripe_errors(func: CallableT) -> CallableT:
 
 @catch_stripe_errors
 def stripe_get_customer(stripe_customer_id: str) -> stripe.Customer:
-    return stripe.Customer.retrieve(stripe_customer_id, expand=["default_source", "sources"])
+    return stripe.Customer.retrieve(
+        stripe_customer_id, expand=["invoice_settings", "invoice_settings.default_payment_method"]
+    )
 
 
 @catch_stripe_errors
-def do_create_stripe_customer(user: UserProfile, stripe_token: Optional[str] = None) -> Customer:
+def do_create_stripe_customer(user: UserProfile, payment_method: Optional[str] = None) -> Customer:
     realm = user.realm
     # We could do a better job of handling race conditions here, but if two
     # people from a realm try to upgrade at exactly the same time, the main
@@ -294,7 +299,10 @@ def do_create_stripe_customer(user: UserProfile, stripe_token: Optional[str] = N
         description=f"{realm.string_id} ({realm.name})",
         email=user.delivery_email,
         metadata={"realm_id": realm.id, "realm_str": realm.string_id},
-        source=stripe_token,
+        payment_method=payment_method,
+    )
+    stripe.Customer.modify(
+        stripe_customer.id, invoice_settings={"default_payment_method": payment_method}
     )
     event_time = timestamp_to_datetime(stripe_customer.created)
     with transaction.atomic():
@@ -304,7 +312,7 @@ def do_create_stripe_customer(user: UserProfile, stripe_token: Optional[str] = N
             event_type=RealmAuditLog.STRIPE_CUSTOMER_CREATED,
             event_time=event_time,
         )
-        if stripe_token is not None:
+        if payment_method is not None:
             RealmAuditLog.objects.create(
                 realm=user.realm,
                 acting_user=user,
@@ -321,17 +329,17 @@ def do_create_stripe_customer(user: UserProfile, stripe_token: Optional[str] = N
 
 
 @catch_stripe_errors
-def do_replace_payment_source(
-    user: UserProfile, stripe_token: str, pay_invoices: bool = False
-) -> stripe.Customer:
+def do_replace_payment_method(
+    user: UserProfile, payment_method: str, pay_invoices: bool = False
+) -> None:
     customer = get_customer_by_realm(user.realm)
     assert customer is not None  # for mypy
     assert customer.stripe_customer_id is not None  # for mypy
 
-    stripe_customer = stripe_get_customer(customer.stripe_customer_id)
-    stripe_customer.source = stripe_token
-    # Deletes existing card: https://stripe.com/docs/api#update_customer-source
-    updated_stripe_customer = stripe.Customer.save(stripe_customer)
+    stripe.Customer.modify(
+        customer.stripe_customer_id, invoice_settings={"default_payment_method": payment_method}
+    )
+
     RealmAuditLog.objects.create(
         realm=user.realm,
         acting_user=user,
@@ -340,27 +348,30 @@ def do_replace_payment_source(
     )
     if pay_invoices:
         for stripe_invoice in stripe.Invoice.list(
-            collection_method="charge_automatically", customer=stripe_customer.id, status="open"
+            collection_method="charge_automatically",
+            customer=customer.stripe_customer_id,
+            status="open",
         ):
             # The user will get either a receipt or a "failed payment" email, but the in-app
             # messaging could be clearer here (e.g. it could explicitly tell the user that there
             # were payment(s) and that they succeeded or failed).
             # Worth fixing if we notice that a lot of cards end up failing at this step.
             stripe.Invoice.pay(stripe_invoice)
-    return updated_stripe_customer
 
 
-def stripe_customer_has_credit_card_as_default_source(stripe_customer: stripe.Customer) -> bool:
-    if not stripe_customer.default_source:
+def stripe_customer_has_credit_card_as_default_payment_method(
+    stripe_customer: stripe.Customer,
+) -> bool:
+    if not stripe_customer.invoice_settings.default_payment_method:
         return False
-    return stripe_customer.default_source.object == "card"
+    return stripe_customer.invoice_settings.default_payment_method.type == "card"
 
 
-def customer_has_credit_card_as_default_source(customer: Customer) -> bool:
+def customer_has_credit_card_as_default_payment_method(customer: Customer) -> bool:
     if not customer.stripe_customer_id:
         return False
     stripe_customer = stripe_get_customer(customer.stripe_customer_id)
-    return stripe_customer_has_credit_card_as_default_source(stripe_customer)
+    return stripe_customer_has_credit_card_as_default_payment_method(stripe_customer)
 
 
 # event_time should roughly be timezone_now(). Not designed to handle
@@ -443,8 +454,11 @@ def make_end_of_cycle_updates_if_needed(
                 licenses_at_next_renewal=licenses_at_next_renewal,
             )
 
+            realm = new_plan.customer.realm
+            assert realm is not None
+
             RealmAuditLog.objects.create(
-                realm=new_plan.customer.realm,
+                realm=realm,
                 event_time=event_time,
                 event_type=RealmAuditLog.CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN,
                 extra_data=orjson.dumps(
@@ -505,15 +519,16 @@ def make_end_of_cycle_updates_if_needed(
 
 # Returns Customer instead of stripe_customer so that we don't make a Stripe
 # API call if there's nothing to update
+@catch_stripe_errors
 def update_or_create_stripe_customer(
-    user: UserProfile, stripe_token: Optional[str] = None
+    user: UserProfile, payment_method: Optional[str] = None
 ) -> Customer:
     realm = user.realm
     customer = get_customer_by_realm(realm)
     if customer is None or customer.stripe_customer_id is None:
-        return do_create_stripe_customer(user, stripe_token=stripe_token)
-    if stripe_token is not None:
-        do_replace_payment_source(user, stripe_token)
+        return do_create_stripe_customer(user, payment_method=payment_method)
+    if payment_method is not None:
+        do_replace_payment_method(user, payment_method, True)
     return customer
 
 
@@ -592,6 +607,18 @@ def is_free_trial_offer_enabled() -> bool:
     return settings.FREE_TRIAL_DAYS not in (None, 0)
 
 
+def ensure_realm_does_not_have_active_plan(realm: Customer) -> None:
+    if get_current_plan_by_realm(realm) is not None:
+        # Unlikely race condition from two people upgrading (clicking "Make payment")
+        # at exactly the same time. Doesn't fully resolve the race condition, but having
+        # a check here reduces the likelihood.
+        billing_logger.warning(
+            "Upgrade of %s failed because of existing active plan.",
+            realm.string_id,
+        )
+        raise UpgradeWithExistingPlanError()
+
+
 # Only used for cloud signups
 @catch_stripe_errors
 def process_initial_upgrade(
@@ -599,27 +626,14 @@ def process_initial_upgrade(
     licenses: int,
     automanage_licenses: bool,
     billing_schedule: int,
-    stripe_token: Optional[str],
+    charge_automatically: bool,
+    free_trial: bool,
 ) -> None:
     realm = user.realm
-    customer = update_or_create_stripe_customer(user, stripe_token=stripe_token)
+    customer = update_or_create_stripe_customer(user)
     assert customer.stripe_customer_id is not None  # for mypy
-
-    charge_automatically = stripe_token is not None
-    free_trial = is_free_trial_offer_enabled()
-
-    if get_current_plan_by_customer(customer) is not None:
-        # Unlikely race condition from two people upgrading (clicking "Make payment")
-        # at exactly the same time. Doesn't fully resolve the race condition, but having
-        # a check here reduces the likelihood.
-        billing_logger.warning(
-            "Customer %s trying to upgrade, but has an active subscription",
-            customer,
-        )
-        raise BillingError(
-            "subscribing with existing subscription", str(BillingError.TRY_RELOADING)
-        )
-
+    assert customer.realm is not None
+    ensure_realm_does_not_have_active_plan(customer.realm)
     (
         billing_cycle_anchor,
         next_invoice_date,
@@ -632,32 +646,6 @@ def process_initial_upgrade(
         customer.default_discount,
         free_trial,
     )
-    # The main design constraint in this function is that if you upgrade with a credit card, and the
-    # charge fails, everything should be rolled back as if nothing had happened. This is because we
-    # expect frequent card failures on initial signup.
-    # Hence, if we're going to charge a card, do it at the beginning, even if we later may have to
-    # adjust the number of licenses.
-    if charge_automatically:
-        if not free_trial:
-            stripe_charge = stripe.Charge.create(
-                amount=price_per_license * licenses,
-                currency="usd",
-                customer=customer.stripe_customer_id,
-                description=f"Upgrade to Zulip Standard, ${price_per_license/100} x {licenses}",
-                receipt_email=user.delivery_email,
-                statement_descriptor="Zulip Standard",
-            )
-            # Not setting a period start and end, but maybe we should? Unclear what will make things
-            # most similar to the renewal case from an accounting perspective.
-            assert isinstance(stripe_charge.source, stripe.Card)
-            description = f"Payment (Card ending in {stripe_charge.source.last4})"
-            stripe.InvoiceItem.create(
-                amount=price_per_license * licenses * -1,
-                currency="usd",
-                customer=customer.stripe_customer_id,
-                description=description,
-                discountable=False,
-            )
 
     # TODO: The correctness of this relies on user creation, deactivation, etc being
     # in a transaction.atomic() with the relevant RealmAuditLog entries
@@ -728,7 +716,7 @@ def process_initial_upgrade(
 
     from zerver.lib.actions import do_change_plan_type
 
-    do_change_plan_type(realm, Realm.STANDARD, acting_user=user)
+    do_change_plan_type(realm, Realm.PLAN_TYPE_STANDARD, acting_user=user)
 
 
 def update_license_ledger_for_manual_plan(
@@ -738,12 +726,14 @@ def update_license_ledger_for_manual_plan(
     licenses_at_next_renewal: Optional[int] = None,
 ) -> None:
     if licenses is not None:
+        assert plan.customer.realm is not None
         assert get_latest_seat_count(plan.customer.realm) <= licenses
         assert licenses > plan.licenses()
         LicenseLedger.objects.create(
             plan=plan, event_time=event_time, licenses=licenses, licenses_at_next_renewal=licenses
         )
     elif licenses_at_next_renewal is not None:
+        assert plan.customer.realm is not None
         assert get_latest_seat_count(plan.customer.realm) <= licenses_at_next_renewal
         LicenseLedger.objects.create(
             plan=plan,
@@ -795,6 +785,7 @@ def invoice_plan(plan: CustomerPlan, event_time: datetime) -> None:
     if plan.invoicing_status == CustomerPlan.STARTED:
         raise NotImplementedError("Plan with invoicing_status==STARTED needs manual resolution.")
     if not plan.customer.stripe_customer_id:
+        assert plan.customer.realm is not None
         raise BillingError(
             f"Realm {plan.customer.realm.string_id} has a paid plan without a Stripe customer."
         )
@@ -948,7 +939,7 @@ def update_sponsorship_status(
 def approve_sponsorship(realm: Realm, *, acting_user: Optional[UserProfile]) -> None:
     from zerver.lib.actions import do_change_plan_type, internal_send_private_message
 
-    do_change_plan_type(realm, Realm.STANDARD_FREE, acting_user=acting_user)
+    do_change_plan_type(realm, Realm.PLAN_TYPE_STANDARD_FREE, acting_user=acting_user)
     customer = get_customer_by_realm(realm)
     if customer is not None and customer.sponsorship_pending:
         customer.sponsorship_pending = False
@@ -973,7 +964,7 @@ def approve_sponsorship(realm: Realm, *, acting_user: Optional[UserProfile]) ->
 
 
 def is_sponsored_realm(realm: Realm) -> bool:
-    return realm.plan_type == Realm.STANDARD_FREE
+    return realm.plan_type == Realm.PLAN_TYPE_STANDARD_FREE
 
 
 def get_discount_for_realm(realm: Realm) -> Optional[Decimal]:
@@ -997,7 +988,8 @@ def do_change_plan_status(plan: CustomerPlan, status: int) -> None:
 def process_downgrade(plan: CustomerPlan) -> None:
     from zerver.lib.actions import do_change_plan_type
 
-    do_change_plan_type(plan.customer.realm, Realm.LIMITED, acting_user=None)
+    assert plan.customer.realm is not None
+    do_change_plan_type(plan.customer.realm, Realm.PLAN_TYPE_LIMITED, acting_user=None)
     plan.status = CustomerPlan.ENDED
     plan.save(update_fields=["status"])
 

corporate/lib/stripe_event_handler.py

@@ -0,0 +1,178 @@
+import logging
+from typing import Any, Callable, Union
+
+import stripe
+from django.conf import settings
+
+from corporate.lib.stripe import (
+    BillingError,
+    UpgradeWithExistingPlanError,
+    ensure_realm_does_not_have_active_plan,
+    process_initial_upgrade,
+    update_or_create_stripe_customer,
+)
+from corporate.models import Event, PaymentIntent, Session
+from zerver.models import get_user_by_delivery_email
+
+billing_logger = logging.getLogger("corporate.stripe")
+
+
+def error_handler(
+    func: Callable[[Any, Any], None],
+) -> Callable[[Union[stripe.checkout.Session, stripe.PaymentIntent], Event], None]:
+    def wrapper(
+        stripe_object: Union[stripe.checkout.Session, stripe.PaymentIntent], event: Event
+    ) -> None:
+        event.status = Event.EVENT_HANDLER_STARTED
+        event.save(update_fields=["status"])
+
+        try:
+            func(stripe_object, event.content_object)
+        except BillingError as e:
+            billing_logger.warning(
+                "BillingError in %s event handler: %s. stripe_object_id=%s, customer_id=%s metadata=%s",
+                event.type,
+                e.error_description,
+                stripe_object.id,
+                stripe_object.customer,
+                stripe_object.metadata,
+            )
+            event.status = Event.EVENT_HANDLER_FAILED
+            event.handler_error = {
+                "message": e.msg,
+                "description": e.error_description,
+            }
+            event.save(update_fields=["status", "handler_error"])
+        except Exception:
+            billing_logger.exception(
+                "Uncaught exception in %s event handler:",
+                event.type,
+                stack_info=True,
+            )
+            event.status = Event.EVENT_HANDLER_FAILED
+            event.handler_error = {
+                "description": f"uncaught exception in {event.type} event handler",
+                "message": BillingError.CONTACT_SUPPORT.format(email=settings.ZULIP_ADMINISTRATOR),
+            }
+            event.save(update_fields=["status", "handler_error"])
+        else:
+            event.status = Event.EVENT_HANDLER_SUCCEEDED
+            event.save()
+
+    return wrapper
+
+
+@error_handler
+def handle_checkout_session_completed_event(
+    stripe_session: stripe.checkout.Session, session: Session
+) -> None:
+    session.status = Session.COMPLETED
+    session.save()
+
+    stripe_setup_intent = stripe.SetupIntent.retrieve(stripe_session.setup_intent)
+    stripe_customer = stripe.Customer.retrieve(stripe_setup_intent.customer)
+    assert session.customer.realm is not None
+    user = get_user_by_delivery_email(stripe_customer.email, session.customer.realm)
+    payment_method = stripe_setup_intent.payment_method
+
+    if session.type in [
+        Session.UPGRADE_FROM_BILLING_PAGE,
+        Session.RETRY_UPGRADE_WITH_ANOTHER_PAYMENT_METHOD,
+    ]:
+        ensure_realm_does_not_have_active_plan(user.realm)
+        update_or_create_stripe_customer(user, payment_method)
+        session.payment_intent.status = PaymentIntent.PROCESSING
+        session.payment_intent.last_payment_error = ()
+        session.payment_intent.save(update_fields=["status", "last_payment_error"])
+        try:
+            stripe.PaymentIntent.confirm(
+                session.payment_intent.stripe_payment_intent_id,
+                payment_method=payment_method,
+                off_session=True,
+            )
+        except stripe.error.CardError:
+            pass
+    elif session.type in [
+        Session.FREE_TRIAL_UPGRADE_FROM_BILLING_PAGE,
+        Session.FREE_TRIAL_UPGRADE_FROM_ONBOARDING_PAGE,
+    ]:
+        ensure_realm_does_not_have_active_plan(user.realm)
+        update_or_create_stripe_customer(user, payment_method)
+        process_initial_upgrade(
+            user,
+            int(stripe_setup_intent.metadata["licenses"]),
+            stripe_setup_intent.metadata["license_management"] == "automatic",
+            int(stripe_setup_intent.metadata["billing_schedule"]),
+            charge_automatically=True,
+            free_trial=True,
+        )
+    elif session.type in [Session.CARD_UPDATE_FROM_BILLING_PAGE]:
+        update_or_create_stripe_customer(user, payment_method)
+
+
+@error_handler
+def handle_payment_intent_succeeded_event(
+    stripe_payment_intent: stripe.PaymentIntent, payment_intent: PaymentIntent
+) -> None:
+    payment_intent.status = PaymentIntent.SUCCEEDED
+    payment_intent.save()
+    metadata = stripe_payment_intent.metadata
+    assert payment_intent.customer.realm is not None
+    user = get_user_by_delivery_email(metadata["user_email"], payment_intent.customer.realm)
+
+    description = ""
+    for charge in stripe_payment_intent.charges:
+        description = f"Payment (Card ending in {charge.payment_method_details.card.last4})"
+        break
+
+    stripe.InvoiceItem.create(
+        amount=stripe_payment_intent.amount * -1,
+        currency="usd",
+        customer=stripe_payment_intent.customer,
+        description=description,
+        discountable=False,
+    )
+    try:
+        ensure_realm_does_not_have_active_plan(user.realm)
+    except UpgradeWithExistingPlanError as e:
+        stripe_invoice = stripe.Invoice.create(
+            auto_advance=True,
+            collection_method="charge_automatically",
+            customer=stripe_payment_intent.customer,
+            days_until_due=None,
+            statement_descriptor="Zulip Standard Credit",
+        )
+        stripe.Invoice.finalize_invoice(stripe_invoice)
+        raise e
+
+    process_initial_upgrade(
+        user,
+        int(metadata["licenses"]),
+        metadata["license_management"] == "automatic",
+        int(metadata["billing_schedule"]),
+        True,
+        False,
+    )
+
+
+@error_handler
+def handle_payment_intent_payment_failed_event(
+    stripe_payment_intent: stripe.PaymentIntent, payment_intent: Event
+) -> None:
+    payment_intent.status = PaymentIntent.get_status_integer_from_status_text(
+        stripe_payment_intent.status
+    )
+    billing_logger.info(
+        "Stripe payment intent failed: %s %s %s %s",
+        payment_intent.customer.realm.string_id,
+        stripe_payment_intent.last_payment_error.get("type"),
+        stripe_payment_intent.last_payment_error.get("code"),
+        stripe_payment_intent.last_payment_error.get("param"),
+    )
+    payment_intent.last_payment_error = {
+        "description": stripe_payment_intent.last_payment_error.get("type"),
+    }
+    payment_intent.last_payment_error["message"] = stripe_payment_intent.last_payment_error.get(
+        "message"
+    )
+    payment_intent.save(update_fields=["status", "last_payment_error"])

corporate/migrations/0015_event_paymentintent_session.py

@@ -0,0 +1,85 @@
+# Generated by Django 3.2.9 on 2021-11-04 16:23
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("corporate", "0014_customerplan_end_date"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PaymentIntent",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("stripe_payment_intent_id", models.CharField(max_length=255, unique=True)),
+                ("status", models.SmallIntegerField()),
+                ("last_payment_error", models.JSONField(default=None, null=True)),
+                (
+                    "customer",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="corporate.customer"
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Session",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("stripe_session_id", models.CharField(max_length=255, unique=True)),
+                ("type", models.SmallIntegerField()),
+                ("status", models.SmallIntegerField(default=1)),
+                (
+                    "customer",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="corporate.customer"
+                    ),
+                ),
+                (
+                    "payment_intent",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="corporate.paymentintent",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("stripe_event_id", models.CharField(max_length=255)),
+                ("type", models.CharField(max_length=255)),
+                ("status", models.SmallIntegerField(default=1)),
+                ("object_id", models.PositiveIntegerField(db_index=True)),
+                ("handler_error", models.JSONField(default=None, null=True)),
+                (
+                    "content_type",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="contenttypes.contenttype"
+                    ),
+                ),
+            ],
+        ),
+    ]

corporate/migrations/0016_customer_add_remote_server_field.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.9 on 2021-11-27 00:08
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("zilencer", "0018_remoterealmauditlog"),
+        ("zerver", "0370_realm_enable_spectator_access"),
+        ("corporate", "0015_event_paymentintent_session"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="customer",
+            name="remote_server",
+            field=models.OneToOneField(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="zilencer.remotezulipserver",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="customer",
+            name="realm",
+            field=models.OneToOneField(
+                null=True, on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+            ),
+        ),
+    ]

corporate/models.py

@@ -1,11 +1,14 @@
 import datetime
 from decimal import Decimal
-from typing import Optional
+from typing import Any, Dict, Optional, Union
 
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.db.models import CASCADE
 
 from zerver.models import Realm, UserProfile
+from zilencer.models import RemoteZulipServer
 
 
 class Customer(models.Model):
@@ -15,7 +18,10 @@ class Customer(models.Model):
     and the active plan, if any.
     """
 
-    realm: Realm = models.OneToOneField(Realm, on_delete=CASCADE)
+    realm: Optional[Realm] = models.OneToOneField(Realm, on_delete=CASCADE, null=True)
+    remote_server: Optional[RemoteZulipServer] = models.OneToOneField(
+        RemoteZulipServer, on_delete=CASCADE, null=True
+    )
     stripe_customer_id: Optional[str] = models.CharField(max_length=255, null=True, unique=True)
     sponsorship_pending: bool = models.BooleanField(default=False)
     # A percentage, like 85.
@@ -28,6 +34,20 @@ class Customer(models.Model):
     # they purchased.
     exempt_from_from_license_number_check: bool = models.BooleanField(default=False)
 
+    @property
+    def is_self_hosted(self) -> bool:
+        is_self_hosted = self.remote_server is not None
+        if is_self_hosted:
+            assert self.realm is None
+        return is_self_hosted
+
+    @property
+    def is_cloud(self) -> bool:
+        is_cloud = self.realm is not None
+        if is_cloud:
+            assert self.remote_server is None
+        return is_cloud
+
     def __str__(self) -> str:
         return f"<Customer {self.realm} {self.stripe_customer_id}>"
 
@@ -36,6 +56,141 @@ def get_customer_by_realm(realm: Realm) -> Optional[Customer]:
     return Customer.objects.filter(realm=realm).first()
 
 
+class Event(models.Model):
+    stripe_event_id = models.CharField(max_length=255)
+
+    type = models.CharField(max_length=255)
+
+    RECEIVED = 1
+    EVENT_HANDLER_STARTED = 30
+    EVENT_HANDLER_FAILED = 40
+    EVENT_HANDLER_SUCCEEDED = 50
+    status = models.SmallIntegerField(default=RECEIVED)
+
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    object_id = models.PositiveIntegerField(db_index=True)
+    content_object = GenericForeignKey("content_type", "object_id")
+
+    handler_error = models.JSONField(default=None, null=True)
+
+    def get_event_handler_details_as_dict(self) -> Dict[str, Any]:
+        details_dict = {}
+        details_dict["status"] = {
+            Event.RECEIVED: "not_started",
+            Event.EVENT_HANDLER_STARTED: "started",
+            Event.EVENT_HANDLER_FAILED: "failed",
+            Event.EVENT_HANDLER_SUCCEEDED: "succeeded",
+        }[self.status]
+        if self.handler_error:
+            details_dict["error"] = self.handler_error
+        return details_dict
+
+
+def get_last_associated_event_by_type(
+    content_object: Union["PaymentIntent", "Session"], event_type: str
+) -> Optional[Event]:
+    content_type = ContentType.objects.get_for_model(type(content_object))
+    return Event.objects.filter(
+        content_type=content_type, object_id=content_object.id, type=event_type
+    ).last()
+
+
+class Session(models.Model):
+    customer: Customer = models.ForeignKey(Customer, on_delete=CASCADE)
+    stripe_session_id: str = models.CharField(max_length=255, unique=True)
+    payment_intent = models.ForeignKey("PaymentIntent", null=True, on_delete=CASCADE)
+
+    UPGRADE_FROM_BILLING_PAGE = 1
+    RETRY_UPGRADE_WITH_ANOTHER_PAYMENT_METHOD = 10
+    FREE_TRIAL_UPGRADE_FROM_BILLING_PAGE = 20
+    FREE_TRIAL_UPGRADE_FROM_ONBOARDING_PAGE = 30
+    CARD_UPDATE_FROM_BILLING_PAGE = 40
+    type: int = models.SmallIntegerField()
+
+    CREATED = 1
+    COMPLETED = 10
+    status: int = models.SmallIntegerField(default=CREATED)
+
+    def get_status_as_string(self) -> str:
+        return {Session.CREATED: "created", Session.COMPLETED: "completed"}[self.status]
+
+    def get_type_as_string(self) -> str:
+        return {
+            Session.UPGRADE_FROM_BILLING_PAGE: "upgrade_from_billing_page",
+            Session.RETRY_UPGRADE_WITH_ANOTHER_PAYMENT_METHOD: "retry_upgrade_with_another_payment_method",
+            Session.FREE_TRIAL_UPGRADE_FROM_BILLING_PAGE: "free_trial_upgrade_from_billing_page",
+            Session.FREE_TRIAL_UPGRADE_FROM_ONBOARDING_PAGE: "free_trial_upgrade_from_onboarding_page",
+            Session.CARD_UPDATE_FROM_BILLING_PAGE: "card_update_from_billing_page",
+        }[self.type]
+
+    def to_dict(self) -> Dict[str, Any]:
+        session_dict: Dict[str, Any] = {}
+
+        session_dict["status"] = self.get_status_as_string()
+        session_dict["type"] = self.get_type_as_string()
+        if self.payment_intent:
+            session_dict["stripe_payment_intent_id"] = self.payment_intent.stripe_payment_intent_id
+        event = self.get_last_associated_event()
+        if event is not None:
+            session_dict["event_handler"] = event.get_event_handler_details_as_dict()
+        return session_dict
+
+    def get_last_associated_event(self) -> Optional[Event]:
+        if self.status == Session.CREATED:
+            return None
+        return get_last_associated_event_by_type(self, "checkout.session.completed")
+
+
+class PaymentIntent(models.Model):
+    customer: Customer = models.ForeignKey(Customer, on_delete=CASCADE)
+    stripe_payment_intent_id: str = models.CharField(max_length=255, unique=True)
+
+    REQUIRES_PAYMENT_METHOD = 1
+    REQUIRES_CONFIRMATION = 20
+    REQUIRES_ACTION = 30
+    PROCESSING = 40
+    REQUIRES_CAPTURE = 50
+    CANCELLED = 60
+    SUCCEEDED = 70
+
+    status: int = models.SmallIntegerField()
+    last_payment_error = models.JSONField(default=None, null=True)
+
+    @classmethod
+    def get_status_integer_from_status_text(cls, status_text: str) -> int:
+        return getattr(cls, status_text.upper())
+
+    def get_status_as_string(self) -> str:
+        return {
+            PaymentIntent.REQUIRES_PAYMENT_METHOD: "requires_payment_method",
+            PaymentIntent.REQUIRES_CONFIRMATION: "requires_confirmation",
+            PaymentIntent.REQUIRES_ACTION: "requires_action",
+            PaymentIntent.PROCESSING: "processing",
+            PaymentIntent.REQUIRES_CAPTURE: "requires_capture",
+            PaymentIntent.CANCELLED: "cancelled",
+            PaymentIntent.SUCCEEDED: "succeeded",
+        }[self.status]
+
+    def get_last_associated_event(self) -> Optional[Event]:
+        if self.status == PaymentIntent.SUCCEEDED:
+            event_type = "payment_intent.succeeded"
+        elif self.status == PaymentIntent.REQUIRES_PAYMENT_METHOD:
+            event_type = "payment_intent.payment_failed"
+        else:
+            return None
+        return get_last_associated_event_by_type(self, event_type)
+
+    def to_dict(self) -> Dict[str, Any]:
+        payment_intent_dict: Dict[str, Any] = {}
+        payment_intent_dict["status"] = self.get_status_as_string()
+        event = self.get_last_associated_event()
+        if self.last_payment_error:
+            payment_intent_dict["last_payment_error"] = self.last_payment_error
+        if event is not None:
+            payment_intent_dict["event_handler"] = event.get_event_handler_details_as_dict()
+        return payment_intent_dict
+
+
 class CustomerPlan(models.Model):
     """
     This is for storing most of the fiddly details