billing: Downgrade small realms that are behind on payments.

An organization with at most 5 users that is behind on payments isn't
worth spending time on investigating the situation.

For larger organizations, we likely want somewhat different logic that
at least does not void invoices.

corporate/lib/stripe.py

@@ -12,6 +12,7 @@ import stripe
 from django.conf import settings
 from django.core.signing import Signer
 from django.db import transaction
+from django.urls import reverse
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
 from django.utils.translation import gettext_lazy
@@ -26,6 +27,7 @@ from corporate.models import (
     get_customer_by_realm,
 )
 from zerver.lib.logging_util import log_to_file
+from zerver.lib.send_email import FromAddress, send_email_to_billing_admins_and_realm_owners
 from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime
 from zerver.models import Realm, RealmAuditLog, UserProfile, get_system_bot
 from zproject.config import get_secret
@@ -988,6 +990,46 @@ def void_all_open_invoices(realm: Realm) -> int:
     return voided_invoices_count
 
 
+def downgrade_small_realms_behind_on_payments_as_needed() -> None:
+    customers = Customer.objects.all()
+    for customer in customers:
+        realm = customer.realm
+
+        # For larger realms, we generally want to talk to the customer
+        # before downgrading; so this logic only applies with 5.
+        if get_latest_seat_count(realm) >= 5:
+            continue
+
+        if get_current_plan_by_customer(customer) is None:
+            continue
+
+        due_invoice_count = 0
+        for invoice in stripe.Invoice.list(customer=customer.stripe_customer_id, limit=2):
+            if invoice.status == "open":
+                due_invoice_count += 1
+
+        # Customers with only 1 overdue invoice are ignored.
+        if due_invoice_count < 2:
+            continue
+
+        # We've now decided to downgrade this customer and void all invoices, and the below will execute this.
+
+        downgrade_now_without_creating_additional_invoices(realm)
+        void_all_open_invoices(realm)
+        context: Dict[str, str] = {
+            "upgrade_url": f"{realm.uri}{reverse('initial_upgrade')}",
+            "realm": realm,
+        }
+        send_email_to_billing_admins_and_realm_owners(
+            "zerver/emails/realm_auto_downgraded",
+            realm,
+            from_name=FromAddress.security_email_from_name(language=realm.default_language),
+            from_address=FromAddress.tokenized_no_reply_address(),
+            language=realm.default_language,
+            context=context,
+        )
+
+
 def update_billing_method_of_current_plan(
     realm: Realm, charge_automatically: bool, *, acting_user: Optional[UserProfile]
 ) -> None:

corporate/tests/test_stripe.py

@@ -1,12 +1,13 @@
 import json
 import operator
 import os
+import random
 import re
 import sys
 from datetime import datetime, timedelta, timezone
 from decimal import Decimal
 from functools import wraps
-from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, TypeVar, cast
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar, cast
 from unittest.mock import Mock, patch
 
 import orjson
@@ -19,6 +20,7 @@ from django.urls.resolvers import get_resolver
 from django.utils.timezone import now as timezone_now
 
 from corporate.lib.stripe import (
+    DEFAULT_INVOICE_DAYS_UNTIL_DUE,
     MAX_INVOICED_LICENSES,
     MIN_INVOICED_LICENSES,
     BillingError,
@@ -31,6 +33,7 @@ from corporate.lib.stripe import (
     compute_plan_parameters,
     customer_has_credit_card_as_default_source,
     do_create_stripe_customer,
+    downgrade_small_realms_behind_on_payments_as_needed,
     get_discount_for_realm,
     get_latest_seat_count,
     get_price_per_license,
@@ -273,6 +276,7 @@ MOCKED_STRIPE_FUNCTION_NAMES = [
         "Invoice.finalize_invoice",
         "Invoice.list",
         "Invoice.pay",
+        "Invoice.refresh",
         "Invoice.upcoming",
         "Invoice.void_invoice",
         "InvoiceItem.create",
@@ -2696,6 +2700,135 @@ class StripeTest(StripeTestCase):
         for invoice in invoices:
             self.assertEqual(invoice.status, "void")
 
+    @mock_stripe()
+    def test_downgrade_small_realms_behind_on_payments_as_needed(self, *mock: Mock) -> None:
+        def create_realm(
+            users_to_create: int,
+            create_stripe_customer: bool,
+            create_plan: bool,
+        ) -> Tuple[Realm, Optional[Customer], Optional[CustomerPlan]]:
+            realm_string_id = "realm_" + str(random.randrange(1, 1000000))
+            realm = Realm.objects.create(string_id=realm_string_id)
+            users = []
+            for i in range(users_to_create):
+                user = UserProfile.objects.create(
+                    delivery_email=f"user-{i}-{realm_string_id}@zulip.com",
+                    email=f"user-{i}-{realm_string_id}@zulip.com",
+                    realm=realm,
+                )
+                users.append(user)
+
+            customer = None
+            if create_stripe_customer:
+                customer = do_create_stripe_customer(users[0])
+            plan = None
+            if create_plan:
+                plan, _ = self.subscribe_realm_to_monthly_plan_on_manual_license_management(
+                    realm, users_to_create, users_to_create
+                )
+            return realm, customer, plan
+
+        def create_invoices(customer: Customer, num_invoices: int) -> List[stripe.Invoice]:
+            invoices = []
+            assert customer.stripe_customer_id is not None
+            for _ in range(num_invoices):
+                stripe.InvoiceItem.create(
+                    amount=10000,
+                    currency="usd",
+                    customer=customer.stripe_customer_id,
+                    description="Zulip standard",
+                    discountable=False,
+                )
+                invoice = stripe.Invoice.create(
+                    auto_advance=True,
+                    billing="send_invoice",
+                    customer=customer.stripe_customer_id,
+                    days_until_due=DEFAULT_INVOICE_DAYS_UNTIL_DUE,
+                    statement_descriptor="Zulip Standard",
+                )
+                stripe.Invoice.finalize_invoice(invoice)
+                invoices.append(invoice)
+            return invoices
+
+        realm_1, _, _ = create_realm(
+            users_to_create=1, create_stripe_customer=True, create_plan=False
+        )
+
+        realm_2, _, plan_2 = create_realm(
+            users_to_create=1, create_stripe_customer=True, create_plan=True
+        )
+        assert plan_2
+
+        realm_3, customer_3, plan_3 = create_realm(
+            users_to_create=1, create_stripe_customer=True, create_plan=True
+        )
+        assert customer_3 and plan_3
+        create_invoices(customer_3, num_invoices=1)
+
+        realm_4, customer_4, plan_4 = create_realm(
+            users_to_create=3, create_stripe_customer=True, create_plan=True
+        )
+        assert customer_4 and plan_4
+        create_invoices(customer_4, num_invoices=2)
+
+        realm_5, customer_5, plan_5 = create_realm(
+            users_to_create=1, create_stripe_customer=True, create_plan=True
+        )
+        assert customer_5 and plan_5
+        realm_5_invoices = create_invoices(customer_5, num_invoices=2)
+        for invoice in realm_5_invoices:
+            stripe.Invoice.pay(invoice, paid_out_of_band=True)
+
+        realm_6, customer_6, plan_6 = create_realm(
+            users_to_create=20, create_stripe_customer=True, create_plan=True
+        )
+        assert customer_6 and plan_6
+        create_invoices(customer_6, num_invoices=2)
+
+        with patch("corporate.lib.stripe.void_all_open_invoices") as void_all_open_invoices_mock:
+            downgrade_small_realms_behind_on_payments_as_needed()
+
+        realm_1.refresh_from_db()
+        self.assertEqual(realm_1.plan_type, Realm.SELF_HOSTED)
+
+        realm_2.refresh_from_db()
+        self.assertEqual(realm_2.plan_type, Realm.STANDARD)
+        plan_2.refresh_from_db()
+        self.assertEqual(plan_2.status, CustomerPlan.ACTIVE)
+
+        realm_3.refresh_from_db()
+        self.assertEqual(realm_3.plan_type, Realm.STANDARD)
+        plan_3.refresh_from_db()
+        self.assertEqual(plan_3.status, CustomerPlan.ACTIVE)
+
+        realm_4.refresh_from_db()
+        self.assertEqual(realm_4.plan_type, Realm.LIMITED)
+        plan_4.refresh_from_db()
+        self.assertEqual(plan_4.status, CustomerPlan.ENDED)
+        void_all_open_invoices_mock.assert_called_once_with(realm_4)
+
+        realm_5.refresh_from_db()
+        self.assertEqual(realm_5.plan_type, Realm.STANDARD)
+        plan_5.refresh_from_db()
+        self.assertEqual(plan_5.status, CustomerPlan.ACTIVE)
+
+        realm_6.refresh_from_db()
+        self.assertEqual(realm_6.plan_type, Realm.STANDARD)
+        plan_6.refresh_from_db()
+        self.assertEqual(plan_6.status, CustomerPlan.ACTIVE)
+
+        from django.core.mail import outbox
+
+        self.assert_length(outbox, 1)
+        self.assertIn(
+            f"Your organization, http://{realm_4.string_id}.testserver, has been downgraded",
+            outbox[0].body,
+        )
+        self.assert_length(outbox[0].to, 1)
+        recipient = UserProfile.objects.get(email=outbox[0].to[0])
+        self.assertEqual(recipient.realm, realm_4)
+        self.assertTrue(recipient.is_billing_admin)
+
     def test_update_billing_method_of_current_plan(self) -> None:
         realm = get_realm("zulip")
         customer = Customer.objects.create(realm=realm, stripe_customer_id="cus_12345")

puppet/zulip/files/cron.d/downgrade-small-realms-behind-on-payments

@@ -0,0 +1,5 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+USER=zulip
+
+0 17 * * * zulip /home/zulip/deployments/current/manage.py downgrade_small_realms_behind_on_payments

puppet/zulip_ops/manifests/prod_app_frontend_once.pp

@@ -17,6 +17,14 @@ class zulip_ops::prod_app_frontend_once {
     source => 'puppet:///modules/zulip/cron.d/invoice-plans',
   }
 
+  file { '/etc/cron.d/downgrade-small-realms-behind-on-payments':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/zulip/cron.d/downgrade-small-realms-behind-on-payments',
+  }
+
   file { '/etc/cron.d/check_send_receive_time':
     ensure => file,
     owner  => 'root',

stubs/stripe/__init__.pyi

@@ -81,7 +81,7 @@ class Invoice:
         ...
 
     @staticmethod
-    def pay(invoice: Invoice) -> Invoice:
+    def pay(invoice: Invoice, paid_out_of_band: bool=False) -> Invoice:
         ...
 
     @staticmethod
@@ -91,6 +91,10 @@ class Invoice:
     def get(self, key: str) -> Any:
         ...
 
+    @staticmethod
+    def refresh(invoice: Invoice) -> Invoice:
+        ...
+
 class Subscription:
     created: int
     status: str

templates/zerver/emails/realm_auto_downgraded.source.html

@@ -0,0 +1,25 @@
+{% extends "zerver/emails/compiled/email_base_default.html" %}
+
+{% block illustration %}
+<img src="{{ email_images_base_uri }}/email_logo.png" alt=""/>
+{% endblock %}
+
+{% block content %}
+    {% trans organization_name_with_link=macros.link_tag(realm.uri, realm.string_id) %}
+    Your organization, {{ organization_name_with_link }}, has been downgraded to the Zulip Cloud
+    Free plan because of unpaid invoices. The unpaid invoices have been voided.
+    {% endtrans %}
+    <br/>
+    <br/>
+
+    {% trans upgrade_url=macros.link_tag(upgrade_url) %}
+    To continue on the Zulip Cloud Standard plan, please upgrade again by going to {{ upgrade_url }}.
+    {% endtrans %}
+
+    <br/>
+    <br/>
+
+    {% trans support_email=macros.email_tag(support_email) %}
+    If you think this was a mistake or need more details, please reach out to us at {{ support_email }}.
+    {% endtrans %}
+{% endblock %}

templates/zerver/emails/realm_auto_downgraded.subject.txt

@@ -0,0 +1 @@
+{{ realm.string_id }}: Your organization has been downgraded to Zulip Cloud Free

templates/zerver/emails/realm_auto_downgraded.txt

@@ -0,0 +1,8 @@
+Your organization, {{ realm.uri }}, has been downgraded to the Zulip Cloud
+Free plan because of unpaid invoices. The unpaid invoices have been voided.
+
+To continue on the Zulip Cloud Standard plan, please upgrade again by going
+to {{ upgrade_url }}.
+
+If you think this was a mistake or need more details, please reach out
+to us at support@zulip.com.

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 75
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = "150.5"
+PROVISION_VERSION = "150.6"

zerver/lib/send_email.py

@@ -380,6 +380,24 @@ def send_email_to_admins(
     )
 
 
+def send_email_to_billing_admins_and_realm_owners(
+    template_prefix: str,
+    realm: Realm,
+    from_name: Optional[str] = None,
+    from_address: Optional[str] = None,
+    language: Optional[str] = None,
+    context: Dict[str, Any] = {},
+) -> None:
+    send_email(
+        template_prefix,
+        to_user_ids=[user.id for user in realm.get_human_billing_admin_and_realm_owner_users()],
+        from_name=from_name,
+        from_address=from_address,
+        language=language,
+        context=context,
+    )
+
+
 def clear_scheduled_invitation_emails(email: str) -> None:
     """Unlike most scheduled emails, invitation emails don't have an
     existing user object to key off of, so we filter by address here."""

zerver/tests/test_management_commands.py

@@ -483,6 +483,19 @@ class TestInvoicePlans(ZulipTestCase):
         m.assert_called_once()
 
 
+@skipUnless(settings.ZILENCER_ENABLED, "requires zilencer")
+class TestDowngradeSmallRealmsBehindOnPayments(ZulipTestCase):
+    COMMAND_NAME = "downgrade_small_realms_behind_on_payments"
+
+    def test_if_command_calls_downgrade_small_realms_behind_on_payments_as_needed(self) -> None:
+        with patch(
+            "zilencer.management.commands.downgrade_small_realms_behind_on_payments.downgrade_small_realms_behind_on_payments_as_needed"
+        ) as m:
+            call_command(self.COMMAND_NAME)
+
+        m.assert_called_once()
+
+
 class TestExport(ZulipTestCase):
     COMMAND_NAME = "export"
 

zilencer/management/commands/downgrade_small_realms_behind_on_payments.py

@@ -0,0 +1,11 @@
+from typing import Any
+
+from corporate.lib.stripe import downgrade_small_realms_behind_on_payments_as_needed
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = "Downgrade small realms that are running behind on payments"
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        downgrade_small_realms_behind_on_payments_as_needed()