settings: Add FAKE_EMAIL_DOMAIN setting.

Fixes #9401.

This adds a FAKE_EMAIL_DOMAIN setting, which should be used if
EXTERNAL_HOST is not a valid domain, and something else is needed to
form bot and dummy user emails (if email visibility is turned off).
It defaults to EXTERNAL_HOST.

get_fake_email_domain() should be used to get this value. It validates
that it's correctly set - that it can be used to form valid emails.

If it's not set correctly, an exception is raised. This is the right
approach, because it's undesirable to have the server seemingly
peacefully operating with that setting misconfigured, as that could
mask some hidden sneaky bugs due to UserProfiles with invalid emails,
which would blow up the moment some code that does validate the emails
is called.

frontend_tests/casper_tests/06-settings.js

@@ -127,7 +127,7 @@ casper.then(function create_bot() {
     casper.click('#create_bot_button');
 });
 
-var bot_email = '1-bot@zulip.zulipdev.com';
+var bot_email = '1-bot@zulip.testserver';
 var button_sel = '.download_bot_zuliprc[data-email="' + bot_email + '"]';
 
 casper.then(function () {
@@ -158,7 +158,7 @@ casper.then(function create_bot() {
     casper.click('#create_bot_button');
 });
 
-var second_bot_email = '2-bot@zulip.zulipdev.com';
+var second_bot_email = '2-bot@zulip.testserver';
 var second_button_sel = '.download_bot_zuliprc[data-email="' + second_bot_email + '"]';
 
 casper.then(function () {

zerver/lib/create_user.py

@@ -1,6 +1,7 @@
 from django.contrib.auth.models import UserManager
 from django.utils.timezone import now as timezone_now
-from zerver.models import UserProfile, Recipient, Subscription, Realm, Stream
+from zerver.models import UserProfile, Recipient, Subscription, Realm, Stream, \
+    get_fake_email_domain
 from zerver.lib.upload import copy_avatar
 from zerver.lib.hotspots import copy_hotpots
 from zerver.lib.utils import generate_api_key
@@ -32,8 +33,7 @@ def copy_user_settings(source_profile: UserProfile, target_profile: UserProfile)
 
 def get_display_email_address(user_profile: UserProfile, realm: Realm) -> str:
     if realm.email_address_visibility != Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
-        # TODO: realm.host isn't always a valid option here.
-        return "user%s@%s" % (user_profile.id, realm.host.split(':')[0])
+        return "user%s@%s" % (user_profile.id, get_fake_email_domain())
     return user_profile.delivery_email
 
 # create_user_profile is based on Django's User.objects.create_user,

zerver/models.py

@@ -11,7 +11,7 @@ from django.contrib.auth.models import AbstractBaseUser, UserManager, \
 import django.contrib.auth
 from django.core.exceptions import ValidationError
 from django.core.validators import URLValidator, MinLengthValidator, \
-    RegexValidator
+    RegexValidator, validate_email
 from django.dispatch import receiver
 from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \
     user_profile_by_api_key_cache_key, active_non_guest_user_ids_cache_key, \
@@ -414,8 +414,7 @@ class Realm(models.Model):
         return UserProfile.objects.filter(realm=self, is_active=True).select_related()
 
     def get_bot_domain(self) -> str:
-        # Remove the port. Mainly needed for development environment.
-        return self.host.split(':')[0]
+        return get_fake_email_domain()
 
     def get_notifications_stream(self) -> Optional['Stream']:
         if self.notifications_stream is not None and not self.notifications_stream.deactivated:
@@ -2800,3 +2799,15 @@ class BotConfigData(models.Model):
 
     class Meta(object):
         unique_together = ("bot_profile", "key")
+
+class InvalidFakeEmailDomain(Exception):
+    pass
+
+def get_fake_email_domain() -> str:
+    try:
+        # Check that the fake email domain can be used to form valid email addresses.
+        validate_email("bot@" + settings.FAKE_EMAIL_DOMAIN)
+    except ValidationError:
+        raise InvalidFakeEmailDomain(settings.FAKE_EMAIL_DOMAIN + ' is not a valid domain.')
+
+    return settings.FAKE_EMAIL_DOMAIN

zerver/tests/test_bots.py

@@ -3,6 +3,7 @@ import os
 import ujson
 
 from django.core import mail
+from django.test import override_settings
 from mock import patch, MagicMock
 from typing import Any, Dict, List, Mapping, Optional
 
@@ -93,6 +94,22 @@ class BotTest(ZulipTestCase, UploadSerializeMixin):
         self.assert_json_error(result, 'Bad name or username')
         self.assert_num_bots_equal(0)
 
+    @override_settings(FAKE_EMAIL_DOMAIN="invaliddomain")
+    def test_add_bot_with_invalid_fake_email_domain(self) -> None:
+        self.login(self.example_email('hamlet'))
+        self.assert_num_bots_equal(0)
+        bot_info = {
+            'full_name': 'The Bot of Hamlet',
+            'short_name': 'hambot',
+            'bot_type': '1',
+        }
+        result = self.client_post("/json/bots", bot_info)
+
+        error_message = ("Can't create bots until FAKE_EMAIL_DOMAIN is correctly configured.\n" +
+                         "Please contact your server administrator.")
+        self.assert_json_error(result, error_message)
+        self.assert_num_bots_equal(0)
+
     def test_add_bot_with_no_name(self) -> None:
         self.login(self.example_email('hamlet'))
         self.assert_num_bots_equal(0)

zerver/tests/test_users.py

@@ -19,7 +19,8 @@ from zerver.models import UserProfile, Recipient, Realm, \
     get_user, get_realm, get_stream, get_stream_recipient, \
     get_source_profile, get_system_bot, \
     ScheduledEmail, check_valid_user_ids, \
-    get_user_by_id_in_realm_including_cross_realm, CustomProfileField
+    get_user_by_id_in_realm_including_cross_realm, CustomProfileField, \
+    InvalidFakeEmailDomain, get_fake_email_domain
 
 from zerver.lib.avatar import avatar_url, get_gravatar_url
 from zerver.lib.exceptions import JsonableError
@@ -42,6 +43,7 @@ from zerver.lib.users import user_ids_to_users, access_user_by_id, \
     get_accounts_for_email
 
 from django.conf import settings
+from django.test import override_settings
 
 import datetime
 import mock
@@ -1272,3 +1274,14 @@ class GetProfileTest(ZulipTestCase):
                     user['avatar_url'],
                     avatar_url(user_profile),
                 )
+
+class FakeEmailDomainTest(ZulipTestCase):
+    @override_settings(FAKE_EMAIL_DOMAIN="invaliddomain")
+    def test_invalid_fake_email_domain(self) -> None:
+        with self.assertRaises(InvalidFakeEmailDomain):
+            get_fake_email_domain()
+
+    @override_settings(FAKE_EMAIL_DOMAIN="127.0.0.1")
+    def test_invalid_fake_email_domain_ip(self) -> None:
+        with self.assertRaises(InvalidFakeEmailDomain):
+            get_fake_email_domain()

zerver/views/users.py

@@ -35,7 +35,8 @@ from zerver.lib.utils import generate_api_key, generate_random_token
 from zerver.models import UserProfile, Stream, Message, email_allowed_for_realm, \
     get_user_by_delivery_email, Service, get_user_including_cross_realm, \
     DomainNotAllowedForRealmError, DisposableEmailError, get_user_profile_by_id_in_realm, \
-    EmailContainsPlusError, get_user_by_id_in_realm_including_cross_realm, Realm
+    EmailContainsPlusError, get_user_by_id_in_realm_including_cross_realm, Realm, \
+    InvalidFakeEmailDomain
 
 def deactivate_user_backend(request: HttpRequest, user_profile: UserProfile,
                             user_id: int) -> HttpResponse:
@@ -276,7 +277,11 @@ def add_bot_backend(
         service_name = service_name or short_name
     short_name += "-bot"
     full_name = check_full_name(full_name_raw)
+    try:
         email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain())
+    except InvalidFakeEmailDomain:
+        return json_error(_("Can't create bots until FAKE_EMAIL_DOMAIN is correctly configured.\n"
+                            "Please contact your server administrator."))
     form = CreateUserForm({'full_name': full_name, 'email': email})
 
     if bot_type == UserProfile.EMBEDDED_BOT:

zproject/prod_settings_template.py

@@ -51,6 +51,11 @@ EXTERNAL_HOST = 'zulip.example.com'
 # Note that these should just be hostnames, without port numbers.
 #ALLOWED_HOSTS = ['zulip-alias.example.com', '192.0.2.1']
 
+# If EXTERNAL_HOST is not a valid domain name (e.g. an IP address),
+# set FAKE_EMAIL_DOMAIN below to a domain that Zulip can use when
+# generating (fake) email addresses for bots, dummy users, etc.
+#FAKE_EMAIL_DOMAIN = 'fake-domain.example.com'
+
 
 ################
 # Outgoing email (SMTP) settings.

zproject/settings.py

@@ -130,6 +130,7 @@ DEFAULT_SETTINGS = {
     'ADD_TOKENS_TO_NOREPLY_ADDRESS': True,
     'TOKENIZED_NOREPLY_EMAIL_ADDRESS': "noreply-{token}@" + EXTERNAL_HOST.split(":")[0],
     'PHYSICAL_ADDRESS': '',
+    'FAKE_EMAIL_DOMAIN': EXTERNAL_HOST.split(":")[0],
 
     # SMTP settings
     'EMAIL_HOST': None,

zproject/test_settings.py

@@ -18,6 +18,8 @@ if os.getenv("EXTERNAL_HOST") is None:
     os.environ["EXTERNAL_HOST"] = "testserver"
 from .settings import *
 
+FAKE_EMAIL_DOMAIN = "zulip.testserver"
+
 # Clear out the REALM_HOSTS set in dev_settings.py
 REALM_HOSTS = {}