populate_db, tests: Restrict emails in zulip realm.

We now restrict emails on the zulip realm, and now
`email` and `delivery_email` will be different for
users.

This change should make it more likely to catch
errors where we leak delivery emails or use the
wrong field for lookups.

frontend_tests/run-casper

@@ -75,6 +75,10 @@ os.makedirs('var/casper', exist_ok=True)
 for f in glob.glob('var/casper/casper-failure*.png'):
     os.remove(f)
 
+def reset_database() -> None:
+    from zerver.lib.test_helpers import reset_emails_in_zulip_realm
+    reset_emails_in_zulip_realm()
+
 def run_tests(files: Iterable[str], external_host: str) -> None:
     test_dir = os.path.join(ZULIP_PATH, 'frontend_tests/casper_tests')
     test_files = []
@@ -118,9 +122,12 @@ def run_tests(files: Iterable[str], external_host: str) -> None:
         return 0
 
     with test_server_running(options.force, external_host):
-        # Important: do this next call inside the `with` block, when Django
+        # Important: do next things inside the `with` block, when Django
         #            will be pointing at the test database.
+        reset_database()
         subprocess.check_call('tools/setup/generate-test-credentials')
+
+        # RUN THE TESTS!!!
         if options.interactive:
             response = input('Press Enter to run tests, "q" to quit: ')
             ret = 1
@@ -131,6 +138,7 @@ def run_tests(files: Iterable[str], external_host: str) -> None:
         else:
             ret = 1
             ret = run_tests()
+
     if ret != 0:
         print("""
 The Casper frontend tests failed!  For help debugging, read:

tools/test-api

@@ -33,14 +33,17 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981')
     from zerver.openapi.python_examples import test_the_api, test_invalid_api_key
     from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success
     from zerver.lib.actions import do_create_user
+    from zerver.lib.test_helpers import reset_emails_in_zulip_realm
     from zerver.lib.users import get_api_key
     from zerver.models import get_user, get_realm
 
     print("Running API tests...")
 
+    reset_emails_in_zulip_realm()
+
     # Prepare the admin client
     email = 'iago@zulip.com'  # Iago is an admin
-    realm = get_realm("zulip")
+    realm = get_realm('zulip')
     user = get_user(email, realm)
     api_key = get_api_key(user)
     site = 'http://zulip.zulipdev.com:9981'

version.py

@@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/12/13/zulip-2-1-relea
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '74.1'
+PROVISION_VERSION = '75.0'

zerver/tests/test_custom_profile_data.py

@@ -642,6 +642,7 @@ class ListCustomProfileFieldTest(CustomProfileFieldTestCase):
     def test_get_custom_profile_fields_from_api(self) -> None:
         iago = self.example_user("iago")
         test_bot = self.create_test_bot("foo-bot", iago)
+        self.login_user(iago)
         assert(test_bot)
 
         url = "/json/users?client_gravatar=false&include_custom_profile_fields=true"
@@ -668,12 +669,14 @@ class ListCustomProfileFieldTest(CustomProfileFieldTestCase):
             raise AssertionError("Could not find required data from the response.")
 
         expected_keys_for_iago = {
+            "delivery_email",
             "email", "user_id", "avatar_url", "is_admin", "is_guest", "is_bot",
             "full_name", "timezone", "is_active", "date_joined", "profile_data"}
         self.assertEqual(set(iago_raw_data.keys()), expected_keys_for_iago)
         self.assertNotEqual(iago_raw_data["profile_data"], {})
 
         expected_keys_for_test_bot = {
+            "delivery_email",
             "email", "user_id", "avatar_url", "is_admin", "is_guest", "is_bot", "full_name",
             "timezone", "is_active", "date_joined", "bot_type", "bot_owner_id"}
         self.assertEqual(set(test_bot_raw_data.keys()), expected_keys_for_test_bot)

zerver/tests/test_email_change.py

@@ -52,7 +52,13 @@ class EmailChangeTestCase(ZulipTestCase):
 
     def test_confirm_email_change(self) -> None:
         user_profile = self.example_user('hamlet')
-        old_email = user_profile.email
+        do_set_realm_property(
+            user_profile.realm,
+            'email_address_visibility',
+            Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE,
+        )
+
+        old_email = user_profile.delivery_email
         new_email = 'hamlet-new@zulip.com'
         new_realm = get_realm('zulip')
         self.login('hamlet')

zilencer/management/commands/populate_db.py

@@ -33,7 +33,7 @@ from zerver.models import CustomProfileField, DefaultStream, Message, Realm, Rea
     RealmDomain, Recipient, Service, Stream, Subscription, \
     UserMessage, UserPresence, UserProfile, Huddle, Client, \
     get_client, get_huddle, get_realm, get_stream, \
-    get_user, get_user_profile_by_id
+    get_user, get_user_by_delivery_email, get_user_profile_by_id
 from zerver.lib.types import ProfileFieldData
 
 from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path
@@ -201,6 +201,7 @@ class Command(BaseCommand):
             create_internal_realm()
             zulip_realm = Realm.objects.create(
                 string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=False,
+                email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS,
                 description="The Zulip development environment default organization."
                             "  It's great for testing!",
                 invite_required=False, org_type=Realm.CORPORATE)
@@ -264,12 +265,12 @@ class Command(BaseCommand):
 
             create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)
 
-            iago = get_user("iago@zulip.com", zulip_realm)
+            iago = get_user_by_delivery_email("iago@zulip.com", zulip_realm)
             do_change_is_admin(iago, True)
             iago.is_staff = True
             iago.save(update_fields=['is_staff'])
 
-            guest_user = get_user("polonius@zulip.com", zulip_realm)
+            guest_user = get_user_by_delivery_email("polonius@zulip.com", zulip_realm)
             guest_user.role = UserProfile.ROLE_GUEST
             guest_user.save(update_fields=['role'])
 
@@ -284,7 +285,7 @@ class Command(BaseCommand):
 
             create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
 
-            zoe = get_user("zoe@zulip.com", zulip_realm)
+            zoe = get_user_by_delivery_email("zoe@zulip.com", zulip_realm)
             zulip_webhook_bots = [
                 ("Zulip Webhook Bot", "webhook-bot@zulip.com"),
             ]
@@ -293,7 +294,7 @@ class Command(BaseCommand):
             # owner of the webhook bot, so bot_owner can't be None
             create_users(zulip_realm, zulip_webhook_bots,
                          bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe)
-            aaron = get_user("AARON@zulip.com", zulip_realm)
+            aaron = get_user_by_delivery_email("AARON@zulip.com", zulip_realm)
 
             zulip_outgoing_bots = [
                 ("Outgoing Webhook", "outgoing-webhook@zulip.com")
@@ -345,10 +346,11 @@ class Command(BaseCommand):
                 }
 
                 for profile in profiles:
-                    if profile.email not in subscriptions_map:
+                    email = profile.delivery_email
-                        raise Exception('Subscriptions not listed for user %s' % (profile.email,))
+                    if email not in subscriptions_map:
+                        raise Exception('Subscriptions not listed for user %s' % (email,))
 
-                    for stream_name in subscriptions_map[profile.email]:
+                    for stream_name in subscriptions_map[email]:
                         stream = Stream.objects.get(name=stream_name)
                         r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
                         subscriptions_list.append((profile, r))
@@ -418,7 +420,7 @@ class Command(BaseCommand):
             github_profile = try_add_realm_default_custom_profile_field(zulip_realm, "github")
 
             # Fill in values for Iago and Hamlet
-            hamlet = get_user("hamlet@zulip.com", zulip_realm)
+            hamlet = get_user_by_delivery_email("hamlet@zulip.com", zulip_realm)
             do_update_user_custom_profile_data_if_changed(iago, [
                 {"id": phone_number.id, "value": "+1-234-567-8901"},
                 {"id": biography.id, "value": "Betrayer of Othello."},
@@ -739,7 +741,7 @@ def choose_date_sent(num_messages: int, tot_messages: int, threads: int) -> date
 
 def create_user_groups() -> None:
     zulip = get_realm('zulip')
-    members = [get_user('cordelia@zulip.com', zulip),
+    members = [get_user_by_delivery_email('cordelia@zulip.com', zulip),
-               get_user('hamlet@zulip.com', zulip)]
+               get_user_by_delivery_email('hamlet@zulip.com', zulip)]
     create_user_group("hamletcharacters", members, zulip,
                       description="Characters of Hamlet")