zerver: Implement ldap group synchronization.

Fixes #9957.

Co-authored-by: Mateusz Mandera <mateusz.mandera@zulip.com>

docs/production/authentication-methods.md

@@ -184,6 +184,7 @@ All of these data synchronization options have the same model:
   Zulip server with
   `/home/zulip/deployments/current/scripts/restart-server` so that
   your configuration changes take effect.
+- Logs are available in `/var/log/zulip/ldap.log`.
 
 When using this feature, you may also want to
 [prevent users from changing their display name in the Zulip UI][restrict-name-changes],
@@ -212,6 +213,61 @@ corresponding LDAP attribute is `linkedinProfile` then you just need
 to add `'custom_profile_field__linkedin_profile': 'linkedinProfile'`
 to the `AUTH_LDAP_USER_ATTR_MAP`.
 
+#### Synchronizing groups
+
+Zulip supports syncing [Zulip groups][zulip-groups] with LDAP
+groups. To configure this feature:
+
+1. Review the [django-auth-ldap
+   documentation](https://django-auth-ldap.readthedocs.io/en/latest/groups.html)
+   to determine which of its supported group type configurations
+   matches how your LDAP directory stores groups.
+
+1. Set `AUTH_LDAP_GROUP_TYPE` to the appropriate class instance for
+   that LDAP group type:
+
+   ```python
+   from django_auth_ldap.config import ActiveDirectoryGroupType
+   AUTH_LDAP_GROUP_TYPE = ActiveDirectoryGroupType()
+   ```
+
+   The default is `GroupOfUniqueNamesType`.
+
+1. Configure `AUTH_LDAP_GROUP_SEARCH` to specify how to find groups in
+   your LDAP directory:
+
+   ```python
+   AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+       "ou=groups,dc=www,dc=example,dc=com", ldap.SCOPE_SUBTREE,
+       "(objectClass=groupOfUniqueNames)"
+   )
+   ```
+
+1. Configure which LDAP groups you want to sync into
+   Zulip. `LDAP_SYNCHRONIZED_GROUPS_BY_REALM` is a map where the keys
+   are subdomains of the realms being configured (use `""` for the
+   root domain), and the value corresponding to the key being a list
+   the names of groups to sync:
+
+   ```python
+   LDAP_SYNCHRONIZED_GROUPS_BY_REALM = {
+     "subdomain1" : [
+         "group1",
+         "group2",
+     ]
+   }
+   ```
+
+   In this example configuration, for the Zulip realm with subdomain
+   `subdomain1`, user membership in the Zulip groups named `group1`
+   and `group2` will match their membership in LDAP groups with those
+   names.
+
+1. Test your configuration and restart the server into the new
+   configuration as [documented above](#synchronizing-data).
+
+[zulip-groups]: https://zulip.com/help/user-groups
+
 #### Synchronizing email addresses
 
 User accounts in Zulip are uniquely identified by their email address,

zerver/tests/fixtures/ldap/directory.json

@@ -77,5 +77,24 @@
         "uid": ["user2_with_shared_email"],
         "sn": ["shortname"],
         "mail": ["shared_email@zulip.com"]
+    },
+
+    "ou=groups,dc=zulip,dc=com": {
+        "ou": "groups"
+    },
+    "cn=cool_test_group,ou=groups,dc=zulip,dc=com": {
+        "objectClass": ["groupOfUniqueNames"],
+        "cn": ["cool_test_group"],
+        "uniqueMember": [
+            "uid=hamlet,ou=users,dc=zulip,dc=com"
+        ]
+    },
+    "cn=another_test_group,ou=groups,dc=zulip,dc=com": {
+        "objectClass": ["groupOfUniqueNames"],
+        "cn": ["another_test_group"],
+        "uniqueMember": [
+            "uid=hamlet,ou=users,dc=zulip,dc=com",
+            "uid=cordelia,ou=users,dc=zulip,dc=com"
+        ]
     }
 }

zerver/tests/test_auth_backends.py

@@ -63,6 +63,10 @@ from zerver.actions.realm_settings import (
     do_set_realm_authentication_methods,
     do_set_realm_property,
 )
+from zerver.actions.user_groups import (
+    bulk_add_members_to_user_groups,
+    create_user_group_in_database,
+)
 from zerver.actions.user_settings import do_change_password, do_change_user_setting
 from zerver.actions.users import change_user_is_active, do_deactivate_user
 from zerver.lib.avatar import avatar_url
@@ -89,6 +93,7 @@ from zerver.lib.test_helpers import (
 )
 from zerver.lib.types import Validator
 from zerver.lib.upload.base import DEFAULT_AVATAR_SIZE, MEDIUM_AVATAR_SIZE, resize_avatar
+from zerver.lib.user_groups import is_user_in_group
 from zerver.lib.users import get_all_api_keys, get_api_key, get_raw_user_data
 from zerver.lib.utils import assert_is_not_none
 from zerver.lib.validator import (
@@ -109,6 +114,7 @@ from zerver.models import (
     Realm,
     RealmDomain,
     Stream,
+    UserGroup,
     UserProfile,
     clear_supported_auth_backends_cache,
     get_realm,
@@ -7248,5 +7254,136 @@ class JWTFetchAPIKeyTest(ZulipTestCase):
             self.assert_json_error_contains(result, "Invalid subdomain", 404)
 
 
+class LDAPGroupSyncTest(ZulipTestCase):
+    @override_settings(AUTHENTICATION_BACKENDS=("zproject.backends.ZulipLDAPAuthBackend",))
+    def test_ldap_group_sync(self) -> None:
+        self.init_default_ldap_database()
+
+        hamlet = self.example_user("hamlet")
+        with self.settings(LDAP_APPEND_DOMAIN="zulip.com"):
+            result = sync_user_from_ldap(hamlet, mock.Mock())
+            self.assertTrue(result)
+        self.assertTrue(hamlet.is_active)
+
+        realm = get_realm("zulip")
+
+        with self.settings(
+            AUTH_LDAP_GROUP_SEARCH=LDAPSearch(
+                "ou=groups,dc=zulip,dc=com",
+                ldap.SCOPE_ONELEVEL,
+                "(objectClass=groupOfUniqueNames)",
+            ),
+            LDAP_SYNCHRONIZED_GROUPS_BY_REALM={
+                "zulip": [
+                    "cool_test_group",
+                ]
+            },
+            LDAP_APPEND_DOMAIN="zulip.com",
+        ), self.assertLogs("zulip.ldap", "DEBUG") as zulip_ldap_log:
+            self.assertFalse(UserGroup.objects.filter(realm=realm, name="cool_test_group").exists())
+
+            create_user_group_in_database(
+                "cool_test_group", [], realm, acting_user=None, description="Created by LDAP sync"
+            )
+
+            self.assertTrue(UserGroup.objects.filter(realm=realm, name="cool_test_group").exists())
+
+            user_group = UserGroup.objects.get(realm=realm, name="cool_test_group")
+
+            self.assertFalse(
+                is_user_in_group(
+                    user_group,
+                    hamlet,
+                    direct_member_only=True,
+                )
+            )
+
+            sync_user_from_ldap(hamlet, mock.Mock())
+            self.assertTrue(
+                is_user_in_group(
+                    user_group,
+                    hamlet,
+                    direct_member_only=True,
+                )
+            )
+
+            # Add a user to a Zulip group that they are not member of in ldap.
+            # This implies that they should be deleted from the Zulip group
+            # upon the next sync.
+            cordelia = self.example_user("cordelia")
+            bulk_add_members_to_user_groups(
+                [user_group],
+                [cordelia.id],
+                acting_user=None,
+            )
+
+            self.assertTrue(
+                is_user_in_group(
+                    UserGroup.objects.get(realm=realm, name="cool_test_group"),
+                    cordelia,
+                    direct_member_only=True,
+                )
+            )
+
+            # This should remove cordelia from cool_test_group
+            sync_user_from_ldap(cordelia, mock.Mock())
+
+            self.assertFalse(
+                is_user_in_group(
+                    UserGroup.objects.get(realm=realm, name="cool_test_group"),
+                    cordelia,
+                    direct_member_only=True,
+                )
+            )
+
+        hamlet = self.example_user("hamlet")
+        cordelia = self.example_user("cordelia")
+
+        self.assertEqual(
+            zulip_ldap_log.output,
+            [
+                f"DEBUG:zulip.ldap:Syncing groups for user: {hamlet.id}",
+                "DEBUG:zulip.ldap:intended groups: {'cool_test_group'}; zulip groups: set()",
+                f"DEBUG:zulip.ldap:add {hamlet.id} to ['cool_test_group']",
+                f"DEBUG:zulip.ldap:Syncing groups for user: {cordelia.id}",
+                "DEBUG:zulip.ldap:intended groups: set(); zulip groups: {'cool_test_group'}",
+                f"DEBUG:zulip.ldap:removing groups {{'cool_test_group'}} from {cordelia.id}",
+            ],
+        )
+
+        # Test an exception using a malformed ldap group search setting.
+        with self.settings(
+            AUTH_LDAP_GROUP_SEARCH=LDAPSearch(
+                "ou=groups,dc=zulip,dc=com",
+                ldap.SCOPE_ONELEVEL,
+                "(objectClass=groupOfUniqueNames",  # this is malformed, missing ")"
+            ),
+            LDAP_SYNCHRONIZED_GROUPS_BY_REALM={
+                "zulip": [
+                    "cool_test_group",
+                ]
+            },
+            LDAP_APPEND_DOMAIN="zulip.com",
+        ), self.assertLogs("django_auth_ldap", "WARN") as django_ldap_log, self.assertLogs(
+            "zulip.ldap", "DEBUG"
+        ) as zulip_ldap_log:
+            with self.assertRaisesRegex(
+                ZulipLDAPError,
+                "search_s.*",
+            ):
+                sync_user_from_ldap(cordelia, mock.Mock())
+
+        self.assertEqual(
+            zulip_ldap_log.output,
+            [f"DEBUG:zulip.ldap:Syncing groups for user: {cordelia.id}"],
+        )
+        self.assertEqual(
+            django_ldap_log.output,
+            [
+                'WARNING:django_auth_ldap:search_s("ou=groups,dc=zulip,dc=com", 1, "(&(objectClass=groupOfUniqueNames(uniqueMember=uid=cordelia,ou=users,dc=zulip,dc=com))", "None", 0) while authenticating cordelia',
+            ],
+        )
+
+
 # Don't load the base class as a test: https://bugs.python.org/issue17519.
 del SocialAuthBase

zproject/backends.py

@@ -80,6 +80,10 @@ from zxcvbn import zxcvbn
 
 from zerver.actions.create_user import do_create_user, do_reactivate_user
 from zerver.actions.custom_profile_fields import do_update_user_custom_profile_data_if_changed
+from zerver.actions.user_groups import (
+    bulk_add_members_to_user_groups,
+    bulk_remove_members_from_user_groups,
+)
 from zerver.actions.user_settings import do_regenerate_api_key
 from zerver.actions.users import do_deactivate_user
 from zerver.lib.avatar import avatar_url, is_avatar_new
@@ -105,6 +109,8 @@ from zerver.models import (
     PreregistrationRealm,
     PreregistrationUser,
     Realm,
+    UserGroup,
+    UserGroupMembership,
     UserProfile,
     custom_profile_fields_for_realm,
     get_realm,
@@ -910,6 +916,78 @@ class ZulipLDAPAuthBackendBase(ZulipAuthMixin, LDAPBackend):
         except SyncUserError as e:
             raise ZulipLDAPError(str(e)) from e
 
+    def sync_groups_from_ldap(self, user_profile: UserProfile, ldap_user: _LDAPUser) -> None:
+        """
+        For the groups set up for syncing for the realm in LDAP_SYNCHRONIZED_GROUPS_BY_REALM:
+
+        (1) Makes sure the user has membership in the Zulip UserGroups corresponding
+            to the LDAP groups ldap_user belongs to.
+        (2) Makes sure the user doesn't have membership in the Zulip UserGroups corresponding
+            to the LDAP groups ldap_user doesn't belong to.
+        """
+
+        if user_profile.realm.string_id not in settings.LDAP_SYNCHRONIZED_GROUPS_BY_REALM:
+            # no groups to sync for this realm
+            return
+
+        configured_ldap_group_names_for_sync = set(
+            settings.LDAP_SYNCHRONIZED_GROUPS_BY_REALM[user_profile.realm.string_id]
+        )
+
+        try:
+            ldap_logger.debug("Syncing groups for user: %s", user_profile.id)
+            intended_group_name_set_for_user = set(ldap_user.group_names).intersection(
+                configured_ldap_group_names_for_sync
+            )
+
+            existing_group_name_set_for_user = set(
+                UserGroupMembership.objects.filter(
+                    user_group__realm=user_profile.realm,
+                    user_group__name__in=set(
+                        settings.LDAP_SYNCHRONIZED_GROUPS_BY_REALM[user_profile.realm.string_id]
+                    ),
+                    user_profile=user_profile,
+                ).values_list("user_group__name", flat=True)
+            )
+
+            ldap_logger.debug(
+                "intended groups: %s; zulip groups: %s",
+                repr(intended_group_name_set_for_user),
+                repr(existing_group_name_set_for_user),
+            )
+
+            new_groups = UserGroup.objects.filter(
+                name__in=intended_group_name_set_for_user.difference(
+                    existing_group_name_set_for_user
+                ),
+                realm=user_profile.realm,
+            )
+            if new_groups:
+                ldap_logger.debug(
+                    "add %s to %s", user_profile.id, [group.name for group in new_groups]
+                )
+                bulk_add_members_to_user_groups(new_groups, [user_profile.id], acting_user=None)
+
+            group_names_for_membership_deletion = existing_group_name_set_for_user.difference(
+                intended_group_name_set_for_user
+            )
+            groups_for_membership_deletion = UserGroup.objects.filter(
+                name__in=group_names_for_membership_deletion, realm=user_profile.realm
+            )
+
+            if group_names_for_membership_deletion:
+                ldap_logger.debug(
+                    "removing groups %s from %s",
+                    group_names_for_membership_deletion,
+                    user_profile.id,
+                )
+                bulk_remove_members_from_user_groups(
+                    groups_for_membership_deletion, [user_profile.id], acting_user=None
+                )
+
+        except Exception as e:
+            raise ZulipLDAPError(str(e)) from e
+
 
 class ZulipLDAPAuthBackend(ZulipLDAPAuthBackendBase):
     REALM_IS_NONE_ERROR = 1
@@ -1136,6 +1214,7 @@ class ZulipLDAPUserPopulator(ZulipLDAPAuthBackendBase):
         self.sync_avatar_from_ldap(user, ldap_user)
         self.sync_full_name_from_ldap(user, ldap_user)
         self.sync_custom_profile_fields_from_ldap(user, ldap_user)
+        self.sync_groups_from_ldap(user, ldap_user)
         return (user, built)
 
 

zproject/default_settings.py

@@ -2,6 +2,8 @@ import os
 from email.headerregistry import Address
 from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple
 
+from django_auth_ldap.config import GroupOfUniqueNamesType, LDAPGroupType
+
 from scripts.lib.zulip_tools import deport
 from zproject.settings_types import JwtAuthKey, OIDCIdPConfigDict, SAMLIdPConfigDict
 
@@ -69,6 +71,8 @@ AUTH_LDAP_ALWAYS_UPDATE_USER = False
 FAKE_LDAP_MODE: Optional[str] = None
 FAKE_LDAP_NUM_USERS = 8
 AUTH_LDAP_ADVANCED_REALM_ACCESS_CONTROL: Optional[Dict[str, Any]] = None
+LDAP_SYNCHRONIZED_GROUPS_BY_REALM: Dict[str, List[str]] = {}
+AUTH_LDAP_GROUP_TYPE: LDAPGroupType = GroupOfUniqueNamesType()
 
 # Social auth; we support providing values for some of these
 # settings in zulip-secrets.conf instead of settings.py in development.

zproject/prod_settings_template.py

@@ -266,6 +266,20 @@ AUTH_LDAP_USER_ATTR_MAP = {
 #    ]
 # }
 
+
+## LDAP group sync configuration.
+## See: https://zulip.readthedocs.io/en/latest/production/authentication-methods.html#synchronizing-groups
+# AUTH_LDAP_GROUP_TYPE = GroupOfUniqueNamesType()
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+#     "ou=groups,dc=www,dc=example,dc=com", ldap.SCOPE_SUBTREE, "(objectClass=groupOfUniqueNames)"
+# )
+# LDAP_SYNCHRONIZED_GROUPS_BY_REALM = {
+#   "subdomain1" : [
+#       "group1",
+#       "group2",
+#   ]
+# }
+
 ########
 ## Google OAuth.
 ##