parse_os_release: Use /etc/os-release always; remove DISTRIB_FAMILY.

To replace DISTRIB_FAMILY, there’s now an os_families function using
the standard ID and ID_LIKE information in /etc/os-release.

Fixes #13070; fixes #13071.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

scripts/lib/create-production-venv

@@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 if ZULIP_PATH not in sys.path:
     sys.path.append(ZULIP_PATH)
 
-from scripts.lib.zulip_tools import overwrite_symlink, run, parse_os_release
+from scripts.lib.zulip_tools import os_families, overwrite_symlink, run, parse_os_release
 from scripts.lib.setup_venv import (
     setup_virtualenv, VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
     FEDORA_VENV_DEPENDENCIES
@@ -20,18 +20,16 @@ args = parser.parse_args()
 
 # install dependencies for setting up the virtualenv
 distro_info = parse_os_release()
-vendor = distro_info['ID']
+if "debian" in os_families():
-family = distro_info['DISTRIB_FAMILY']
-if family == 'debian':
     run(["apt-get", "-y", "install"] + VENV_DEPENDENCIES)
-elif family == 'redhat':
+elif "fedora" in os_families():
-    if vendor in ["CentOS", "RedHat"]:
+    if "rhel" in os_families():
         _VENV_DEPS = REDHAT_VENV_DEPENDENCIES
-    elif vendor == "Fedora":
+    else:
         _VENV_DEPS = FEDORA_VENV_DEPENDENCIES
     run(["yum", "-y", "install"] + _VENV_DEPS)
 else:
-    print("Unsupported platform: {}".format(family))
+    print("Unsupported platform: {}".format(distro_info['ID']))
     sys.exit(1)
 
 python_version = sys.version_info[0]

scripts/lib/create-thumbor-venv

@@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 if ZULIP_PATH not in sys.path:
     sys.path.append(ZULIP_PATH)
 
-from scripts.lib.zulip_tools import run, parse_os_release
+from scripts.lib.zulip_tools import os_families, run, parse_os_release
 from scripts.lib.setup_venv import (
     setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES
 )
@@ -19,13 +19,12 @@ args = parser.parse_args()
 
 # install dependencies for setting up the virtualenv
 distro_info = parse_os_release()
-family = distro_info['DISTRIB_FAMILY']
+if "debian" in os_families():
-if family == 'debian':
     run(["apt-get", "-y", "install"] + THUMBOR_VENV_DEPENDENCIES)
-elif family == 'redhat':
+elif "fedora" in os_families():
     run(["yum", "-y", "install"] + YUM_THUMBOR_VENV_DEPENDENCIES)
 else:
-    print("Unsupported platform: {}".format(family))
+    print("Unsupported platform: {}".format(distro_info['ID']))
     sys.exit(1)
 
 venv_name = "zulip-thumbor-venv"

scripts/lib/zulip_tools.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 import datetime
+import functools
 import hashlib
 import logging
 import os
@@ -337,12 +338,12 @@ def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbo
         if verbose:
             print("Keeping used %s: %s" % (dir_type, directory))
 
+@functools.lru_cache(None)
 def parse_os_release():
     # type: () -> Dict[str, str]
     """
     Example of the useful subset of the data:
     {
-     'DISTRIB_FAMILY': 'debian'
      'ID': 'ubuntu',
      'VERSION_ID': '18.04',
      'NAME': 'Ubuntu',
@@ -354,26 +355,6 @@ def parse_os_release():
     we avoid using it, as it is not available on RHEL-based platforms.
     """
     distro_info = {}  # type: Dict[str, str]
-    if os.path.exists("/etc/redhat-release"):
-        with open('/etc/redhat-release', 'r') as fp:
-            info = fp.read().strip().split(' ')
-        vendor = info[0]
-        if vendor == 'CentOS':
-            # E.g. "CentOS Linux release 7.5.1804 (Core)"
-            os_version = vendor.lower() + info[3][0]
-        elif vendor == 'Fedora':
-            # E.g. "Fedora release 29 (Twenty Nine)"
-            os_version = vendor.lower() + info[2]
-        elif vendor == 'Red':
-            # E.g. "Red Hat Enterprise Linux Server release 7.6 (Maipo)"
-            vendor = 'RedHat'
-            os_version = 'rhel' + info[6][0]  # 7
-        distro_info = dict(
-            VERSION_ID=os_version,
-            ID=vendor,
-            DISTRIB_FAMILY='redhat',
-        )
-        return distro_info
     with open('/etc/os-release', 'r') as fp:
         for line in fp:
             line = line.strip()
@@ -383,9 +364,21 @@ def parse_os_release():
                 continue
             k, v = line.split('=', 1)
             [distro_info[k]] = shlex.split(v)
-    distro_info['DISTRIB_FAMILY'] = 'debian'
     return distro_info
 
+@functools.lru_cache(None)
+def os_families() -> Set[str]:
+    """
+    Known families:
+    debian (includes: debian, ubuntu)
+    ubuntu (includes: ubuntu)
+    fedora (includes: fedora, rhel, centos)
+    rhel (includes: rhel, centos)
+    centos (includes: centos)
+    """
+    distro_info = parse_os_release()
+    return {distro_info["ID"], *distro_info.get("ID_LIKE", "").split()}
+
 def file_or_package_hash_updated(paths, hash_name, is_force, package_versions=[]):
     # type: (List[str], str, bool, List[str]) -> bool
     # Check whether the files or package_versions passed as arguments

tools/lib/provision.py

@@ -14,7 +14,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 
 sys.path.append(ZULIP_PATH)
 from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \
-    get_dev_uuid_var_path, FAIL, parse_os_release, \
+    get_dev_uuid_var_path, FAIL, os_families, parse_os_release, \
     overwrite_symlink
 from scripts.lib.setup_venv import (
     VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
@@ -29,28 +29,6 @@ if TYPE_CHECKING:
     # typing_extensions might not be installed yet
     from typing_extensions import NoReturn
 
-SUPPORTED_PLATFORMS = {
-    "ubuntu": [
-        "16.04",  # xenial
-        "18.04",  # bionic
-        "18.10",  # cosmic
-        "19.04",  # disco
-    ],
-    "debian": [
-        "9",   # stretch
-        "10",  # buster
-    ],
-    "CentOS": [
-        "centos7",
-    ],
-    "Fedora": [
-        "fedora29",
-    ],
-    "RedHat": [
-        "rhel7",
-    ]
-}
-
 VAR_DIR_PATH = os.path.join(ZULIP_PATH, 'var')
 
 is_travis = 'TRAVIS' in os.environ
@@ -102,17 +80,26 @@ else:
     # architectures.
     sys.exit(1)
 
-# Ideally we wouldn't need to install a dependency here, before we
-# know the OS version.
-is_rhel_based = os.path.exists("/etc/redhat-release")
-if (not is_rhel_based) and (not os.path.exists("/usr/bin/lsb_release")):
-    run_as_root(["apt-get", "install", "-y", "lsb-release"])
-
 distro_info = parse_os_release()
 vendor = distro_info['ID']
 os_version = distro_info['VERSION_ID']
-family = distro_info['DISTRIB_FAMILY']
+if vendor == "debian" and os_version == "9":  # stretch
-if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vendor]):
+    POSTGRES_VERSION = "9.6"
+elif vendor == "debian" and os_version == "10":  # buster
+    POSTGRES_VERSION = "11"
+elif vendor == "ubuntu" and os_version == "16.04":  # xenial
+    POSTGRES_VERSION = "9.5"
+elif vendor == "ubuntu" and os_version in ["18.04", "18.10"]:  # bionic, cosmic
+    POSTGRES_VERSION = "10"
+elif vendor == "ubuntu" and os_version == "19.04":  # disco
+    POSTGRES_VERSION = "11"
+elif vendor == "fedora" and os_version == "29":
+    POSTGRES_VERSION = "10"
+elif vendor == "rhel" and os_version.startswith("7."):
+    POSTGRES_VERSION = "10"
+elif vendor == "centos" and os_version == "7":
+    POSTGRES_VERSION = "10"
+else:
     logging.critical("Unsupported platform: {} {}".format(vendor, os_version))
     if vendor == 'ubuntu' and os_version == '14.04':
         print()
@@ -122,19 +109,6 @@ if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vend
             print("See: https://zulip.readthedocs.io/en/latest/development/setup-vagrant.html")
     sys.exit(1)
 
-POSTGRES_VERSION_MAP = {
-    ("debian", "9"): "9.6",
-    ("debian", "10"): "11",
-    ("ubuntu", "16.04"): "9.5",
-    ("ubuntu", "18.04"): "10",
-    ("ubuntu", "18.10"): "10",
-    ("ubuntu", "19.04"): "11",
-    ("CentOS", "centos7"): "10",
-    ("Fedora", "fedora29"): "10",
-    ("RedHat", "rhel7"): "10",
-}
-POSTGRES_VERSION = POSTGRES_VERSION_MAP[(vendor, os_version)]
-
 COMMON_DEPENDENCIES = [
     "memcached",
     "rabbitmq-server",
@@ -184,14 +158,14 @@ if vendor == 'debian' and os_version in []:
             "libmsgpack-dev",
         ]
     ]
-elif vendor in ["ubuntu", "debian"]:
+elif "debian" in os_families():
     SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql-{0}",
             "postgresql-{0}-pgroonga",
         ]
     ]
-elif vendor in ["CentOS", "RedHat"]:
+elif "rhel" in os_families():
     SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql{0}-server",
@@ -199,7 +173,7 @@ elif vendor in ["CentOS", "RedHat"]:
             "postgresql{0}-pgroonga",
         ]
     ] + REDHAT_VENV_DEPENDENCIES
-elif vendor == "Fedora":
+elif "fedora" in os_families():
     SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql{0}-server",
@@ -212,7 +186,7 @@ elif vendor == "Fedora":
     ] + FEDORA_VENV_DEPENDENCIES
     BUILD_PGROONGA_FROM_SOURCE = True
 
-if family == 'redhat':
+if "fedora" in os_families():
     TSEARCH_STOPWORDS_PATH = "/usr/pgsql-%s/share/tsearch_data/" % (POSTGRES_VERSION,)
 else:
     TSEARCH_STOPWORDS_PATH = "/usr/share/postgresql/%s/tsearch_data/" % (POSTGRES_VERSION,)
@@ -231,9 +205,9 @@ def install_system_deps():
     # By doing list -> set -> list conversion, we remove duplicates.
     deps_to_install = sorted(set(SYSTEM_DEPENDENCIES))
 
-    if family == 'redhat':
+    if "fedora" in os_families():
         install_yum_deps(deps_to_install)
-    elif vendor in ["debian", "ubuntu"]:
+    elif "debian" in os_families():
         install_apt_deps(deps_to_install)
     else:
         raise AssertionError("Invalid vendor")
@@ -267,7 +241,7 @@ def install_yum_deps(deps_to_install):
     # Error: Package: moreutils-0.49-2.el7.x86_64 (epel)
     #        Requires: perl(IPC::Run)
     yum_extra_flags = []  # type: List[str]
-    if vendor == 'RedHat':
+    if vendor == "rhel":
         exitcode, subs_status = subprocess.getstatusoutput("sudo subscription-manager status")
         if exitcode == 1:
             # TODO this might overkill since `subscription-manager` is already
@@ -279,7 +253,7 @@ def install_yum_deps(deps_to_install):
                 print("Unrecognized output. `subscription-manager` might not be available")
 
     run_as_root(["yum", "install", "-y"] + yum_extra_flags + deps_to_install)
-    if vendor in ["CentOS", "RedHat"]:
+    if "rhel" in os_families():
         # This is how a pip3 is installed to /usr/bin in CentOS/RHEL
         # for python35 and later.
         run_as_root(["python36", "-m", "ensurepip"])
@@ -328,7 +302,7 @@ def main(options):
 
     for apt_depedency in SYSTEM_DEPENDENCIES:
         sha_sum.update(apt_depedency.encode('utf8'))
-    if vendor in ["ubuntu", "debian"]:
+    if "debian" in os_families():
         sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
     else:
         # hash the content of setup-yum-repo and build-*
@@ -395,7 +369,9 @@ def main(options):
         run_as_root(["service", "redis-server", "restart"])
         run_as_root(["service", "memcached", "restart"])
         run_as_root(["service", "postgresql", "restart"])
-    elif family == 'redhat':
+    elif "fedora" in os_families():
+        # These platforms don't enable and start services on
+        # installing their package, so we do that here.
         for service in ["postgresql-%s" % (POSTGRES_VERSION,), "rabbitmq-server", "memcached", "redis"]:
             run_as_root(["systemctl", "enable", service], sudo_args = ['-H'])
             run_as_root(["systemctl", "start", service], sudo_args = ['-H'])