diff --git a/scripts/lib/create-production-venv b/scripts/lib/create-production-venv index 493e9dd980..6ba7bfb798 100755 --- a/scripts/lib/create-production-venv +++ b/scripts/lib/create-production-venv @@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import overwrite_symlink, run, parse_lsb_release +from scripts.lib.zulip_tools import overwrite_symlink, run, parse_os_release from scripts.lib.setup_venv import ( setup_virtualenv, VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES, FEDORA_VENV_DEPENDENCIES @@ -19,8 +19,8 @@ parser.add_argument("deploy_path") args = parser.parse_args() # install dependencies for setting up the virtualenv -distro_info = parse_lsb_release() -vendor = distro_info['DISTRIB_ID'] +distro_info = parse_os_release() +vendor = distro_info['ID'] family = distro_info['DISTRIB_FAMILY'] if family == 'debian': run(["apt-get", "-y", "install"] + VENV_DEPENDENCIES) @@ -31,7 +31,7 @@ elif family == 'redhat': _VENV_DEPS = FEDORA_VENV_DEPENDENCIES run(["yum", "-y", "install"] + _VENV_DEPS) else: - print("Unsupported platform: {}".format(vendor)) + print("Unsupported platform: {}".format(family)) sys.exit(1) python_version = sys.version_info[0] diff --git a/scripts/lib/create-thumbor-venv b/scripts/lib/create-thumbor-venv index 656c6ce25c..42143180ca 100755 --- a/scripts/lib/create-thumbor-venv +++ b/scripts/lib/create-thumbor-venv @@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run, parse_lsb_release +from scripts.lib.zulip_tools import run, parse_os_release from scripts.lib.setup_venv import ( setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES ) @@ -18,15 +18,14 @@ parser.add_argument("deploy_path") args = parser.parse_args() # install dependencies for setting up the virtualenv -distro_info = parse_lsb_release() -vendor = distro_info['DISTRIB_ID'] +distro_info = parse_os_release() family = distro_info['DISTRIB_FAMILY'] if family == 'debian': run(["apt-get", "-y", "install"] + THUMBOR_VENV_DEPENDENCIES) elif family == 'redhat': run(["yum", "-y", "install"] + YUM_THUMBOR_VENV_DEPENDENCIES) else: - print("Unsupported platform: {}".format(vendor)) + print("Unsupported platform: {}".format(family)) sys.exit(1) venv_name = "zulip-thumbor-venv" diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py index 2f8d3acdc6..f0675d3002 100755 --- a/scripts/lib/zulip_tools.py +++ b/scripts/lib/zulip_tools.py @@ -337,8 +337,22 @@ def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbo if verbose: print("Keeping used %s: %s" % (dir_type, directory)) -def parse_lsb_release(): +def parse_os_release(): # type: () -> Dict[str, str] + """ + Example of the useful subset of the data: + { + 'DISTRIB_FAMILY': 'debian' + 'ID': 'ubuntu', + 'VERSION_ID': '18.04', + 'NAME': 'Ubuntu', + 'VERSION': '18.04.3 LTS (Bionic Beaver)', + 'PRETTY_NAME': 'Ubuntu 18.04.3 LTS', + } + + VERSION_CODENAME (e.g. 'bionic') is nice and human-readable, but + we avoid using it, as it is not available on RHEL-based platforms. + """ distro_info = {} # type: Dict[str, str] if os.path.exists("/etc/redhat-release"): with open('/etc/redhat-release', 'r') as fp: @@ -346,44 +360,30 @@ def parse_lsb_release(): vendor = info[0] if vendor == 'CentOS': # E.g. "CentOS Linux release 7.5.1804 (Core)" - codename = vendor.lower() + info[3][0] + os_version = vendor.lower() + info[3][0] elif vendor == 'Fedora': # E.g. "Fedora release 29 (Twenty Nine)" - codename = vendor.lower() + info[2] + os_version = vendor.lower() + info[2] elif vendor == 'Red': # E.g. "Red Hat Enterprise Linux Server release 7.6 (Maipo)" vendor = 'RedHat' - codename = 'rhel' + info[6][0] # 7 + os_version = 'rhel' + info[6][0] # 7 distro_info = dict( - DISTRIB_CODENAME=codename, - DISTRIB_ID=vendor, + VERSION_ID=os_version, + ID=vendor, DISTRIB_FAMILY='redhat', ) return distro_info - try: - # For performance reasons, we read /etc/lsb-release directly, - # rather than using the lsb_release command; this saves ~50ms - # in several places in provisioning and the installer - with open('/etc/lsb-release', 'r') as fp: - data = [line.strip().split('=') for line in fp] - for k, v in data: - if k not in ['DISTRIB_CODENAME', 'DISTRIB_ID']: - # We only return to the caller the values that we get - # from lsb_release in the exception code path. + with open('/etc/os-release', 'r') as fp: + for line in fp: + line = line.strip() + if not line or line.startswith('#'): + # The line may be blank or a comment, see: + # https://www.freedesktop.org/software/systemd/man/os-release.html continue - distro_info[k] = v - distro_info['DISTRIB_FAMILY'] = 'debian' - except FileNotFoundError: - # Unfortunately, Debian stretch doesn't yet have an - # /etc/lsb-release, so we instead fetch the pieces of data - # that we use from the `lsb_release` command directly. - vendor = subprocess_text_output(["lsb_release", "-is"]) - codename = subprocess_text_output(["lsb_release", "-cs"]) - distro_info = dict( - DISTRIB_CODENAME=codename, - DISTRIB_ID=vendor, - DISTRIB_FAMILY='debian', - ) + k, v = line.split('=', 1) + [distro_info[k]] = shlex.split(v) + distro_info['DISTRIB_FAMILY'] = 'debian' return distro_info def file_or_package_hash_updated(paths, hash_name, is_force, package_versions=[]): diff --git a/scripts/zulip-puppet-apply b/scripts/zulip-puppet-apply index 218c227f64..fa8498e2fd 100755 --- a/scripts/zulip-puppet-apply +++ b/scripts/zulip-puppet-apply @@ -5,7 +5,7 @@ import sys import subprocess import configparser import re -from lib.zulip_tools import parse_lsb_release, assert_running_as_root +from lib.zulip_tools import parse_os_release, assert_running_as_root assert_running_as_root() BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -21,8 +21,8 @@ config = configparser.RawConfigParser() config.read("/etc/zulip/zulip.conf") if not os.path.exists("/etc/puppet/hiera.yaml"): - codename = parse_lsb_release()['DISTRIB_CODENAME'] - if codename in ["stretch", "xenial"]: + distro_info = parse_os_release() + if (distro_info['ID'], distro_info['VERSION_ID']) in [('debian', '9'), ('ubuntu', '16.04')]: # Suppress warnings in old puppet about hiera.yaml not existing. open("/etc/puppet/hiera.yaml", "a").close() diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 1280281831..13d4db7e51 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -14,7 +14,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f sys.path.append(ZULIP_PATH) from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \ - get_dev_uuid_var_path, FAIL, parse_lsb_release, \ + get_dev_uuid_var_path, FAIL, parse_os_release, \ overwrite_symlink from scripts.lib.setup_venv import ( VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES, @@ -30,15 +30,15 @@ if TYPE_CHECKING: from typing_extensions import NoReturn SUPPORTED_PLATFORMS = { - "Ubuntu": [ - "xenial", - "bionic", - "cosmic", - "disco", + "ubuntu": [ + "16.04", # xenial + "18.04", # bionic + "18.10", # cosmic + "19.04", # disco ], - "Debian": [ - "stretch", - "buster", + "debian": [ + "9", # stretch + "10", # buster ], "CentOS": [ "centos7", @@ -104,18 +104,18 @@ else: sys.exit(1) # Ideally we wouldn't need to install a dependency here, before we -# know the codename. +# know the OS version. is_rhel_based = os.path.exists("/etc/redhat-release") if (not is_rhel_based) and (not os.path.exists("/usr/bin/lsb_release")): run_as_root(["apt-get", "install", "-y", "lsb-release"]) -distro_info = parse_lsb_release() -vendor = distro_info['DISTRIB_ID'] -codename = distro_info['DISTRIB_CODENAME'] +distro_info = parse_os_release() +vendor = distro_info['ID'] +os_version = distro_info['VERSION_ID'] family = distro_info['DISTRIB_FAMILY'] -if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]): - logging.critical("Unsupported platform: {} {}".format(vendor, codename)) - if codename == 'trusty': +if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vendor]): + logging.critical("Unsupported platform: {} {}".format(vendor, os_version)) + if vendor == 'ubuntu' and os_version == '14.04': print() print("Ubuntu Trusty reached end-of-life upstream and is no longer a supported platform for Zulip") if os.path.exists('/home/vagrant'): @@ -124,17 +124,17 @@ if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor sys.exit(1) POSTGRES_VERSION_MAP = { - "stretch": "9.6", - "buster": "11", - "xenial": "9.5", - "bionic": "10", - "cosmic": "10", - "disco": "11", - "centos7": "10", - "fedora29": "10", - "rhel7": "10", + ("debian", "9"): "9.6", + ("debian", "10"): "11", + ("ubuntu", "16.04"): "9.5", + ("ubuntu", "18.04"): "10", + ("ubuntu", "18.10"): "10", + ("ubuntu", "19.04"): "11", + ("CentOS", "centos7"): "10", + ("Fedora", "fedora29"): "10", + ("RedHat", "rhel7"): "10", } -POSTGRES_VERSION = POSTGRES_VERSION_MAP[codename] +POSTGRES_VERSION = POSTGRES_VERSION_MAP[(vendor, os_version)] COMMON_DEPENDENCIES = [ "memcached", @@ -173,43 +173,42 @@ COMMON_YUM_DEPENDENCIES = COMMON_DEPENDENCIES + [ BUILD_TSEARCH_FROM_SOURCE = False BUILD_PGROONGA_FROM_SOURCE = False -if vendor in ["Ubuntu", "Debian"]: - if codename in ("cosmic", "disco"): - # For platforms without a tsearch-extras package distributed - # from our PPA, we need to build from source. - BUILD_TSEARCH_FROM_SOURCE = True - SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ - pkg.format(POSTGRES_VERSION) for pkg in [ - "postgresql-{0}", - "postgresql-{0}-pgroonga", - # Dependency for building tsearch_extras from source - "postgresql-server-dev-{0}", - ] +if vendor == "ubuntu" and os_version in ("18.10", "19.04"): + # For platforms without a tsearch-extras package distributed + # from our PPA, we need to build from source. + BUILD_TSEARCH_FROM_SOURCE = True + SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ + pkg.format(POSTGRES_VERSION) for pkg in [ + "postgresql-{0}", + "postgresql-{0}-pgroonga", + # Dependency for building tsearch_extras from source + "postgresql-server-dev-{0}", ] - elif codename == "buster": - # For platforms without a tsearch-extras package distributed - # from our PPA or a pgroonga release, we need to build both - # from source. - BUILD_PGROONGA_FROM_SOURCE = True - BUILD_TSEARCH_FROM_SOURCE = True - SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ - pkg.format(POSTGRES_VERSION) for pkg in [ - "postgresql-{0}", - # Dependency for building tsearch_extras from source - "postgresql-server-dev-{0}", - # Dependency for building pgroonga from source - "libgroonga-dev", - "libmsgpack-dev", - ] + ] +elif vendor == 'debian' and os_version == "10": + # For platforms without a tsearch-extras package distributed + # from our PPA or a pgroonga release, we need to build both + # from source. + BUILD_PGROONGA_FROM_SOURCE = True + BUILD_TSEARCH_FROM_SOURCE = True + SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ + pkg.format(POSTGRES_VERSION) for pkg in [ + "postgresql-{0}", + # Dependency for building tsearch_extras from source + "postgresql-server-dev-{0}", + # Dependency for building pgroonga from source + "libgroonga-dev", + "libmsgpack-dev", ] - else: - SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ - pkg.format(POSTGRES_VERSION) for pkg in [ - "postgresql-{0}", - "postgresql-{0}-pgroonga", - "postgresql-{0}-tsearch-extras", - ] + ] +elif vendor in ["ubuntu", "debian"]: + SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ + pkg.format(POSTGRES_VERSION) for pkg in [ + "postgresql-{0}", + "postgresql-{0}-pgroonga", + "postgresql-{0}-tsearch-extras", ] + ] elif vendor in ["CentOS", "RedHat"]: SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [ pkg.format(POSTGRES_VERSION) for pkg in [ @@ -255,7 +254,7 @@ def install_system_deps(): if family == 'redhat': install_yum_deps(deps_to_install) - elif vendor in ["Debian", "Ubuntu"]: + elif vendor in ["debian", "ubuntu"]: install_apt_deps(deps_to_install) else: raise AssertionError("Invalid vendor") @@ -352,7 +351,7 @@ def main(options): for apt_depedency in SYSTEM_DEPENDENCIES: sha_sum.update(apt_depedency.encode('utf8')) - if vendor in ["Ubuntu", "Debian"]: + if vendor in ["ubuntu", "debian"]: sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read()) else: # hash the content of setup-yum-repo and build-* diff --git a/zerver/management/commands/backup.py b/zerver/management/commands/backup.py index 3740ce598d..3c6cc5adec 100644 --- a/zerver/management/commands/backup.py +++ b/zerver/management/commands/backup.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import connection from django.utils.timezone import now as timezone_now -from scripts.lib.zulip_tools import parse_lsb_release, run, TIMESTAMP_FORMAT +from scripts.lib.zulip_tools import parse_os_release, run, TIMESTAMP_FORMAT from version import ZULIP_VERSION from zerver.lib.management import ZulipBaseCommand from zerver.logging_handlers import try_git_describe @@ -46,7 +46,7 @@ class Command(ZulipBaseCommand): with open(os.path.join(tmp, "zulip-backup", "os-version"), "w") as f: print( - "{DISTRIB_ID} {DISTRIB_CODENAME}".format(**parse_lsb_release()), + "{ID} {VERSION_ID}".format(**parse_os_release()), file=f, ) members.append("zulip-backup/os-version")