Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

parse_os_release: Use /etc/os-release always; remove DISTRIB_FAMILY. 

To replace DISTRIB_FAMILY, there’s now an os_families function using
the standard ID and ID_LIKE information in /etc/os-release.

Fixes #13070; fixes #13071.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
This commit is contained in:
Anders Kaseorg 2019-08-29 15:14:43 -07:00 committed by Tim Abbott
parent 875002108f
commit 096ef1445f
4 changed files with 55 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scripts/lib/create-production-venv
View File

@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
if ZULIP_PATH not in sys.path:
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import overwrite_symlink, run, parse_os_release
from scripts.lib.zulip_tools import os_families, overwrite_symlink, run, parse_os_release
from scripts.lib.setup_venv import (
setup_virtualenv, VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
FEDORA_VENV_DEPENDENCIES
@ -20,18 +20,16 @@ args = parser.parse_args()
# install dependencies for setting up the virtualenv
distro_info = parse_os_release()
vendor = distro_info['ID']
family = distro_info['DISTRIB_FAMILY']
if family == 'debian':
if "debian" in os_families():
run(["apt-get", "-y", "install"] + VENV_DEPENDENCIES)
elif family == 'redhat':
if vendor in ["CentOS", "RedHat"]:
elif "fedora" in os_families():
if "rhel" in os_families():
_VENV_DEPS = REDHAT_VENV_DEPENDENCIES
elif vendor == "Fedora":
else:
_VENV_DEPS = FEDORA_VENV_DEPENDENCIES
run(["yum", "-y", "install"] + _VENV_DEPS)
else:
print("Unsupported platform: {}".format(family))
print("Unsupported platform: {}".format(distro_info['ID']))
sys.exit(1)
python_version = sys.version_info[0]

9
scripts/lib/create-thumbor-venv
View File

@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
if ZULIP_PATH not in sys.path:
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import run, parse_os_release
from scripts.lib.zulip_tools import os_families, run, parse_os_release
from scripts.lib.setup_venv import (
setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES
)
@ -19,13 +19,12 @@ args = parser.parse_args()
# install dependencies for setting up the virtualenv
distro_info = parse_os_release()
family = distro_info['DISTRIB_FAMILY']
if family == 'debian':
if "debian" in os_families():
run(["apt-get", "-y", "install"] + THUMBOR_VENV_DEPENDENCIES)
elif family == 'redhat':
elif "fedora" in os_families():
run(["yum", "-y", "install"] + YUM_THUMBOR_VENV_DEPENDENCIES)
else:
print("Unsupported platform: {}".format(family))
print("Unsupported platform: {}".format(distro_info['ID']))
sys.exit(1)
venv_name = "zulip-thumbor-venv"

37
scripts/lib/zulip_tools.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
import datetime
import functools
import hashlib
import logging
import os
@ -337,12 +338,12 @@ def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbo
if verbose:
print("Keeping used %s: %s" % (dir_type, directory))
@functools.lru_cache(None)
def parse_os_release():
# type: () -> Dict[str, str]
"""
Example of the useful subset of the data:
{
'DISTRIB_FAMILY': 'debian'
'ID': 'ubuntu',
'VERSION_ID': '18.04',
'NAME': 'Ubuntu',
@ -354,26 +355,6 @@ def parse_os_release():
we avoid using it, as it is not available on RHEL-based platforms.
"""
distro_info = {} # type: Dict[str, str]
if os.path.exists("/etc/redhat-release"):
with open('/etc/redhat-release', 'r') as fp:
info = fp.read().strip().split(' ')
vendor = info[0]
if vendor == 'CentOS':
# E.g. "CentOS Linux release 7.5.1804 (Core)"
os_version = vendor.lower() + info[3][0]
elif vendor == 'Fedora':
# E.g. "Fedora release 29 (Twenty Nine)"
os_version = vendor.lower() + info[2]
elif vendor == 'Red':
# E.g. "Red Hat Enterprise Linux Server release 7.6 (Maipo)"
vendor = 'RedHat'
os_version = 'rhel' + info[6][0] # 7
distro_info = dict(
VERSION_ID=os_version,
ID=vendor,
DISTRIB_FAMILY='redhat',
)
return distro_info
with open('/etc/os-release', 'r') as fp:
for line in fp:
line = line.strip()
@ -383,9 +364,21 @@ def parse_os_release():
continue
k, v = line.split('=', 1)
[distro_info[k]] = shlex.split(v)
distro_info['DISTRIB_FAMILY'] = 'debian'
return distro_info
@functools.lru_cache(None)
def os_families() -> Set[str]:
"""
Known families:
debian (includes: debian, ubuntu)
ubuntu (includes: ubuntu)
fedora (includes: fedora, rhel, centos)
rhel (includes: rhel, centos)
centos (includes: centos)
"""
distro_info = parse_os_release()
return {distro_info["ID"], *distro_info.get("ID_LIKE", "").split()}
def file_or_package_hash_updated(paths, hash_name, is_force, package_versions=[]):
# type: (List[str], str, bool, List[str]) -> bool
# Check whether the files or package_versions passed as arguments

84
tools/lib/provision.py
View File

@ -14,7 +14,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \
get_dev_uuid_var_path, FAIL, parse_os_release, \
get_dev_uuid_var_path, FAIL, os_families, parse_os_release, \
overwrite_symlink
from scripts.lib.setup_venv import (
VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
@ -29,28 +29,6 @@ if TYPE_CHECKING:
# typing_extensions might not be installed yet
from typing_extensions import NoReturn
SUPPORTED_PLATFORMS = {
"ubuntu": [
"16.04", # xenial
"18.04", # bionic
"18.10", # cosmic
"19.04", # disco
],
"debian": [
"9", # stretch
"10", # buster
],
"CentOS": [
"centos7",
],
"Fedora": [
"fedora29",
],
"RedHat": [
"rhel7",
]
}
VAR_DIR_PATH = os.path.join(ZULIP_PATH, 'var')
is_travis = 'TRAVIS' in os.environ
@ -102,17 +80,26 @@ else:
# architectures.
sys.exit(1)
# Ideally we wouldn't need to install a dependency here, before we
# know the OS version.
is_rhel_based = os.path.exists("/etc/redhat-release")
if (not is_rhel_based) and (not os.path.exists("/usr/bin/lsb_release")):
run_as_root(["apt-get", "install", "-y", "lsb-release"])
distro_info = parse_os_release()
vendor = distro_info['ID']
os_version = distro_info['VERSION_ID']
family = distro_info['DISTRIB_FAMILY']
if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vendor]):
if vendor == "debian" and os_version == "9": # stretch
POSTGRES_VERSION = "9.6"
elif vendor == "debian" and os_version == "10": # buster
POSTGRES_VERSION = "11"
elif vendor == "ubuntu" and os_version == "16.04": # xenial
POSTGRES_VERSION = "9.5"
elif vendor == "ubuntu" and os_version in ["18.04", "18.10"]: # bionic, cosmic
POSTGRES_VERSION = "10"
elif vendor == "ubuntu" and os_version == "19.04": # disco
POSTGRES_VERSION = "11"
elif vendor == "fedora" and os_version == "29":
POSTGRES_VERSION = "10"
elif vendor == "rhel" and os_version.startswith("7."):
POSTGRES_VERSION = "10"
elif vendor == "centos" and os_version == "7":
POSTGRES_VERSION = "10"
else:
logging.critical("Unsupported platform: {} {}".format(vendor, os_version))
if vendor == 'ubuntu' and os_version == '14.04':
print()
@ -122,19 +109,6 @@ if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vend
print("See: https://zulip.readthedocs.io/en/latest/development/setup-vagrant.html")
sys.exit(1)
POSTGRES_VERSION_MAP = {
("debian", "9"): "9.6",
("debian", "10"): "11",
("ubuntu", "16.04"): "9.5",
("ubuntu", "18.04"): "10",
("ubuntu", "18.10"): "10",
("ubuntu", "19.04"): "11",
("CentOS", "centos7"): "10",
("Fedora", "fedora29"): "10",
("RedHat", "rhel7"): "10",
}
POSTGRES_VERSION = POSTGRES_VERSION_MAP[(vendor, os_version)]
COMMON_DEPENDENCIES = [
"memcached",
"rabbitmq-server",
@ -184,14 +158,14 @@ if vendor == 'debian' and os_version in []:
"libmsgpack-dev",
]
]
elif vendor in ["ubuntu", "debian"]:
elif "debian" in os_families():
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
"postgresql-{0}-pgroonga",
]
]
elif vendor in ["CentOS", "RedHat"]:
elif "rhel" in os_families():
SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql{0}-server",
@ -199,7 +173,7 @@ elif vendor in ["CentOS", "RedHat"]:
"postgresql{0}-pgroonga",
]
] + REDHAT_VENV_DEPENDENCIES
elif vendor == "Fedora":
elif "fedora" in os_families():
SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql{0}-server",
@ -212,7 +186,7 @@ elif vendor == "Fedora":
] + FEDORA_VENV_DEPENDENCIES
BUILD_PGROONGA_FROM_SOURCE = True
if family == 'redhat':
if "fedora" in os_families():
TSEARCH_STOPWORDS_PATH = "/usr/pgsql-%s/share/tsearch_data/" % (POSTGRES_VERSION,)
else:
TSEARCH_STOPWORDS_PATH = "/usr/share/postgresql/%s/tsearch_data/" % (POSTGRES_VERSION,)
@ -231,9 +205,9 @@ def install_system_deps():
# By doing list -> set -> list conversion, we remove duplicates.
deps_to_install = sorted(set(SYSTEM_DEPENDENCIES))
if family == 'redhat':
if "fedora" in os_families():
install_yum_deps(deps_to_install)
elif vendor in ["debian", "ubuntu"]:
elif "debian" in os_families():
install_apt_deps(deps_to_install)
else:
raise AssertionError("Invalid vendor")
@ -267,7 +241,7 @@ def install_yum_deps(deps_to_install):
# Error: Package: moreutils-0.49-2.el7.x86_64 (epel)
# Requires: perl(IPC::Run)
yum_extra_flags = [] # type: List[str]
if vendor == 'RedHat':
if vendor == "rhel":
exitcode, subs_status = subprocess.getstatusoutput("sudo subscription-manager status")
if exitcode == 1:
# TODO this might overkill since `subscription-manager` is already
@ -279,7 +253,7 @@ def install_yum_deps(deps_to_install):
print("Unrecognized output. `subscription-manager` might not be available")
run_as_root(["yum", "install", "-y"] + yum_extra_flags + deps_to_install)
if vendor in ["CentOS", "RedHat"]:
if "rhel" in os_families():
# This is how a pip3 is installed to /usr/bin in CentOS/RHEL
# for python35 and later.
run_as_root(["python36", "-m", "ensurepip"])
@ -328,7 +302,7 @@ def main(options):
for apt_depedency in SYSTEM_DEPENDENCIES:
sha_sum.update(apt_depedency.encode('utf8'))
if vendor in ["ubuntu", "debian"]:
if "debian" in os_families():
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
else:
# hash the content of setup-yum-repo and build-*
@ -395,7 +369,9 @@ def main(options):
run_as_root(["service", "redis-server", "restart"])
run_as_root(["service", "memcached", "restart"])
run_as_root(["service", "postgresql", "restart"])
elif family == 'redhat':
elif "fedora" in os_families():
# These platforms don't enable and start services on
# installing their package, so we do that here.
for service in ["postgresql-%s" % (POSTGRES_VERSION,), "rabbitmq-server", "memcached", "redis"]:
run_as_root(["systemctl", "enable", service], sudo_args = ['-H'])
run_as_root(["systemctl", "start", service], sudo_args = ['-H'])