Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

parse_lsb_release: Use /etc/os-release instead of /etc/lsb-release.

This commit is contained in:
rht 2019-08-25 00:23:14 +01:00 committed by Tim Abbott
parent 83a012eaeb
commit 07808e35be
6 changed files with 102 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/lib/create-production-venv
View File

@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
if ZULIP_PATH not in sys.path:
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import overwrite_symlink, run, parse_lsb_release
from scripts.lib.zulip_tools import overwrite_symlink, run, parse_os_release
from scripts.lib.setup_venv import (
setup_virtualenv, VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
FEDORA_VENV_DEPENDENCIES
@ -19,8 +19,8 @@ parser.add_argument("deploy_path")
args = parser.parse_args()
# install dependencies for setting up the virtualenv
distro_info = parse_lsb_release()
vendor = distro_info['DISTRIB_ID']
distro_info = parse_os_release()
vendor = distro_info['ID']
family = distro_info['DISTRIB_FAMILY']
if family == 'debian':
run(["apt-get", "-y", "install"] + VENV_DEPENDENCIES)
@ -31,7 +31,7 @@ elif family == 'redhat':
_VENV_DEPS = FEDORA_VENV_DEPENDENCIES
run(["yum", "-y", "install"] + _VENV_DEPS)
else:
print("Unsupported platform: {}".format(vendor))
print("Unsupported platform: {}".format(family))
sys.exit(1)
python_version = sys.version_info[0]

7
scripts/lib/create-thumbor-venv
View File

@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
if ZULIP_PATH not in sys.path:
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import run, parse_lsb_release
from scripts.lib.zulip_tools import run, parse_os_release
from scripts.lib.setup_venv import (
setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES
)
@ -18,15 +18,14 @@ parser.add_argument("deploy_path")
args = parser.parse_args()
# install dependencies for setting up the virtualenv
distro_info = parse_lsb_release()
vendor = distro_info['DISTRIB_ID']
distro_info = parse_os_release()
family = distro_info['DISTRIB_FAMILY']
if family == 'debian':
run(["apt-get", "-y", "install"] + THUMBOR_VENV_DEPENDENCIES)
elif family == 'redhat':
run(["yum", "-y", "install"] + YUM_THUMBOR_VENV_DEPENDENCIES)
else:
print("Unsupported platform: {}".format(vendor))
print("Unsupported platform: {}".format(family))
sys.exit(1)
venv_name = "zulip-thumbor-venv"

58
scripts/lib/zulip_tools.py
View File

@ -337,8 +337,22 @@ def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbo
if verbose:
print("Keeping used %s: %s" % (dir_type, directory))
def parse_lsb_release():
def parse_os_release():
# type: () -> Dict[str, str]
"""
Example of the useful subset of the data:
{
'DISTRIB_FAMILY': 'debian'
'ID': 'ubuntu',
'VERSION_ID': '18.04',
'NAME': 'Ubuntu',
'VERSION': '18.04.3 LTS (Bionic Beaver)',
'PRETTY_NAME': 'Ubuntu 18.04.3 LTS',
}
VERSION_CODENAME (e.g. 'bionic') is nice and human-readable, but
we avoid using it, as it is not available on RHEL-based platforms.
"""
distro_info = {} # type: Dict[str, str]
if os.path.exists("/etc/redhat-release"):
with open('/etc/redhat-release', 'r') as fp:
@ -346,44 +360,30 @@ def parse_lsb_release():
vendor = info[0]
if vendor == 'CentOS':
# E.g. "CentOS Linux release 7.5.1804 (Core)"
codename = vendor.lower() + info[3][0]
os_version = vendor.lower() + info[3][0]
elif vendor == 'Fedora':
# E.g. "Fedora release 29 (Twenty Nine)"
codename = vendor.lower() + info[2]
os_version = vendor.lower() + info[2]
elif vendor == 'Red':
# E.g. "Red Hat Enterprise Linux Server release 7.6 (Maipo)"
vendor = 'RedHat'
codename = 'rhel' + info[6][0] # 7
os_version = 'rhel' + info[6][0] # 7
distro_info = dict(
DISTRIB_CODENAME=codename,
DISTRIB_ID=vendor,
VERSION_ID=os_version,
ID=vendor,
DISTRIB_FAMILY='redhat',
)
return distro_info
try:
# For performance reasons, we read /etc/lsb-release directly,
# rather than using the lsb_release command; this saves ~50ms
# in several places in provisioning and the installer
with open('/etc/lsb-release', 'r') as fp:
data = [line.strip().split('=') for line in fp]
for k, v in data:
if k not in ['DISTRIB_CODENAME', 'DISTRIB_ID']:
# We only return to the caller the values that we get
# from lsb_release in the exception code path.
with open('/etc/os-release', 'r') as fp:
for line in fp:
line = line.strip()
if not line or line.startswith('#'):
# The line may be blank or a comment, see:
# https://www.freedesktop.org/software/systemd/man/os-release.html
continue
distro_info[k] = v
distro_info['DISTRIB_FAMILY'] = 'debian'
except FileNotFoundError:
# Unfortunately, Debian stretch doesn't yet have an
# /etc/lsb-release, so we instead fetch the pieces of data
# that we use from the `lsb_release` command directly.
vendor = subprocess_text_output(["lsb_release", "-is"])
codename = subprocess_text_output(["lsb_release", "-cs"])
distro_info = dict(
DISTRIB_CODENAME=codename,
DISTRIB_ID=vendor,
DISTRIB_FAMILY='debian',
)
k, v = line.split('=', 1)
[distro_info[k]] = shlex.split(v)
distro_info['DISTRIB_FAMILY'] = 'debian'
return distro_info
def file_or_package_hash_updated(paths, hash_name, is_force, package_versions=[]):

6
scripts/zulip-puppet-apply
View File

@ -5,7 +5,7 @@ import sys
import subprocess
import configparser
import re
from lib.zulip_tools import parse_lsb_release, assert_running_as_root
from lib.zulip_tools import parse_os_release, assert_running_as_root
assert_running_as_root()
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@ -21,8 +21,8 @@ config = configparser.RawConfigParser()
config.read("/etc/zulip/zulip.conf")
if not os.path.exists("/etc/puppet/hiera.yaml"):
codename = parse_lsb_release()['DISTRIB_CODENAME']
if codename in ["stretch", "xenial"]:
distro_info = parse_os_release()
if (distro_info['ID'], distro_info['VERSION_ID']) in [('debian', '9'), ('ubuntu', '16.04')]:
# Suppress warnings in old puppet about hiera.yaml not existing.
open("/etc/puppet/hiera.yaml", "a").close()

123
tools/lib/provision.py
View File

@ -14,7 +14,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \
get_dev_uuid_var_path, FAIL, parse_lsb_release, \
get_dev_uuid_var_path, FAIL, parse_os_release, \
overwrite_symlink
from scripts.lib.setup_venv import (
VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
@ -30,15 +30,15 @@ if TYPE_CHECKING:
from typing_extensions import NoReturn
SUPPORTED_PLATFORMS = {
"Ubuntu": [
"xenial",
"bionic",
"cosmic",
"disco",
"ubuntu": [
"16.04", # xenial
"18.04", # bionic
"18.10", # cosmic
"19.04", # disco
],
"Debian": [
"stretch",
"buster",
"debian": [
"9", # stretch
"10", # buster
],
"CentOS": [
"centos7",
@ -104,18 +104,18 @@ else:
sys.exit(1)
# Ideally we wouldn't need to install a dependency here, before we
# know the codename.
# know the OS version.
is_rhel_based = os.path.exists("/etc/redhat-release")
if (not is_rhel_based) and (not os.path.exists("/usr/bin/lsb_release")):
run_as_root(["apt-get", "install", "-y", "lsb-release"])
distro_info = parse_lsb_release()
vendor = distro_info['DISTRIB_ID']
codename = distro_info['DISTRIB_CODENAME']
distro_info = parse_os_release()
vendor = distro_info['ID']
os_version = distro_info['VERSION_ID']
family = distro_info['DISTRIB_FAMILY']
if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]):
logging.critical("Unsupported platform: {} {}".format(vendor, codename))
if codename == 'trusty':
if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vendor]):
logging.critical("Unsupported platform: {} {}".format(vendor, os_version))
if vendor == 'ubuntu' and os_version == '14.04':
print()
print("Ubuntu Trusty reached end-of-life upstream and is no longer a supported platform for Zulip")
if os.path.exists('/home/vagrant'):
@ -124,17 +124,17 @@ if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor
sys.exit(1)
POSTGRES_VERSION_MAP = {
"stretch": "9.6",
"buster": "11",
"xenial": "9.5",
"bionic": "10",
"cosmic": "10",
"disco": "11",
"centos7": "10",
"fedora29": "10",
"rhel7": "10",
("debian", "9"): "9.6",
("debian", "10"): "11",
("ubuntu", "16.04"): "9.5",
("ubuntu", "18.04"): "10",
("ubuntu", "18.10"): "10",
("ubuntu", "19.04"): "11",
("CentOS", "centos7"): "10",
("Fedora", "fedora29"): "10",
("RedHat", "rhel7"): "10",
}
POSTGRES_VERSION = POSTGRES_VERSION_MAP[codename]
POSTGRES_VERSION = POSTGRES_VERSION_MAP[(vendor, os_version)]
COMMON_DEPENDENCIES = [
"memcached",
@ -173,43 +173,42 @@ COMMON_YUM_DEPENDENCIES = COMMON_DEPENDENCIES + [
BUILD_TSEARCH_FROM_SOURCE = False
BUILD_PGROONGA_FROM_SOURCE = False
if vendor in ["Ubuntu", "Debian"]:
if codename in ("cosmic", "disco"):
# For platforms without a tsearch-extras package distributed
# from our PPA, we need to build from source.
BUILD_TSEARCH_FROM_SOURCE = True
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
"postgresql-{0}-pgroonga",
# Dependency for building tsearch_extras from source
"postgresql-server-dev-{0}",
]
if vendor == "ubuntu" and os_version in ("18.10", "19.04"):
# For platforms without a tsearch-extras package distributed
# from our PPA, we need to build from source.
BUILD_TSEARCH_FROM_SOURCE = True
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
"postgresql-{0}-pgroonga",
# Dependency for building tsearch_extras from source
"postgresql-server-dev-{0}",
]
elif codename == "buster":
# For platforms without a tsearch-extras package distributed
# from our PPA or a pgroonga release, we need to build both
# from source.
BUILD_PGROONGA_FROM_SOURCE = True
BUILD_TSEARCH_FROM_SOURCE = True
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
# Dependency for building tsearch_extras from source
"postgresql-server-dev-{0}",
# Dependency for building pgroonga from source
"libgroonga-dev",
"libmsgpack-dev",
]
]
elif vendor == 'debian' and os_version == "10":
# For platforms without a tsearch-extras package distributed
# from our PPA or a pgroonga release, we need to build both
# from source.
BUILD_PGROONGA_FROM_SOURCE = True
BUILD_TSEARCH_FROM_SOURCE = True
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
# Dependency for building tsearch_extras from source
"postgresql-server-dev-{0}",
# Dependency for building pgroonga from source
"libgroonga-dev",
"libmsgpack-dev",
]
else:
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
"postgresql-{0}-pgroonga",
"postgresql-{0}-tsearch-extras",
]
]
elif vendor in ["ubuntu", "debian"]:
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
"postgresql-{0}-pgroonga",
"postgresql-{0}-tsearch-extras",
]
]
elif vendor in ["CentOS", "RedHat"]:
SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
@ -255,7 +254,7 @@ def install_system_deps():
if family == 'redhat':
install_yum_deps(deps_to_install)
elif vendor in ["Debian", "Ubuntu"]:
elif vendor in ["debian", "ubuntu"]:
install_apt_deps(deps_to_install)
else:
raise AssertionError("Invalid vendor")
@ -352,7 +351,7 @@ def main(options):
for apt_depedency in SYSTEM_DEPENDENCIES:
sha_sum.update(apt_depedency.encode('utf8'))
if vendor in ["Ubuntu", "Debian"]:
if vendor in ["ubuntu", "debian"]:
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
else:
# hash the content of setup-yum-repo and build-*

4
zerver/management/commands/backup.py
View File

@ -8,7 +8,7 @@ from django.conf import settings
from django.db import connection
from django.utils.timezone import now as timezone_now
from scripts.lib.zulip_tools import parse_lsb_release, run, TIMESTAMP_FORMAT
from scripts.lib.zulip_tools import parse_os_release, run, TIMESTAMP_FORMAT
from version import ZULIP_VERSION
from zerver.lib.management import ZulipBaseCommand
from zerver.logging_handlers import try_git_describe
@ -46,7 +46,7 @@ class Command(ZulipBaseCommand):
with open(os.path.join(tmp, "zulip-backup", "os-version"), "w") as f:
print(
"{DISTRIB_ID} {DISTRIB_CODENAME}".format(**parse_lsb_release()),
"{ID} {VERSION_ID}".format(**parse_os_release()),
file=f,
)
members.append("zulip-backup/os-version")