mirror of https://github.com/zulip/zulip.git
puppet: Pull in pinned puppet modules.
Using puppet modules from the puppet forge judiciously will allow us to simplify the configuration somewhat; this specifically pulls in the stdlib module, which we were already using parts of.
This commit is contained in:
Alex Vandiver
Alex Vandiver
parent
6107679f61
commit
87a109e3e0
|
|
@ -28,11 +28,6 @@ Files: docs/code-of-conduct.md
|
|||
Copyright: 2017, Kandra Labs Inc.
|
||||
License: CC-BY-SA-4.0
|
||||
|
||||
Files: puppet/zulip/lib/puppet/parser/functions/join.rb puppet/zulip/lib/puppet/parser/functions/range.rb
|
||||
Copyright: 2011, Krzysztof Wilczynski
|
||||
2011, Puppet Labs Inc
|
||||
License: Apache-2.0
|
||||
|
||||
Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages
|
||||
Copyright: 2005 Francesc Guasch
|
||||
License: GPL-2.0
|
||||
|
|
|
|||
|
|
@ -256,6 +256,14 @@ wrapper to access the desired version conveniently and efficiently
|
|||
`/srv/zulip-yarn`. We don't do anything special to try to manage
|
||||
multiple versions of `yarn`.
|
||||
|
||||
## Puppet packages
|
||||
|
||||
Third-party puppet modules are downloaded from the Puppet Forge into
|
||||
subdirectories under `/srv/zulip-puppet-cache`, hashed based on their
|
||||
versions; the latest is always symlinked as
|
||||
`/srv/zulip-puppet-cache/current`. `zulip-puppet-apply` installs
|
||||
these dependencies immediately before they are needed.
|
||||
|
||||
## Other third-party and generated files
|
||||
|
||||
In this section, we discuss the other third-party dependencies,
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
puppetlabs-stdlib: 7.1.0
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/join.rb
|
||||
#
|
||||
# join.rb
|
||||
#
|
||||
module Puppet::Parser::Functions
|
||||
newfunction(:join, :type => :rvalue, :doc => <<-DOC
|
||||
This function joins an array into a string using a separator.
|
||||
|
||||
*Examples:*
|
||||
|
||||
join(['a','b','c'], ",")
|
||||
|
||||
Would result in: "a,b,c"
|
||||
|
||||
Note: from Puppet 5.4.0, the compatible function with the same name in Puppet core
|
||||
will be used instead of this function.
|
||||
DOC
|
||||
) do |arguments|
|
||||
|
||||
# Technically we support two arguments but only first is mandatory ...
|
||||
raise(Puppet::ParseError, "join(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty?
|
||||
|
||||
array = arguments[0]
|
||||
|
||||
unless array.is_a?(Array)
|
||||
raise(Puppet::ParseError, 'join(): Requires array to work with')
|
||||
end
|
||||
|
||||
suffix = arguments[1] if arguments[1]
|
||||
|
||||
if suffix
|
||||
unless suffix.is_a?(String)
|
||||
raise(Puppet::ParseError, 'join(): Requires string to work with')
|
||||
end
|
||||
end
|
||||
|
||||
result = suffix ? array.join(suffix) : array.join
|
||||
|
||||
return result
|
||||
end
|
||||
end
|
||||
|
||||
# vim: set ts=2 sw=2 et :
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/keys.rb
|
||||
#
|
||||
# keys.rb
|
||||
#
|
||||
module Puppet::Parser::Functions
|
||||
newfunction(:keys, :type => :rvalue, :doc => <<-DOC
|
||||
Returns the keys of a hash as an array.
|
||||
Note: from Puppet 5.5.0, the compatible function with the same name in Puppet core
|
||||
will be used instead of this function.
|
||||
DOC
|
||||
) do |arguments|
|
||||
|
||||
raise(Puppet::ParseError, "keys(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty?
|
||||
|
||||
hash = arguments[0]
|
||||
|
||||
unless hash.is_a?(Hash)
|
||||
raise(Puppet::ParseError, 'keys(): Requires hash to work with')
|
||||
end
|
||||
|
||||
result = hash.keys
|
||||
|
||||
return result
|
||||
end
|
||||
end
|
||||
|
||||
# vim: set ts=2 sw=2 et :
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/range.rb
|
||||
#
|
||||
# range.rb
|
||||
#
|
||||
# TODO(Krzysztof Wilczynski): We probably need to approach numeric values differently ...
|
||||
module Puppet::Parser::Functions
|
||||
newfunction(:range, :type => :rvalue, :doc => <<-DOC
|
||||
When given range in the form of (start, stop) it will extrapolate a range as
|
||||
an array.
|
||||
|
||||
*Examples:*
|
||||
|
||||
range("0", "9")
|
||||
|
||||
Will return: [0,1,2,3,4,5,6,7,8,9]
|
||||
|
||||
range("00", "09")
|
||||
|
||||
Will return: [0,1,2,3,4,5,6,7,8,9] (Zero padded strings are converted to
|
||||
integers automatically)
|
||||
|
||||
range("a", "c")
|
||||
|
||||
Will return: ["a","b","c"]
|
||||
|
||||
range("host01", "host10")
|
||||
Will return: ["host01", "host02", ..., "host09", "host10"]
|
||||
NB Be explicit in including trailing zeros. Otherwise the underlying ruby function will fail.
|
||||
|
||||
Passing a third argument will cause the generated range to step by that
|
||||
interval, e.g.
|
||||
|
||||
range("0", "9", "2")
|
||||
|
||||
Will return: [0,2,4,6,8]
|
||||
|
||||
The Puppet Language support Integer and Float ranges by using the type system. Those are suitable for
|
||||
iterating a given number of times. Also see the step() function in Puppet for skipping values.
|
||||
|
||||
Integer[0, 9].each |$x| { notice($x) } # notices 0, 1, 2, ... 9
|
||||
DOC
|
||||
) do |arguments|
|
||||
|
||||
raise(Puppet::ParseError, 'range(): Wrong number of arguments given (0 for 1)') if arguments.empty?
|
||||
|
||||
if arguments.size > 1
|
||||
start = arguments[0]
|
||||
stop = arguments[1]
|
||||
step = arguments[2].nil? ? 1 : arguments[2].to_i.abs
|
||||
|
||||
type = '..' # Use the simplest type of Range available in Ruby
|
||||
|
||||
else # arguments.size == 1
|
||||
value = arguments[0]
|
||||
|
||||
m = value.match(%r{^(\w+)(\.\.\.?|\-)(\w+)$})
|
||||
if m
|
||||
start = m[1]
|
||||
stop = m[3]
|
||||
|
||||
type = m[2]
|
||||
step = 1
|
||||
elsif value =~ %r{^.+$}
|
||||
raise(Puppet::ParseError, "range(): Unable to compute range from the value: #{value}")
|
||||
else
|
||||
raise(Puppet::ParseError, "range(): Unknown range format: #{value}")
|
||||
end
|
||||
end
|
||||
|
||||
# If we were given an integer, ensure we work with one
|
||||
if start.to_s =~ %r{^\d+$}
|
||||
start = start.to_i
|
||||
stop = stop.to_i
|
||||
else
|
||||
start = start.to_s
|
||||
stop = stop.to_s
|
||||
end
|
||||
|
||||
range = case type
|
||||
when %r{^(..|-)$} then (start..stop)
|
||||
when '...' then (start...stop) # Exclusive of last element
|
||||
end
|
||||
|
||||
result = range.step(step).to_a
|
||||
|
||||
return result
|
||||
end
|
||||
end
|
||||
|
||||
# vim: set ts=2 sw=2 et :
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
|
||||
from .zulip_tools import parse_os_release, run
|
||||
|
||||
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
ZULIP_SRV_PATH = "/srv"
|
||||
|
||||
PUPPET_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-puppet-cache")
|
||||
PUPPET_DEPS_FILE_PATH = os.path.join(ZULIP_PATH, "puppet/deps.yaml")
|
||||
PUPPET_THIRDPARTY = os.path.join(PUPPET_MODULES_CACHE_PATH, "current")
|
||||
|
||||
|
||||
def generate_sha1sum_puppet_modules() -> str:
|
||||
data = {}
|
||||
with open(PUPPET_DEPS_FILE_PATH, "r") as fb:
|
||||
data["deps.yaml"] = fb.read().strip()
|
||||
data["puppet-version"] = subprocess.check_output(
|
||||
# This is 10x faster than `puppet --version`
|
||||
["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"],
|
||||
universal_newlines=True,
|
||||
).strip()
|
||||
|
||||
sha1sum = hashlib.sha1()
|
||||
sha1sum.update(json.dumps(data, sort_keys=True).encode("utf-8"))
|
||||
return sha1sum.hexdigest()
|
||||
|
||||
|
||||
def setup_puppet_modules() -> None:
|
||||
sha1sum = generate_sha1sum_puppet_modules()
|
||||
target_path = os.path.join(PUPPET_MODULES_CACHE_PATH, sha1sum)
|
||||
success_stamp = os.path.join(target_path, ".success-stamp")
|
||||
# Check if a cached version already exists
|
||||
if not os.path.exists(success_stamp):
|
||||
do_puppet_module_install(target_path, success_stamp)
|
||||
|
||||
if os.path.islink(PUPPET_THIRDPARTY):
|
||||
os.remove(PUPPET_THIRDPARTY)
|
||||
elif os.path.isdir(PUPPET_THIRDPARTY):
|
||||
shutil.rmtree(PUPPET_THIRDPARTY)
|
||||
os.symlink(target_path, PUPPET_THIRDPARTY)
|
||||
|
||||
|
||||
def do_puppet_module_install(
|
||||
target_path: str,
|
||||
success_stamp: str,
|
||||
) -> None:
|
||||
# This is to suppress Puppet warnings with ruby 2.7.
|
||||
distro_info = parse_os_release()
|
||||
puppet_env = os.environ.copy()
|
||||
if (distro_info["ID"], distro_info["VERSION_ID"]) in [("ubuntu", "20.04")]:
|
||||
puppet_env["RUBYOPT"] = "-W0"
|
||||
|
||||
os.makedirs(target_path, exist_ok=True)
|
||||
with open(PUPPET_DEPS_FILE_PATH, "r") as yaml_file:
|
||||
deps = yaml.safe_load(yaml_file)
|
||||
for module, version in deps.items():
|
||||
run(
|
||||
[
|
||||
"puppet",
|
||||
"module",
|
||||
"--modulepath",
|
||||
target_path,
|
||||
"install",
|
||||
module,
|
||||
"--version",
|
||||
version,
|
||||
],
|
||||
env=puppet_env,
|
||||
)
|
||||
with open(success_stamp, "w"):
|
||||
pass
|
||||
|
|
@ -9,6 +9,7 @@ import tempfile
|
|||
|
||||
import yaml
|
||||
|
||||
from lib.puppet_cache import setup_puppet_modules
|
||||
from lib.zulip_tools import assert_running_as_root, parse_os_release
|
||||
|
||||
assert_running_as_root()
|
||||
|
|
@ -25,6 +26,8 @@ args, extra_args = parser.parse_known_args()
|
|||
config = configparser.RawConfigParser()
|
||||
config.read(args.config)
|
||||
|
||||
setup_puppet_modules()
|
||||
|
||||
distro_info = parse_os_release()
|
||||
puppet_config = """
|
||||
Exec { path => "/usr/sbin:/usr/bin:/sbin:/bin" }
|
||||
|
|
@ -36,7 +39,13 @@ for pclass in re.split(r"\s*,\s*", config.get("machine", "puppet_classes")):
|
|||
# We use the Puppet configuration from the same Zulip checkout as this script
|
||||
scripts_path = os.path.join(BASE_DIR, "scripts")
|
||||
puppet_module_path = os.path.join(BASE_DIR, "puppet")
|
||||
puppet_cmd = ["puppet", "apply", f"--modulepath={puppet_module_path}", "-e", puppet_config]
|
||||
puppet_cmd = [
|
||||
"puppet",
|
||||
"apply",
|
||||
f"--modulepath={puppet_module_path}:/srv/zulip-puppet-cache/current",
|
||||
"-e",
|
||||
puppet_config,
|
||||
]
|
||||
if args.noop:
|
||||
puppet_cmd += ["--noop"]
|
||||
puppet_cmd += extra_args
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
# Exclude some directories and files from lint checking
|
||||
EXCLUDED_FILES = [
|
||||
# Third-party code that doesn't match our style
|
||||
"puppet/zulip/lib/puppet/parser/functions/join.rb",
|
||||
"puppet/zulip/lib/puppet/parser/functions/range.rb",
|
||||
"puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh",
|
||||
"scripts/lib/third",
|
||||
"static/third",
|
||||
|
|
|
|||
Loading…
Reference in New Issue