From 87a109e3e05e61c333d6c45166b6f6b25cb13093 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Tue, 25 May 2021 01:05:40 +0000 Subject: [PATCH] puppet: Pull in pinned puppet modules. Using puppet modules from the puppet forge judiciously will allow us to simplify the configuration somewhat; this specifically pulls in the stdlib module, which we were already using parts of. --- docs/THIRDPARTY | 5 -- docs/subsystems/dependencies.md | 8 ++ puppet/deps.yaml | 1 + .../zulip/lib/puppet/parser/functions/join.rb | 43 --------- .../zulip/lib/puppet/parser/functions/keys.rb | 27 ------ .../lib/puppet/parser/functions/range.rb | 90 ------------------- scripts/lib/puppet_cache.py | 77 ++++++++++++++++ scripts/zulip-puppet-apply | 11 ++- tools/linter_lib/exclude.py | 2 - 9 files changed, 96 insertions(+), 168 deletions(-) create mode 100644 puppet/deps.yaml delete mode 100644 puppet/zulip/lib/puppet/parser/functions/join.rb delete mode 100644 puppet/zulip/lib/puppet/parser/functions/keys.rb delete mode 100644 puppet/zulip/lib/puppet/parser/functions/range.rb create mode 100644 scripts/lib/puppet_cache.py diff --git a/docs/THIRDPARTY b/docs/THIRDPARTY index 4714371a78..78724bf0be 100644 --- a/docs/THIRDPARTY +++ b/docs/THIRDPARTY @@ -28,11 +28,6 @@ Files: docs/code-of-conduct.md Copyright: 2017, Kandra Labs Inc. License: CC-BY-SA-4.0 -Files: puppet/zulip/lib/puppet/parser/functions/join.rb puppet/zulip/lib/puppet/parser/functions/range.rb -Copyright: 2011, Krzysztof Wilczynski - 2011, Puppet Labs Inc -License: Apache-2.0 - Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages Copyright: 2005 Francesc Guasch License: GPL-2.0 diff --git a/docs/subsystems/dependencies.md b/docs/subsystems/dependencies.md index 39519b0423..b58c2eeda6 100644 --- a/docs/subsystems/dependencies.md +++ b/docs/subsystems/dependencies.md @@ -256,6 +256,14 @@ wrapper to access the desired version conveniently and efficiently `/srv/zulip-yarn`. We don't do anything special to try to manage multiple versions of `yarn`. +## Puppet packages + +Third-party puppet modules are downloaded from the Puppet Forge into +subdirectories under `/srv/zulip-puppet-cache`, hashed based on their +versions; the latest is always symlinked as +`/srv/zulip-puppet-cache/current`. `zulip-puppet-apply` installs +these dependencies immediately before they are needed. + ## Other third-party and generated files In this section, we discuss the other third-party dependencies, diff --git a/puppet/deps.yaml b/puppet/deps.yaml new file mode 100644 index 0000000000..0723418c4d --- /dev/null +++ b/puppet/deps.yaml @@ -0,0 +1 @@ +puppetlabs-stdlib: 7.1.0 diff --git a/puppet/zulip/lib/puppet/parser/functions/join.rb b/puppet/zulip/lib/puppet/parser/functions/join.rb deleted file mode 100644 index 9671f3578f..0000000000 --- a/puppet/zulip/lib/puppet/parser/functions/join.rb +++ /dev/null @@ -1,43 +0,0 @@ -# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/join.rb -# -# join.rb -# -module Puppet::Parser::Functions - newfunction(:join, :type => :rvalue, :doc => <<-DOC - This function joins an array into a string using a separator. - - *Examples:* - - join(['a','b','c'], ",") - - Would result in: "a,b,c" - - Note: from Puppet 5.4.0, the compatible function with the same name in Puppet core - will be used instead of this function. - DOC - ) do |arguments| - - # Technically we support two arguments but only first is mandatory ... - raise(Puppet::ParseError, "join(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty? - - array = arguments[0] - - unless array.is_a?(Array) - raise(Puppet::ParseError, 'join(): Requires array to work with') - end - - suffix = arguments[1] if arguments[1] - - if suffix - unless suffix.is_a?(String) - raise(Puppet::ParseError, 'join(): Requires string to work with') - end - end - - result = suffix ? array.join(suffix) : array.join - - return result - end -end - -# vim: set ts=2 sw=2 et : diff --git a/puppet/zulip/lib/puppet/parser/functions/keys.rb b/puppet/zulip/lib/puppet/parser/functions/keys.rb deleted file mode 100644 index e35bb42b6b..0000000000 --- a/puppet/zulip/lib/puppet/parser/functions/keys.rb +++ /dev/null @@ -1,27 +0,0 @@ -# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/keys.rb -# -# keys.rb -# -module Puppet::Parser::Functions - newfunction(:keys, :type => :rvalue, :doc => <<-DOC - Returns the keys of a hash as an array. - Note: from Puppet 5.5.0, the compatible function with the same name in Puppet core - will be used instead of this function. - DOC - ) do |arguments| - - raise(Puppet::ParseError, "keys(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty? - - hash = arguments[0] - - unless hash.is_a?(Hash) - raise(Puppet::ParseError, 'keys(): Requires hash to work with') - end - - result = hash.keys - - return result - end -end - -# vim: set ts=2 sw=2 et : diff --git a/puppet/zulip/lib/puppet/parser/functions/range.rb b/puppet/zulip/lib/puppet/parser/functions/range.rb deleted file mode 100644 index 4463166b3f..0000000000 --- a/puppet/zulip/lib/puppet/parser/functions/range.rb +++ /dev/null @@ -1,90 +0,0 @@ -# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/range.rb -# -# range.rb -# -# TODO(Krzysztof Wilczynski): We probably need to approach numeric values differently ... -module Puppet::Parser::Functions - newfunction(:range, :type => :rvalue, :doc => <<-DOC - When given range in the form of (start, stop) it will extrapolate a range as - an array. - - *Examples:* - - range("0", "9") - - Will return: [0,1,2,3,4,5,6,7,8,9] - - range("00", "09") - - Will return: [0,1,2,3,4,5,6,7,8,9] (Zero padded strings are converted to - integers automatically) - - range("a", "c") - - Will return: ["a","b","c"] - - range("host01", "host10") - Will return: ["host01", "host02", ..., "host09", "host10"] - NB Be explicit in including trailing zeros. Otherwise the underlying ruby function will fail. - - Passing a third argument will cause the generated range to step by that - interval, e.g. - - range("0", "9", "2") - - Will return: [0,2,4,6,8] - - The Puppet Language support Integer and Float ranges by using the type system. Those are suitable for - iterating a given number of times. Also see the step() function in Puppet for skipping values. - - Integer[0, 9].each |$x| { notice($x) } # notices 0, 1, 2, ... 9 - DOC - ) do |arguments| - - raise(Puppet::ParseError, 'range(): Wrong number of arguments given (0 for 1)') if arguments.empty? - - if arguments.size > 1 - start = arguments[0] - stop = arguments[1] - step = arguments[2].nil? ? 1 : arguments[2].to_i.abs - - type = '..' # Use the simplest type of Range available in Ruby - - else # arguments.size == 1 - value = arguments[0] - - m = value.match(%r{^(\w+)(\.\.\.?|\-)(\w+)$}) - if m - start = m[1] - stop = m[3] - - type = m[2] - step = 1 - elsif value =~ %r{^.+$} - raise(Puppet::ParseError, "range(): Unable to compute range from the value: #{value}") - else - raise(Puppet::ParseError, "range(): Unknown range format: #{value}") - end - end - - # If we were given an integer, ensure we work with one - if start.to_s =~ %r{^\d+$} - start = start.to_i - stop = stop.to_i - else - start = start.to_s - stop = stop.to_s - end - - range = case type - when %r{^(..|-)$} then (start..stop) - when '...' then (start...stop) # Exclusive of last element - end - - result = range.step(step).to_a - - return result - end -end - -# vim: set ts=2 sw=2 et : diff --git a/scripts/lib/puppet_cache.py b/scripts/lib/puppet_cache.py new file mode 100644 index 0000000000..7440f8f7a0 --- /dev/null +++ b/scripts/lib/puppet_cache.py @@ -0,0 +1,77 @@ +import hashlib +import json +import os +import shutil +import subprocess + +import yaml + +from .zulip_tools import parse_os_release, run + +ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +ZULIP_SRV_PATH = "/srv" + +PUPPET_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-puppet-cache") +PUPPET_DEPS_FILE_PATH = os.path.join(ZULIP_PATH, "puppet/deps.yaml") +PUPPET_THIRDPARTY = os.path.join(PUPPET_MODULES_CACHE_PATH, "current") + + +def generate_sha1sum_puppet_modules() -> str: + data = {} + with open(PUPPET_DEPS_FILE_PATH, "r") as fb: + data["deps.yaml"] = fb.read().strip() + data["puppet-version"] = subprocess.check_output( + # This is 10x faster than `puppet --version` + ["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"], + universal_newlines=True, + ).strip() + + sha1sum = hashlib.sha1() + sha1sum.update(json.dumps(data, sort_keys=True).encode("utf-8")) + return sha1sum.hexdigest() + + +def setup_puppet_modules() -> None: + sha1sum = generate_sha1sum_puppet_modules() + target_path = os.path.join(PUPPET_MODULES_CACHE_PATH, sha1sum) + success_stamp = os.path.join(target_path, ".success-stamp") + # Check if a cached version already exists + if not os.path.exists(success_stamp): + do_puppet_module_install(target_path, success_stamp) + + if os.path.islink(PUPPET_THIRDPARTY): + os.remove(PUPPET_THIRDPARTY) + elif os.path.isdir(PUPPET_THIRDPARTY): + shutil.rmtree(PUPPET_THIRDPARTY) + os.symlink(target_path, PUPPET_THIRDPARTY) + + +def do_puppet_module_install( + target_path: str, + success_stamp: str, +) -> None: + # This is to suppress Puppet warnings with ruby 2.7. + distro_info = parse_os_release() + puppet_env = os.environ.copy() + if (distro_info["ID"], distro_info["VERSION_ID"]) in [("ubuntu", "20.04")]: + puppet_env["RUBYOPT"] = "-W0" + + os.makedirs(target_path, exist_ok=True) + with open(PUPPET_DEPS_FILE_PATH, "r") as yaml_file: + deps = yaml.safe_load(yaml_file) + for module, version in deps.items(): + run( + [ + "puppet", + "module", + "--modulepath", + target_path, + "install", + module, + "--version", + version, + ], + env=puppet_env, + ) + with open(success_stamp, "w"): + pass diff --git a/scripts/zulip-puppet-apply b/scripts/zulip-puppet-apply index 7fe4f0c09f..ad1d0372c1 100755 --- a/scripts/zulip-puppet-apply +++ b/scripts/zulip-puppet-apply @@ -9,6 +9,7 @@ import tempfile import yaml +from lib.puppet_cache import setup_puppet_modules from lib.zulip_tools import assert_running_as_root, parse_os_release assert_running_as_root() @@ -25,6 +26,8 @@ args, extra_args = parser.parse_known_args() config = configparser.RawConfigParser() config.read(args.config) +setup_puppet_modules() + distro_info = parse_os_release() puppet_config = """ Exec { path => "/usr/sbin:/usr/bin:/sbin:/bin" } @@ -36,7 +39,13 @@ for pclass in re.split(r"\s*,\s*", config.get("machine", "puppet_classes")): # We use the Puppet configuration from the same Zulip checkout as this script scripts_path = os.path.join(BASE_DIR, "scripts") puppet_module_path = os.path.join(BASE_DIR, "puppet") -puppet_cmd = ["puppet", "apply", f"--modulepath={puppet_module_path}", "-e", puppet_config] +puppet_cmd = [ + "puppet", + "apply", + f"--modulepath={puppet_module_path}:/srv/zulip-puppet-cache/current", + "-e", + puppet_config, +] if args.noop: puppet_cmd += ["--noop"] puppet_cmd += extra_args diff --git a/tools/linter_lib/exclude.py b/tools/linter_lib/exclude.py index 709804ffeb..86dbc95bd2 100644 --- a/tools/linter_lib/exclude.py +++ b/tools/linter_lib/exclude.py @@ -1,8 +1,6 @@ # Exclude some directories and files from lint checking EXCLUDED_FILES = [ # Third-party code that doesn't match our style - "puppet/zulip/lib/puppet/parser/functions/join.rb", - "puppet/zulip/lib/puppet/parser/functions/range.rb", "puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh", "scripts/lib/third", "static/third",