From 4fbe201187fcf655795179c18f127d48f1b6901e Mon Sep 17 00:00:00 2001
From: Tim Abbott <tabbott@mit.edu>
Date: Wed, 3 Aug 2016 12:53:03 -0700
Subject: [PATCH] puppet: Automate autossh process monitoring maintenance.

Previously, the Zulip Nagios configuration effectively hardcoded the
count for how many system should have autossh connections.
---
 .../zulip_ops/files/nagios3/conf.d/localhost.cfg  |  9 ---------
 puppet/zulip_ops/manifests/base.pp                |  3 +++
 puppet/zulip_ops/manifests/munin.pp               | 15 +--------------
 puppet/zulip_ops/manifests/nagios.pp              |  9 +++++++++
 .../templates/nagios_autossh.template.erb         |  7 +++++++
 5 files changed, 20 insertions(+), 23 deletions(-)
 create mode 100644 puppet/zulip_ops/templates/nagios_autossh.template.erb

diff --git a/puppet/zulip_ops/files/nagios3/conf.d/localhost.cfg b/puppet/zulip_ops/files/nagios3/conf.d/localhost.cfg
index 224e0a1776..83e6c51038 100644
--- a/puppet/zulip_ops/files/nagios3/conf.d/localhost.cfg
+++ b/puppet/zulip_ops/files/nagios3/conf.d/localhost.cfg
@@ -35,15 +35,6 @@ define service{
         check_command                   check_load!7.0!6.0!5.0!10.0!8.0!6.0
         }
 
-define service{
-        use                             generic-service
-        host_name                       nagios
-        service_description             Number of autossh processes
-        # If you change the number of munin autossh processes, change
-        # the second and third arguments, below.
-        check_command                   check_named_procs!autossh!10:10!10:17
-        }
-
 define service{
         use                             generic-service
         host_name                       nagios
diff --git a/puppet/zulip_ops/manifests/base.pp b/puppet/zulip_ops/manifests/base.pp
index fc6d9f8a35..8f13ee2296 100644
--- a/puppet/zulip_ops/manifests/base.pp
+++ b/puppet/zulip_ops/manifests/base.pp
@@ -34,6 +34,9 @@ class zulip_ops::base {
   # Add system users here
   $users = []
 
+  # Add hosts to monitor here
+  $hosts = []
+
   file { '/etc/apt/apt.conf.d/02periodic':
     ensure     => file,
     mode       => 644,
diff --git a/puppet/zulip_ops/manifests/munin.pp b/puppet/zulip_ops/manifests/munin.pp
index 1a48b95af7..ee68960192 100644
--- a/puppet/zulip_ops/manifests/munin.pp
+++ b/puppet/zulip_ops/manifests/munin.pp
@@ -7,20 +7,7 @@ class zulip_ops::munin {
                      ]
   package { $munin_packages: ensure => "installed" }
 
-  # If you add a new Munin node, change the number of autossh processes that we
-  # check for with Nagios.
-
-  $hosts = ["trac",
-            "zmirror",
-            "staging",
-            "git",
-            "bots",
-            "prod0",
-            "stats",
-            "postgres1",
-            "postgres3",
-            "redis0",
-            ]
+  $hosts = $zulip_ops::base::hosts
 
   file { "/etc/munin":
     require => Package["munin"],
diff --git a/puppet/zulip_ops/manifests/nagios.pp b/puppet/zulip_ops/manifests/nagios.pp
index c8eb5ea059..670d30b2e5 100644
--- a/puppet/zulip_ops/manifests/nagios.pp
+++ b/puppet/zulip_ops/manifests/nagios.pp
@@ -55,6 +55,15 @@ class zulip_ops::nagios {
     group      => "root",
     source => '/root/zulip/api/integrations/nagios/zulip_nagios.cfg',
   }
+
+  $hosts = $zulip_ops::base::hosts
+  file { '/etc/nagios3/conf.d/zulip_autossh.cfg':
+    ensure     => file,
+    mode       => 644,
+    owner      => "root",
+    group      => "root",
+    content => template('zulip_ops/nagios_autossh.template.erb'),
+  }
   file { '/etc/nagios3/zuliprc':
     ensure     => file,
     mode       => 644,
diff --git a/puppet/zulip_ops/templates/nagios_autossh.template.erb b/puppet/zulip_ops/templates/nagios_autossh.template.erb
new file mode 100644
index 0000000000..73c7e3f881
--- /dev/null
+++ b/puppet/zulip_ops/templates/nagios_autossh.template.erb
@@ -0,0 +1,7 @@
+
+define service{
+        use                             generic-service
+        host_name                       nagios
+        service_description             Number of autossh processes
+        check_command                   check_named_procs!autossh!<%= @hosts.length %>:<%= @hosts.length %>!<%= @hosts.length %>:<%= @hosts.length + 2 %>
+        }