[manual] add nagios checks for email_deliver

manual step: puppet apply, make sure that these nagios checks are working properly

(imported from commit abc75b8a5b153510243c14035b820fbc864b7776)

puppet/zulip_internal/files/nagios3/commands.cfg

@@ -150,3 +150,13 @@ define command {
         command_name    check_email_mirror
         command_line    /usr/lib/nagios/plugins/check_by_ssh -l zulip -t 30 -i /var/lib/nagios/.ssh/id_rsa -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/check_email_mirror'
 }
+
+define command {
+        command_name    check_email_deliverer_process
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -l zulip -t 30 -i /var/lib/nagios/.ssh/id_rsa -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/check_email_deliverer_process'
+}
+
+define command {
+        command_name    check_email_deliverer_backlog
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -l zulip -t 30 -i /var/lib/nagios/.ssh/id_rsa -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/check_email_deliverer_backlog'
+}

puppet/zulip_internal/files/nagios3/conf.d/services.cfg

@@ -329,3 +329,19 @@ define service {
         check_command                   check_email_mirror!22
         contact_groups                  admins
 }
+
+define service {
+        use                             generic-service
+        host                            staging
+        service_description             Check email deliverer process (Currently only used on Zulip Enterprise)
+        check_command                   check_email_deliverer_process
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check email deliverer backlog (Currently only used on Zulip Enterprise)
+        check_command                   check_email_deliverer_backlog
+        hostgroup_name                  frontends
+        contact_groups                  admins
+}

puppet/zulip_internal/files/nagios_plugins/check_email_deliverer_backlog

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+#"OK": 0
+#"WARNING": 1
+#"CRITICAL": 2
+#"UNKNOWN": 3
+
+cd /home/zulip/deployments/current
+
+BACKLOG=$(./manage.py print_email_delivery_backlog)
+
+if [ $BACKLOG -gt 0 -a $BACKLOG -lt 10 ]
+then
+    echo "backlog of $BACKLOG"
+    exit 0;
+elif [ $BACKLOG -ge 10 ]
+    echo "backlog of $BACKLOG"
+    exit 2;
+else
+    echo "no backlog"
+    exit 0;
+fi

puppet/zulip_internal/files/nagios_plugins/check_email_deliverer_process

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#"OK": 0
+#"WARNING": 1
+#"CRITICAL": 2
+#"UNKNOWN": 3
+
+SUPERVISOR_STATUS=$(supervisorctl status zulip-workers:zulip-deliver-enqueued-emails 2>&1)
+STATUS=$(echo "$SUPERVISOR_STATUS" | awk '{ print $2 }')
+
+
+if [ "$STATUS" == "RUNNING" ];
+then
+    echo "Running";
+    exit 0;
+elif [ $(echo "$STATUS" | egrep '(STOPPED)|(STARTING)|(BACKOFF)|(STOPPING)|(EXITED)|(FATAL)|(UNKNOWN)$') ];
+    # not "RUNNING", but a recognized supervisor status
+    echo $STATUS;
+    exit 1;
+else
+    # we don't recognize the second column in this SUPERVISOR_STATUS.
+    # This may be indicative of a supervisor configuration problem
+    echo $SUPERVISOR_STATUS;
+    exit 3;
+fi
+