nagios: Switch check_user_zephyr_mirror_liveness to run via cron.

This check loads Django, and as such must be run as the zulip user.
Repeat the same pattern used elsewhere in nagios, of writing a state
file, which is read by `check_cron_file`.

puppet/zulip_ops/files/cron.d/check_user_zephyr_mirror_liveness

@@ -0,0 +1,5 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+USER=zulip
+
+* * * * * zulip /usr/lib/nagios/plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness >/dev/null

puppet/zulip_ops/files/nagios4/commands.cfg

@@ -72,7 +72,7 @@ define command{
 
 define command{
         command_name    check_user_zephyr_mirror_liveness
-        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness'
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file /var/lib/nagios_state/check_user_zephyr_mirror_liveness'
 }
 
 define command{

puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness

@@ -35,6 +35,9 @@ states: Dict[str, int] = {
     "UNKNOWN": 3,
 }
 
+state_file_path = "/var/lib/nagios_state/check_user_zephyr_mirror_liveness"
+now = timezone_now()
+
 
 def report(state: str, short_msg: str, too_old: AbstractSet[Any] = set()) -> None:
     too_old_data = ""
@@ -46,13 +49,14 @@ def report(state: str, short_msg: str, too_old: AbstractSet[Any] = set()) -> Non
             )
             for user in too_old
         )
-    print(f"{state}: {short_msg}{too_old_data}")
 
+    with open(state_file_path + ".tmp", "w") as f:
+        f.write(f"{now}|{states[state]}|{state}|{short_msg}{too_old_data}")
+    os.rename(state_file_path + ".tmp", state_file_path)
+    print(f"{state}: {short_msg}{too_old_data}")
     exit(states[state])
 
 
-now = timezone_now()
-
 all_users = UserActivity.objects.filter(
     query__in=["get_events", "/api/v1/events"], client__name="zephyr_mirror"
 )

puppet/zulip_ops/manifests/prod_app_frontend_once.pp

@@ -32,4 +32,12 @@ class zulip_ops::prod_app_frontend_once {
     mode   => '0644',
     source => 'puppet:///modules/zulip_ops/cron.d/check_send_receive_time',
   }
+
+  file { '/etc/cron.d/check_user_zephyr_mirror_liveness':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/zulip_ops/cron.d/check_user_zephyr_mirror_liveness',
+  }
 }