From 6e981c18d5b42b040bf755bef3fe6e22c3200a07 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Fri, 19 Apr 2024 11:52:04 -0400 Subject: [PATCH] puppet: Factor out cron job creation. --- .../files/cron.d/check_send_receive_time | 5 - .../cron.d/check_user_zephyr_mirror_liveness | 5 - .../files/cron.d/fetch-contributor-data | 5 - puppet/kandra/manifests/app_frontend.pp | 10 +- .../manifests/prod_app_frontend_once.pp | 52 ++++------ puppet/zulip/files/cron.d/archive-messages | 10 -- .../cron.d/calculate-first-visible-message-id | 5 - .../zulip/files/cron.d/check-analytics-state | 5 - puppet/zulip/files/cron.d/clearsessions | 4 - .../downgrade-small-realms-behind-on-payments | 5 - .../zulip/files/cron.d/fetch-tor-exit-nodes | 5 - puppet/zulip/files/cron.d/invoice-plans | 5 - puppet/zulip/files/cron.d/pg-backup-and-purge | 3 - .../files/cron.d/promote-new-full-members | 6 -- puppet/zulip/files/cron.d/restart-zulip | 7 -- puppet/zulip/files/cron.d/send-digest-emails | 4 - .../cron.d/send_zulip_update_announcements | 6 -- .../zulip/files/cron.d/soft-deactivate-users | 6 -- .../files/cron.d/update-analytics-counts | 6 -- puppet/zulip/manifests/app_frontend_base.pp | 8 +- puppet/zulip/manifests/app_frontend_once.pp | 94 +++++++------------ puppet/zulip/manifests/cron.pp | 25 +++++ puppet/zulip/manifests/postgresql_backups.pp | 11 +-- .../zulip/manifests/profile/app_frontend.pp | 11 +-- puppet/zulip/templates/cron.template.erb | 7 ++ 25 files changed, 102 insertions(+), 208 deletions(-) delete mode 100644 puppet/kandra/files/cron.d/check_send_receive_time delete mode 100644 puppet/kandra/files/cron.d/check_user_zephyr_mirror_liveness delete mode 100644 puppet/kandra/files/cron.d/fetch-contributor-data delete mode 100644 puppet/zulip/files/cron.d/archive-messages delete mode 100644 puppet/zulip/files/cron.d/calculate-first-visible-message-id delete mode 100644 puppet/zulip/files/cron.d/check-analytics-state delete mode 100644 puppet/zulip/files/cron.d/clearsessions delete mode 100644 puppet/zulip/files/cron.d/downgrade-small-realms-behind-on-payments delete mode 100644 puppet/zulip/files/cron.d/fetch-tor-exit-nodes delete mode 100644 puppet/zulip/files/cron.d/invoice-plans delete mode 100644 puppet/zulip/files/cron.d/pg-backup-and-purge delete mode 100644 puppet/zulip/files/cron.d/promote-new-full-members delete mode 100644 puppet/zulip/files/cron.d/restart-zulip delete mode 100644 puppet/zulip/files/cron.d/send-digest-emails delete mode 100644 puppet/zulip/files/cron.d/send_zulip_update_announcements delete mode 100644 puppet/zulip/files/cron.d/soft-deactivate-users delete mode 100644 puppet/zulip/files/cron.d/update-analytics-counts create mode 100644 puppet/zulip/manifests/cron.pp create mode 100644 puppet/zulip/templates/cron.template.erb diff --git a/puppet/kandra/files/cron.d/check_send_receive_time b/puppet/kandra/files/cron.d/check_send_receive_time deleted file mode 100644 index b84d837a8b..0000000000 --- a/puppet/kandra/files/cron.d/check_send_receive_time +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -* * * * * zulip /usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time --site=https://$(/home/zulip/deployments/current/scripts/get-django-setting NAGIOS_BOT_HOST) >/dev/null diff --git a/puppet/kandra/files/cron.d/check_user_zephyr_mirror_liveness b/puppet/kandra/files/cron.d/check_user_zephyr_mirror_liveness deleted file mode 100644 index 1aa512a822..0000000000 --- a/puppet/kandra/files/cron.d/check_user_zephyr_mirror_liveness +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -* * * * * zulip /usr/lib/nagios/plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness >/dev/null diff --git a/puppet/kandra/files/cron.d/fetch-contributor-data b/puppet/kandra/files/cron.d/fetch-contributor-data deleted file mode 100644 index 4fc73952d8..0000000000 --- a/puppet/kandra/files/cron.d/fetch-contributor-data +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -0 8 * * * zulip /home/zulip/deployments/current/tools/fetch-contributor-data diff --git a/puppet/kandra/manifests/app_frontend.pp b/puppet/kandra/manifests/app_frontend.pp index ad8e2ff4b8..cf092b8715 100644 --- a/puppet/kandra/manifests/app_frontend.pp +++ b/puppet/kandra/manifests/app_frontend.pp @@ -62,11 +62,9 @@ class kandra::app_frontend { # Each server does its own fetching of contributor data, since # we don't have a way to synchronize that among several servers. - file { '/etc/cron.d/fetch-contributor-data': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/kandra/cron.d/fetch-contributor-data', + zulip::cron { 'fetch-contributor-data': + hour => '8', + minute => '0', + command => '/home/zulip/deployments/current/tools/fetch-contributor-data', } } diff --git a/puppet/kandra/manifests/prod_app_frontend_once.pp b/puppet/kandra/manifests/prod_app_frontend_once.pp index 9001b62944..25b4e21ddd 100644 --- a/puppet/kandra/manifests/prod_app_frontend_once.pp +++ b/puppet/kandra/manifests/prod_app_frontend_once.pp @@ -3,43 +3,29 @@ class kandra::prod_app_frontend_once { include zulip::hooks::push_git_ref include zulip::hooks::zulip_notify - file { '/etc/cron.d/update-first-visible-message-id': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/calculate-first-visible-message-id', + zulip::cron { 'update-first-visible-message-id': + hour => '7', + minute => '*', + manage => 'calculate_first_visible_message_id --lookback-hours 30', } - file { '/etc/cron.d/invoice-plans': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/invoice-plans', + zulip::cron { 'invoice-plans': + hour => '22', + minute => '0', + } + zulip::cron { 'downgrade-small-realms-behind-on-payments': + hour => '17', + minute => '0', } - file { '/etc/cron.d/downgrade-small-realms-behind-on-payments': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/downgrade-small-realms-behind-on-payments', + zulip::cron { 'check_send_receive_time': + hour => '*', + minute => '*', + command => '/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time --site=https://$(/home/zulip/deployments/current/scripts/get-django-setting NAGIOS_BOT_HOST) >/dev/null', } - - file { '/etc/cron.d/check_send_receive_time': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/kandra/cron.d/check_send_receive_time', - } - - file { '/etc/cron.d/check_user_zephyr_mirror_liveness': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/kandra/cron.d/check_user_zephyr_mirror_liveness', + zulip::cron { 'check_user_zephyr_mirror_liveness': + hour => '*', + minute => '*', + command => '/usr/lib/nagios/plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness >/dev/null', } } diff --git a/puppet/zulip/files/cron.d/archive-messages b/puppet/zulip/files/cron.d/archive-messages deleted file mode 100644 index 50c77d42c7..0000000000 --- a/puppet/zulip/files/cron.d/archive-messages +++ /dev/null @@ -1,10 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -# Daily cron job for retention policy - archives existing messages based on the policy -# and cleans expired entries from the archive. -0 6 * * * zulip /home/zulip/deployments/current/manage.py archive_messages >/dev/null -# Daily cron job for cleaning up old, unclaimed attachments and their corresponding files. -# Ideally should not run at the same time as the previous job, to avoid races. -0 5 * * * zulip /home/zulip/deployments/current/manage.py delete_old_unclaimed_attachments -f >/dev/null diff --git a/puppet/zulip/files/cron.d/calculate-first-visible-message-id b/puppet/zulip/files/cron.d/calculate-first-visible-message-id deleted file mode 100644 index 9d179afd24..0000000000 --- a/puppet/zulip/files/cron.d/calculate-first-visible-message-id +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -0 7 * * * zulip /home/zulip/deployments/current/manage.py calculate_first_visible_message_id --lookback-hours 30 diff --git a/puppet/zulip/files/cron.d/check-analytics-state b/puppet/zulip/files/cron.d/check-analytics-state deleted file mode 100644 index abf002a996..0000000000 --- a/puppet/zulip/files/cron.d/check-analytics-state +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -30 * * * * zulip /home/zulip/deployments/current/manage.py check_analytics_state diff --git a/puppet/zulip/files/cron.d/clearsessions b/puppet/zulip/files/cron.d/clearsessions deleted file mode 100644 index 3aebe991bc..0000000000 --- a/puppet/zulip/files/cron.d/clearsessions +++ /dev/null @@ -1,4 +0,0 @@ -MAILTO=root - -# Clear all expired Django sessions at 10:22 PM every day. -22 22 * * * zulip cd /home/zulip/deployments/current && ./manage.py clearsessions diff --git a/puppet/zulip/files/cron.d/downgrade-small-realms-behind-on-payments b/puppet/zulip/files/cron.d/downgrade-small-realms-behind-on-payments deleted file mode 100644 index e0b3d126f3..0000000000 --- a/puppet/zulip/files/cron.d/downgrade-small-realms-behind-on-payments +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -0 17 * * * zulip /home/zulip/deployments/current/manage.py downgrade_small_realms_behind_on_payments diff --git a/puppet/zulip/files/cron.d/fetch-tor-exit-nodes b/puppet/zulip/files/cron.d/fetch-tor-exit-nodes deleted file mode 100644 index 50a38950bd..0000000000 --- a/puppet/zulip/files/cron.d/fetch-tor-exit-nodes +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -17 * * * * zulip /home/zulip/deployments/current/manage.py fetch_tor_exit_nodes diff --git a/puppet/zulip/files/cron.d/invoice-plans b/puppet/zulip/files/cron.d/invoice-plans deleted file mode 100644 index bbf53ec3bf..0000000000 --- a/puppet/zulip/files/cron.d/invoice-plans +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -0 22 * * * zulip /home/zulip/deployments/current/manage.py invoice_plans diff --git a/puppet/zulip/files/cron.d/pg-backup-and-purge b/puppet/zulip/files/cron.d/pg-backup-and-purge deleted file mode 100644 index dcf077962e..0000000000 --- a/puppet/zulip/files/cron.d/pg-backup-and-purge +++ /dev/null @@ -1,3 +0,0 @@ -PATH=/bin:/usr/bin:/usr/local/bin - -0 2 * * * postgres /usr/local/bin/pg_backup_and_purge >/var/log/pg_backup_and_purge.log 2>&1 diff --git a/puppet/zulip/files/cron.d/promote-new-full-members b/puppet/zulip/files/cron.d/promote-new-full-members deleted file mode 100644 index 9af9c7ac38..0000000000 --- a/puppet/zulip/files/cron.d/promote-new-full-members +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -# Cron job to promote users to full members system group. -35 * * * * zulip /home/zulip/deployments/current/manage.py promote_new_full_members diff --git a/puppet/zulip/files/cron.d/restart-zulip b/puppet/zulip/files/cron.d/restart-zulip deleted file mode 100644 index 08c01d5ff8..0000000000 --- a/puppet/zulip/files/cron.d/restart-zulip +++ /dev/null @@ -1,7 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -# Cron job to restart the Zulip server weekly, with the goal of -# minimizing the impact of any memory leaks that we might grow. -0 6 * * 7 zulip /home/zulip/deployments/current/scripts/restart-server --fill-cache diff --git a/puppet/zulip/files/cron.d/send-digest-emails b/puppet/zulip/files/cron.d/send-digest-emails deleted file mode 100644 index a3cfe373d1..0000000000 --- a/puppet/zulip/files/cron.d/send-digest-emails +++ /dev/null @@ -1,4 +0,0 @@ -MAILTO=root - -# Send digest emails once a day. Time is in UTC. -0 18 * * * zulip cd /home/zulip/deployments/current && ./manage.py enqueue_digest_emails diff --git a/puppet/zulip/files/cron.d/send_zulip_update_announcements b/puppet/zulip/files/cron.d/send_zulip_update_announcements deleted file mode 100644 index 902f0df2e0..0000000000 --- a/puppet/zulip/files/cron.d/send_zulip_update_announcements +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -# Cron job to send zulip update announcements hourly, if there are any. -47 * * * * zulip /home/zulip/deployments/current/manage.py send_zulip_update_announcements diff --git a/puppet/zulip/files/cron.d/soft-deactivate-users b/puppet/zulip/files/cron.d/soft-deactivate-users deleted file mode 100644 index 7352e99bda..0000000000 --- a/puppet/zulip/files/cron.d/soft-deactivate-users +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -# Cron job to soft deactivate long term idle users daily. -0 5 * * * zulip /home/zulip/deployments/current/manage.py soft_deactivate_users -d >/dev/null diff --git a/puppet/zulip/files/cron.d/update-analytics-counts b/puppet/zulip/files/cron.d/update-analytics-counts deleted file mode 100644 index 5175c6bd0b..0000000000 --- a/puppet/zulip/files/cron.d/update-analytics-counts +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -USER=zulip - -# Run update analytics tables every 5 minutes past the hour -5 * * * * zulip /home/zulip/deployments/current/manage.py update_analytics_counts diff --git a/puppet/zulip/manifests/app_frontend_base.pp b/puppet/zulip/manifests/app_frontend_base.pp index 605c49ad30..edfc45c2b9 100644 --- a/puppet/zulip/manifests/app_frontend_base.pp +++ b/puppet/zulip/manifests/app_frontend_base.pp @@ -245,12 +245,8 @@ class zulip::app_frontend_base { } # This cron job does nothing unless RATE_LIMIT_TOR_TOGETHER is enabled. - file { '/etc/cron.d/fetch-tor-exit-nodes': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/fetch-tor-exit-nodes', + zulip::cron { 'fetch-tor-exit-nodes': + minute => '17', } # This was originally added with a typo in the name. file { '/etc/cron.d/fetch-for-exit-nodes': diff --git a/puppet/zulip/manifests/app_frontend_once.pp b/puppet/zulip/manifests/app_frontend_once.pp index b75b9f6b5c..65d40e6244 100644 --- a/puppet/zulip/manifests/app_frontend_once.pp +++ b/puppet/zulip/manifests/app_frontend_once.pp @@ -21,67 +21,43 @@ class zulip::app_frontend_once { notify => Service[$zulip::common::supervisor_service], } - file { '/etc/cron.d/send-digest-emails': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/send-digest-emails', + # Every-hour + zulip::cron { 'update-analytics-counts': + minute => '5', + } + zulip::cron { 'check-analytics-state': + minute => '30', + } + zulip::cron { 'promote-new-full-members': + minute => '35', + } + zulip::cron { 'send_zulip_update_announcements': + minute => '47', } - file { '/etc/cron.d/update-analytics-counts': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/update-analytics-counts', + # Daily + zulip::cron { 'soft-deactivate-users': + hour => '5', + minute => '0', + manage => 'soft_deactivate_users -d', + } + zulip::cron { 'delete-old-unclaimed-attachments': + hour => '5', + minute => '0', + manage => 'delete_old_unclaimed_attachments -f', + } + zulip::cron { 'archive-messages': + hour => '6', + minute => '0', + } + zulip::cron { 'send-digest-emails': + hour => '18', + minute => '0', + manage => 'enqueue_digest_emails', + } + zulip::cron { 'clearsessions': + hour => '22', + minute => '22', } - file { '/etc/cron.d/check-analytics-state': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/check-analytics-state', - } - - file { '/etc/cron.d/soft-deactivate-users': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/soft-deactivate-users', - } - - file { '/etc/cron.d/promote-new-full-members': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/promote-new-full-members', - } - - file { '/etc/cron.d/archive-messages': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/archive-messages', - } - - file { '/etc/cron.d/clearsessions': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/clearsessions', - } - - file { '/etc/cron.d/send_zulip_update_announcements': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/send_zulip_update_announcements', - } } diff --git a/puppet/zulip/manifests/cron.pp b/puppet/zulip/manifests/cron.pp new file mode 100644 index 0000000000..f8fdd2bd42 --- /dev/null +++ b/puppet/zulip/manifests/cron.pp @@ -0,0 +1,25 @@ + +define zulip::cron( + String $minute, + String $hour = '*', + String $dow = '*', + String $user = 'zulip', + Optional[String] $command = undef, + Optional[String] $manage = undef, +) { + if $command != undef { + $run = $command + } elsif $manage != undef { + $run = "cd /home/zulip/deployments/current/ && ./manage.py ${manage} >/dev/null" + } else { + $underscores = regsubst($title, '-', '_', 'G') + $run = "cd /home/zulip/deployments/current/ && ./manage.py ${underscores} >/dev/null" + } + file { "/etc/cron.d/${title}": + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('zulip/cron.template.erb'), + } +} diff --git a/puppet/zulip/manifests/postgresql_backups.pp b/puppet/zulip/manifests/postgresql_backups.pp index 90032e78bb..86b407cd5a 100644 --- a/puppet/zulip/manifests/postgresql_backups.pp +++ b/puppet/zulip/manifests/postgresql_backups.pp @@ -25,12 +25,11 @@ class zulip::postgresql_backups { ], } - file { '/etc/cron.d/pg_backup_and_purge': - ensure => present, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/pg-backup-and-purge', + zulip::cron { 'pg_backup_and_purge': + hour => '2', + minute => '0', + command => '/usr/local/bin/pg_backup_and_purge >/var/log/pg_backup_and_purge.log 2>&1', + user => 'postgres', require => [ File['/var/log/pg_backup_and_purge.log'], File['/usr/local/bin/pg_backup_and_purge'], diff --git a/puppet/zulip/manifests/profile/app_frontend.pp b/puppet/zulip/manifests/profile/app_frontend.pp index 7321180886..365b83aaff 100644 --- a/puppet/zulip/manifests/profile/app_frontend.pp +++ b/puppet/zulip/manifests/profile/app_frontend.pp @@ -66,11 +66,10 @@ class zulip::profile::app_frontend { } # Restart the server regularly to avoid potential memory leak problems. - file { '/etc/cron.d/restart-zulip': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/restart-zulip', + zulip::cron { 'restart-zulip': + hour => '6', + minute => '0', + dow => '7', + command => '/home/zulip/deployments/current/scripts/restart-server --fill-cache', } } diff --git a/puppet/zulip/templates/cron.template.erb b/puppet/zulip/templates/cron.template.erb new file mode 100644 index 0000000000..180bd26c10 --- /dev/null +++ b/puppet/zulip/templates/cron.template.erb @@ -0,0 +1,7 @@ +MAILTO=zulip +PATH=/usr/local/bin:/usr/bin:/bin +SHELL=/bin/bash +USER=<%= @user %> + +<%= @minute %> <%= @hour %> * * <%= @dow %> <%= @user %> <%= @run %> +