diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_backlog b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_backlog index 0115b918bf..f12e7e7958 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_backlog +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_backlog @@ -15,12 +15,10 @@ fi cd /home/zulip/deployments/current BACKLOG="$(./manage.py print_email_delivery_backlog)" -if [ "$BACKLOG" -gt 0 ] && [ "$BACKLOG" -lt 10 ] -then +if [ "$BACKLOG" -gt 0 ] && [ "$BACKLOG" -lt 10 ]; then echo "backlog of $BACKLOG" exit 1 -elif [ "$BACKLOG" -ge 10 ] -then +elif [ "$BACKLOG" -ge 10 ]; then echo "backlog of $BACKLOG" exit 2 else diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_process b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_process index ed9070adbf..6fcb996a20 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_process +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_email_deliverer_process @@ -8,13 +8,12 @@ SUPERVISOR_STATUS=$(supervisorctl status zulip-workers:zulip_deliver_enqueued_emails 2>&1) STATUS=$(echo "$SUPERVISOR_STATUS" | awk '{ print $2 }') - case "$STATUS" in RUNNING) echo "Running" exit 0 ;; - STOPPED|STARTING|BACKOFF|STOPPING|EXITED|FATAL|UNKNOWN) + STOPPED | STARTING | BACKOFF | STOPPING | EXITED | FATAL | UNKNOWN) # not "RUNNING", but a recognized supervisor status echo "$STATUS" exit 1 diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_worker_memory b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_worker_memory index b7d0d22225..c1a8d69375 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_worker_memory +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_worker_memory @@ -9,16 +9,16 @@ if [ -z "$processes" ]; then echo "No workers running" exit 0 fi -mapfile -t processes <<< "$processes" -ps -o vsize,size,pid,user,command --sort -vsize "${processes[@]}" > "$datafile" +mapfile -t processes <<<"$processes" +ps -o vsize,size,pid,user,command --sort -vsize "${processes[@]}" >"$datafile" cat "$datafile" top_worker=$(head -n2 "$datafile" | tail -n1) top_worker_memory_usage=$(echo "$top_worker" | cut -f1 -d" ") rm -f "$datafile" if [ "$top_worker_memory_usage" -gt 800000 ]; then - exit 2 + exit 2 elif [ "$top_worker_memory_usage" -gt 600000 ]; then - exit 1 + exit 1 else - exit 0 + exit 0 fi diff --git a/puppet/zulip_ops/files/munin-plugins/rabbitmq_connections b/puppet/zulip_ops/files/munin-plugins/rabbitmq_connections index 2a6e762b90..c65d2b0400 100755 --- a/puppet/zulip_ops/files/munin-plugins/rabbitmq_connections +++ b/puppet/zulip_ops/files/munin-plugins/rabbitmq_connections @@ -20,8 +20,8 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi HOME=/tmp/ @@ -30,33 +30,33 @@ HOME=/tmp/ # graphs should look. if [ "$1" = "config" ]; then - CONN_WARN=${queue_warn:-500} - CONN_CRIT=${queue_crit:-1000} + CONN_WARN=${queue_warn:-500} + CONN_CRIT=${queue_crit:-1000} - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo 'graph_title RabbitMQ connections' - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1000 -l 0' - # The Y-axis label - echo 'graph_vlabel connections' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category RabbitMQ' + # The title of the graph + echo 'graph_title RabbitMQ connections' + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1000 -l 0' + # The Y-axis label + echo 'graph_vlabel connections' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category RabbitMQ' - echo "connections.label Connections" - echo "connections.warning $CONN_WARN" - echo "connections.critical $CONN_CRIT" - echo "connections.info Number of active connections" + echo "connections.label Connections" + echo "connections.warning $CONN_WARN" + echo "connections.critical $CONN_CRIT" + echo "connections.info Number of active connections" - echo 'graph_info Shows the number of connections to RabbitMQ' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Shows the number of connections to RabbitMQ' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the diff --git a/puppet/zulip_ops/files/munin-plugins/rabbitmq_consumers b/puppet/zulip_ops/files/munin-plugins/rabbitmq_consumers index 03b8f812e0..6de46ec14d 100755 --- a/puppet/zulip_ops/files/munin-plugins/rabbitmq_consumers +++ b/puppet/zulip_ops/files/munin-plugins/rabbitmq_consumers @@ -20,54 +20,54 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi # If run with the "config"-parameter, give out information on how the # graphs should look. HOME=/tmp/ -QUEUES=$(HOME=$HOME rabbitmqctl list_queues name | \ - grep -v '^Listing' | \ - grep -v 'done\.$' | sed -e 's/[.=-]/_/g' ) +QUEUES=$(HOME=$HOME rabbitmqctl list_queues name \ + | grep -v '^Listing' \ + | grep -v 'done\.$' | sed -e 's/[.=-]/_/g') if [ "$1" = "config" ]; then - QUEUE_WARN=${queue_warn:-100} - QUEUE_CRIT=${queue_crit:-500} + QUEUE_WARN=${queue_warn:-100} + QUEUE_CRIT=${queue_crit:-500} - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo "graph_title RabbitMQ consumers" - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1000 -l 0' - # The Y-axis label - echo 'graph_vlabel consumers' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category RabbitMQ' + # The title of the graph + echo "graph_title RabbitMQ consumers" + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1000 -l 0' + # The Y-axis label + echo 'graph_vlabel consumers' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category RabbitMQ' - for queue in $QUEUES; do - echo "$queue.label $queue" - echo "$queue.warning $QUEUE_WARN" - echo "$queue.critical $QUEUE_CRIT" - echo "$queue.info Active consumers for $queue" - done + for queue in $QUEUES; do + echo "$queue.label $queue" + echo "$queue.warning $QUEUE_WARN" + echo "$queue.critical $QUEUE_CRIT" + echo "$queue.info Active consumers for $queue" + done - echo 'graph_info Lists active consumers for a queue.' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Lists active consumers for a queue.' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the # real work - i.e. display the data. Almost always this will be # "value" subfield for every data field. -HOME=$HOME rabbitmqctl list_queues name consumers| \ - grep -v "^Listing" | grep -v "done.$" | \ - perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' +HOME=$HOME rabbitmqctl list_queues name consumers \ + | grep -v "^Listing" | grep -v "done.$" \ + | perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' diff --git a/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages b/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages index b7269ad5ba..389e1b3ea4 100755 --- a/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages +++ b/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages @@ -20,54 +20,54 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi # If run with the "config"-parameter, give out information on how the # graphs should look. HOME=/tmp/ -QUEUES=$(HOME=$HOME rabbitmqctl list_queues name | \ - grep -v '^Listing' | \ - grep -v 'done\.$' | sed -e 's/[.=-]/_/g' ) +QUEUES=$(HOME=$HOME rabbitmqctl list_queues name \ + | grep -v '^Listing' \ + | grep -v 'done\.$' | sed -e 's/[.=-]/_/g') if [ "$1" = "config" ]; then - QUEUE_WARN=${queue_warn:-10000} - QUEUE_CRIT=${queue_crit:-20000} + QUEUE_WARN=${queue_warn:-10000} + QUEUE_CRIT=${queue_crit:-20000} - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo "graph_title RabbitMQ list_queues" - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1000 -l 0' - # The Y-axis label - echo 'graph_vlabel queue_size' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category RabbitMQ' + # The title of the graph + echo "graph_title RabbitMQ list_queues" + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1000 -l 0' + # The Y-axis label + echo 'graph_vlabel queue_size' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category RabbitMQ' - for queue in $QUEUES; do - echo "$queue.label $queue" - echo "$queue.warning $QUEUE_WARN" - echo "$queue.critical $QUEUE_CRIT" - echo "$queue.info Queue size for $queue" - done + for queue in $QUEUES; do + echo "$queue.label $queue" + echo "$queue.warning $QUEUE_WARN" + echo "$queue.critical $QUEUE_CRIT" + echo "$queue.info Queue size for $queue" + done - echo 'graph_info Lists how many messages are in each queue.' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Lists how many messages are in each queue.' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the # real work - i.e. display the data. Almost always this will be # "value" subfield for every data field. -HOME=$HOME rabbitmqctl list_queues | \ - grep -v "^Listing" | grep -v "done.$" | \ - perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' +HOME=$HOME rabbitmqctl list_queues \ + | grep -v "^Listing" | grep -v "done.$" \ + | perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' diff --git a/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_unacknowledged b/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_unacknowledged index 2d7660b607..2c078d47bc 100755 --- a/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_unacknowledged +++ b/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_unacknowledged @@ -20,54 +20,54 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi # If run with the "config"-parameter, give out information on how the # graphs should look. HOME=/tmp/ -QUEUES=$(HOME=$HOME rabbitmqctl list_queues name | \ - grep -v '^Listing' | \ - grep -v 'done\.$' | sed -e 's/[.=-]/_/g' ) +QUEUES=$(HOME=$HOME rabbitmqctl list_queues name \ + | grep -v '^Listing' \ + | grep -v 'done\.$' | sed -e 's/[.=-]/_/g') if [ "$1" = "config" ]; then - QUEUE_WARN=${queue_warn:-10000} - QUEUE_CRIT=${queue_crit:-20000} + QUEUE_WARN=${queue_warn:-10000} + QUEUE_CRIT=${queue_crit:-20000} - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo "graph_title RabbitMQ Unacknowledged Messages" - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1000 -l 0' - # The Y-axis label - echo 'graph_vlabel unacknowledged' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category RabbitMQ' + # The title of the graph + echo "graph_title RabbitMQ Unacknowledged Messages" + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1000 -l 0' + # The Y-axis label + echo 'graph_vlabel unacknowledged' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category RabbitMQ' - for queue in $QUEUES; do - echo "$queue.label $queue" - echo "$queue.warning $QUEUE_WARN" - echo "$queue.critical $QUEUE_CRIT" - echo "$queue.info Unacknowledged messages for $queue" - done + for queue in $QUEUES; do + echo "$queue.label $queue" + echo "$queue.warning $QUEUE_WARN" + echo "$queue.critical $QUEUE_CRIT" + echo "$queue.info Unacknowledged messages for $queue" + done - echo 'graph_info Lists how many messages are in each queue.' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Lists how many messages are in each queue.' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the # real work - i.e. display the data. Almost always this will be # "value" subfield for every data field. -HOME=$HOME rabbitmqctl list_queues name messages_unacknowledged | \ - grep -v "^Listing" | grep -v "done.$" | \ - perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' +HOME=$HOME rabbitmqctl list_queues name messages_unacknowledged \ + | grep -v "^Listing" | grep -v "done.$" \ + | perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' diff --git a/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_uncommitted b/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_uncommitted index 1a960c6d9c..0f9aac8c20 100755 --- a/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_uncommitted +++ b/puppet/zulip_ops/files/munin-plugins/rabbitmq_messages_uncommitted @@ -20,54 +20,54 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi # If run with the "config"-parameter, give out information on how the # graphs should look. HOME=/tmp/ -QUEUES=$(HOME=$HOME rabbitmqctl list_queues name | \ - grep -v '^Listing' | \ - grep -v 'done\.$' | sed -e 's/[.=-]/_/g' ) +QUEUES=$(HOME=$HOME rabbitmqctl list_queues name \ + | grep -v '^Listing' \ + | grep -v 'done\.$' | sed -e 's/[.=-]/_/g') if [ "$1" = "config" ]; then - QUEUE_WARN=${queue_warn:-10000} - QUEUE_CRIT=${queue_crit:-20000} + QUEUE_WARN=${queue_warn:-10000} + QUEUE_CRIT=${queue_crit:-20000} - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo "graph_title RabbitMQ Uncommitted Messages" - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1000 -l 0' - # The Y-axis label - echo 'graph_vlabel uncommitted' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category RabbitMQ' + # The title of the graph + echo "graph_title RabbitMQ Uncommitted Messages" + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1000 -l 0' + # The Y-axis label + echo 'graph_vlabel uncommitted' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category RabbitMQ' - for queue in $QUEUES; do - echo "$queue.label $queue" - echo "$queue.warning $QUEUE_WARN" - echo "$queue.critical $QUEUE_CRIT" - echo "$queue.info Uncommitted messages for $queue" - done + for queue in $QUEUES; do + echo "$queue.label $queue" + echo "$queue.warning $QUEUE_WARN" + echo "$queue.critical $QUEUE_CRIT" + echo "$queue.info Uncommitted messages for $queue" + done - echo 'graph_info Lists how many messages are in each queue.' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Lists how many messages are in each queue.' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the # real work - i.e. display the data. Almost always this will be # "value" subfield for every data field. -HOME=$HOME rabbitmqctl list_channels name messages_uncommitted | \ - grep -v "^Listing" | grep -v "done.$" | \ - perl -nle'($q, $s) = /^(.*)\s+(\d+)$/; $q =~ s/[.=-]/_/g; print("$q.value $s")' +HOME=$HOME rabbitmqctl list_channels name messages_uncommitted \ + | grep -v "^Listing" | grep -v "done.$" \ + | perl -nle'($q, $s) = /^(.*)\s+(\d+)$/; $q =~ s/[.=-]/_/g; print("$q.value $s")' diff --git a/puppet/zulip_ops/files/munin-plugins/rabbitmq_queue_memory b/puppet/zulip_ops/files/munin-plugins/rabbitmq_queue_memory index e1ae4a70e5..29e9f43c9e 100755 --- a/puppet/zulip_ops/files/munin-plugins/rabbitmq_queue_memory +++ b/puppet/zulip_ops/files/munin-plugins/rabbitmq_queue_memory @@ -20,54 +20,54 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi # If run with the "config"-parameter, give out information on how the # graphs should look. HOME=/tmp/ -QUEUES=$(rabbitmqctl list_queues name | \ - grep -v '^Listing' | \ - grep -v 'done\.$' | sed -e 's/[.=-]/_/g' ) +QUEUES=$(rabbitmqctl list_queues name \ + | grep -v '^Listing' \ + | grep -v 'done\.$' | sed -e 's/[.=-]/_/g') if [ "$1" = "config" ]; then - QUEUE_WARN=${queue_warn:-10000} - QUEUE_CRIT=${queue_crit:-20000} + QUEUE_WARN=${queue_warn:-10000} + QUEUE_CRIT=${queue_crit:-20000} - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo "graph_title RabbitMQ Memory used by queue" - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1024 --vertical-label Bytes -l 0' - # The Y-axis label - echo 'graph_vlabel memory' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category RabbitMQ' + # The title of the graph + echo "graph_title RabbitMQ Memory used by queue" + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1024 --vertical-label Bytes -l 0' + # The Y-axis label + echo 'graph_vlabel memory' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category RabbitMQ' - for queue in $QUEUES; do - echo "$queue.label $queue" - echo "$queue.warning $QUEUE_WARN" - echo "$queue.critical $QUEUE_CRIT" - echo "$queue.info Memory used by $queue" - done + for queue in $QUEUES; do + echo "$queue.label $queue" + echo "$queue.warning $QUEUE_WARN" + echo "$queue.critical $QUEUE_CRIT" + echo "$queue.info Memory used by $queue" + done - echo 'graph_info Show memory usage by queue' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Show memory usage by queue' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the # real work - i.e. display the data. Almost always this will be # "value" subfield for every data field. -HOME=$HOME rabbitmqctl list_queues name memory | \ - grep -v "^Listing" | grep -v "done.$" | \ - perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' +HOME=$HOME rabbitmqctl list_queues name memory \ + | grep -v "^Listing" | grep -v "done.$" \ + | perl -nle'($q, $s) = split; $q =~ s/[.=-]/_/g; print("$q.value $s")' diff --git a/puppet/zulip_ops/files/munin-plugins/tornado_event_queues b/puppet/zulip_ops/files/munin-plugins/tornado_event_queues index 94fb252c6b..9e45b91b8e 100755 --- a/puppet/zulip_ops/files/munin-plugins/tornado_event_queues +++ b/puppet/zulip_ops/files/munin-plugins/tornado_event_queues @@ -18,8 +18,8 @@ # always be included. if [ "$1" = "autoconf" ]; then - echo yes - exit 0 + echo yes + exit 0 fi HOME=/tmp/ @@ -28,30 +28,30 @@ HOME=/tmp/ # graphs should look. if [ "$1" = "config" ]; then - # The host name this plugin is for. (Can be overridden to have - # one machine answer for several) + # The host name this plugin is for. (Can be overridden to have + # one machine answer for several) - # The title of the graph - echo 'graph_title Event queues' - # Arguments to "rrdtool graph". In this case, tell it that the - # lower limit of the graph is '0', and that 1k=1000 (not 1024) - echo 'graph_args --base 1000 -l 0' - # The Y-axis label - echo 'graph_vlabel Number' - # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of - # 420 milliload) - #echo 'graph_scale no' - echo 'graph_category Tornado' + # The title of the graph + echo 'graph_title Event queues' + # Arguments to "rrdtool graph". In this case, tell it that the + # lower limit of the graph is '0', and that 1k=1000 (not 1024) + echo 'graph_args --base 1000 -l 0' + # The Y-axis label + echo 'graph_vlabel Number' + # We want Cur/Min/Avg/Max unscaled (i.e. 0.42 load instead of + # 420 milliload) + #echo 'graph_scale no' + echo 'graph_category Tornado' - echo "active_queues.label Total active event queues" - echo "active_queues.info Total number of active event queues" - echo "active_users.label Users with active event queues" - echo "active_users.info Number of users with active event queues" + echo "active_queues.label Total active event queues" + echo "active_queues.info Total number of active event queues" + echo "active_users.label Users with active event queues" + echo "active_users.info Number of users with active event queues" - echo 'graph_info Shows the number of active event queues' - # Last, if run with the "config"-parameter, quit here (don't - # display any data) - exit 0 + echo 'graph_info Shows the number of active event queues' + # Last, if run with the "config"-parameter, quit here (don't + # display any data) + exit 0 fi # If not run with any parameters at all (or only unknown ones), do the diff --git a/puppet/zulip_ops/files/postgresql/setup_disks.sh b/puppet/zulip_ops/files/postgresql/setup_disks.sh index c7739ab949..5367c1f9cd 100755 --- a/puppet/zulip_ops/files/postgresql/setup_disks.sh +++ b/puppet/zulip_ops/files/postgresql/setup_disks.sh @@ -5,7 +5,7 @@ set -e LOCALDISK=/dev/nvme0n1 if ! grep -q $LOCALDISK /etc/fstab; then - echo "$LOCALDISK /srv xfs nofail,noatime 1 1" >> /etc/fstab + echo "$LOCALDISK /srv xfs nofail,noatime 1 1" >>/etc/fstab fi if ! mountpoint -q /srv; then diff --git a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces_if-up.d.sh b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces_if-up.d.sh index b5dfd97fef..23c9215d4b 100755 --- a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces_if-up.d.sh +++ b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces_if-up.d.sh @@ -5,9 +5,9 @@ set -e # Only run from ifup. if [ "$MODE" != start ]; then - exit 0 + exit 0 fi if [ "$IFACE" = eth0 ]; then - /usr/local/sbin/zulip-ec2-configure-interfaces + /usr/local/sbin/zulip-ec2-configure-interfaces fi diff --git a/scripts/lib/certbot-maybe-renew b/scripts/lib/certbot-maybe-renew index 97999d2ebb..575e9056ca 100755 --- a/scripts/lib/certbot-maybe-renew +++ b/scripts/lib/certbot-maybe-renew @@ -6,7 +6,7 @@ zulip_conf_get_boolean() { # Treat absent and invalid values as false. value=$(crudini --get /etc/zulip/zulip.conf "$1" "$2" 2>/dev/null) case "$(echo "$value" | tr '[:upper:]' '[:lower:]')" in - 1|yes|true|on) return 0 ;; + 1 | yes | true | on) return 0 ;; *) return 1 ;; esac } @@ -18,5 +18,5 @@ fi deploy_hook="${ZULIP_CERTBOT_DEPLOY_HOOK:-service nginx reload}" certbot renew --quiet \ - --webroot --webroot-path=/var/lib/zulip/certbot-webroot/ \ - --deploy-hook "$deploy_hook" + --webroot --webroot-path=/var/lib/zulip/certbot-webroot/ \ + --deploy-hook "$deploy_hook" diff --git a/scripts/lib/install b/scripts/lib/install index 9a0ae1ca42..4200efa814 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -43,7 +43,7 @@ Options: Skip the initial `apt-get dist-upgrade`. EOF -}; +} # Shell option parsing. Over time, we'll want to move some of the # environment variables below into this self-documenting system. @@ -51,22 +51,62 @@ args="$(getopt -o '' --long help,hostname:,email:,certbot,self-signed-cert,cacer eval "set -- $args" while true; do case "$1" in - --help) usage; exit 0;; + --help) + usage + exit 0 + ;; - --hostname) EXTERNAL_HOST="$2"; shift; shift;; - --email) ZULIP_ADMINISTRATOR="$2"; shift; shift;; + --hostname) + EXTERNAL_HOST="$2" + shift + shift + ;; + --email) + ZULIP_ADMINISTRATOR="$2" + shift + shift + ;; - --certbot) USE_CERTBOT=1; shift;; - --cacert) export CUSTOM_CA_CERTIFICATES="$2"; shift; shift;; - --self-signed-cert) SELF_SIGNED_CERT=1; shift;; + --certbot) + USE_CERTBOT=1 + shift + ;; + --cacert) + export CUSTOM_CA_CERTIFICATES="$2" + shift + shift + ;; + --self-signed-cert) + SELF_SIGNED_CERT=1 + shift + ;; - --postgres-version) POSTGRES_VERSION="$2"; shift; shift;; - --postgres-missing-dictionaries) POSTGRES_MISSING_DICTIONARIES=1; shift;; - --no-init-db) NO_INIT_DB=1; shift;; + --postgres-version) + POSTGRES_VERSION="$2" + shift + shift + ;; + --postgres-missing-dictionaries) + POSTGRES_MISSING_DICTIONARIES=1 + shift + ;; + --no-init-db) + NO_INIT_DB=1 + shift + ;; - --no-overwrite-settings) NO_OVERWRITE_SETTINGS=1; shift;; - --no-dist-upgrade) NO_DIST_UPGRADE=1; shift;; - --) shift; break;; + --no-overwrite-settings) + NO_OVERWRITE_SETTINGS=1 + shift + ;; + --no-dist-upgrade) + NO_DIST_UPGRADE=1 + shift + ;; + --) + shift + break + ;; esac done @@ -78,9 +118,9 @@ fi ## Options from environment variables. # # Specify options for apt. -read -r -a APT_OPTIONS <<< "${APT_OPTIONS:-}" +read -r -a APT_OPTIONS <<<"${APT_OPTIONS:-}" # Install additional packages. -read -r -a ADDITIONAL_PACKAGES <<< "${ADDITIONAL_PACKAGES:-}" +read -r -a ADDITIONAL_PACKAGES <<<"${ADDITIONAL_PACKAGES:-}" # Comma-separated list of puppet manifests to install. default is # zulip::voyager for an all-in-one system or zulip::dockervoyager for # Docker. Use e.g. zulip::app_frontend for a Zulip frontend server. @@ -111,8 +151,8 @@ if [ -z "$EXTERNAL_HOST" ] || [ -z "$ZULIP_ADMINISTRATOR" ]; then fi fi -if [ "$EXTERNAL_HOST" = zulip.example.com ] || - [ "$ZULIP_ADMINISTRATOR" = zulip-admin@example.com ]; then +if [ "$EXTERNAL_HOST" = zulip.example.com ] \ + || [ "$ZULIP_ADMINISTRATOR" = zulip-admin@example.com ]; then # These example values are specifically checked for and would fail # later; see check_config in zerver/lib/management.py. echo 'error: The example hostname and email must be replaced with real values.' >&2 @@ -134,8 +174,16 @@ export LANGUAGE="en_US.UTF-8" # Check for a supported OS release. if [ -f /etc/os-release ]; then - os_info="$(. /etc/os-release; printf '%s\n' "$ID" "$ID_LIKE" "$VERSION_ID" "$VERSION_CODENAME")" - { read -r os_id; read -r os_id_like; read -r os_version_id; read -r os_version_codename || true; } <<< "$os_info" + os_info="$( + . /etc/os-release + printf '%s\n' "$ID" "$ID_LIKE" "$VERSION_ID" "$VERSION_CODENAME" + )" + { + read -r os_id + read -r os_id_like + read -r os_version_id + read -r os_version_codename || true + } <<<"$os_info" case " $os_id $os_id_like " in *' debian '*) package_system="apt" @@ -147,7 +195,7 @@ if [ -f /etc/os-release ]; then fi case "$os_id$os_version_id" in - debian10|ubuntu18.04|ubuntu20.04) ;; + debian10 | ubuntu18.04 | ubuntu20.04) ;; *) set +x cat <&2 echo "Installing packages failed; is network working and (on Ubuntu) the universe repository enabled?" >&2 @@ -286,9 +335,9 @@ if [ "$package_system" = apt ]; then fi elif [ "$package_system" = yum ]; then if ! yum install -y \ - puppet git curl wget jq \ - python3 crudini \ - "${ADDITIONAL_PACKAGES[@]}"; then + puppet git curl wget jq \ + python3 crudini \ + "${ADDITIONAL_PACKAGES[@]}"; then set +x echo -e '\033[0;31m' >&2 echo "Installing packages failed; is network working?" >&2 @@ -328,13 +377,13 @@ has_class() { id -u zulip &>/dev/null || useradd -m zulip --home-dir /home/zulip if [ -n "$NO_OVERWRITE_SETTINGS" ] && [ -e "/etc/zulip/zulip.conf" ]; then "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \ - --write-catalog-summary \ - --classfile=/var/lib/puppet/classes.txt \ - >/dev/null + --write-catalog-summary \ + --classfile=/var/lib/puppet/classes.txt \ + >/dev/null else # Write out more than we need, and remove sections that are not # applicable to the classes that are actually necessary. - cat < /etc/zulip/zulip.conf + cat </etc/zulip/zulip.conf [machine] puppet_classes = $PUPPET_CLASSES deploy_type = production @@ -352,9 +401,9 @@ EOF fi "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \ - --write-catalog-summary \ - --classfile=/var/lib/puppet/classes.txt \ - >/dev/null + --write-catalog-summary \ + --classfile=/var/lib/puppet/classes.txt \ + >/dev/null # We only need the postgres version setting on database hosts; but # we don't know if this is a database host until we have the catalog summary. diff --git a/scripts/lib/install-node b/scripts/lib/install-node index 617166f1ab..798a8946ec 100755 --- a/scripts/lib/install-node +++ b/scripts/lib/install-node @@ -31,7 +31,10 @@ fi if [ "$current_node_version" != "v$node_version" ] || ! [ -L "$node_wrapper_path" ]; then export NVM_DIR=/usr/local/nvm # shellcheck source=/dev/null - if ! [ -e "$NVM_DIR/nvm.sh" ] || { . "$NVM_DIR/nvm.sh"; [ "$(nvm --version)" != "$nvm_version" ]; }; then + if ! [ -e "$NVM_DIR/nvm.sh" ] || { + . "$NVM_DIR/nvm.sh" + [ "$(nvm --version)" != "$nvm_version" ] + }; then mkdir -p "$NVM_DIR" wget_opts=(-nv) if [ -n "${CUSTOM_CA_CERTIFICATES:-}" ]; then diff --git a/scripts/lib/setup-apt-repo b/scripts/lib/setup-apt-repo index 40602cbc2f..af6423fdbf 100755 --- a/scripts/lib/setup-apt-repo +++ b/scripts/lib/setup-apt-repo @@ -36,7 +36,7 @@ apt-get -y install "${pre_setup_deps[@]}" SCRIPTS_PATH="$(dirname "$(dirname "$0")")" release=$(lsb_release -sc) -if [[ "$release" =~ ^(bionic|cosmic|disco|eoan|focal)$ ]] ; then +if [[ "$release" =~ ^(bionic|cosmic|disco|eoan|focal)$ ]]; then apt-key add "$SCRIPTS_PATH"/setup/pgdg.asc apt-key add "$SCRIPTS_PATH"/setup/pgroonga-ppa.asc cat >$SOURCES_FILE <$SOURCES_FILE < "$DEPENDENCIES_HASH_FILE" +echo "$DEPENDENCIES_HASH" >"$DEPENDENCIES_HASH_FILE" diff --git a/scripts/lib/setup-apt-repo-debathena b/scripts/lib/setup-apt-repo-debathena index 75b6e55c61..876a9ff13f 100755 --- a/scripts/lib/setup-apt-repo-debathena +++ b/scripts/lib/setup-apt-repo-debathena @@ -54,4 +54,4 @@ else apt-get update && rm -f "$STAMP_FILE" fi -echo "$DEPENDENCIES_HASH" > "$DEPENDENCIES_HASH_FILE" +echo "$DEPENDENCIES_HASH" >"$DEPENDENCIES_HASH_FILE" diff --git a/scripts/lib/setup-yum-repo b/scripts/lib/setup-yum-repo index 6cbdeb6c70..8207366f85 100755 --- a/scripts/lib/setup-yum-repo +++ b/scripts/lib/setup-yum-repo @@ -7,8 +7,14 @@ args="$(getopt -o '' --long prod -- "$@")" eval "set -- $args" while true; do case "$1" in - --prod) is_prod=true; shift;; - --) shift; break;; + --prod) + is_prod=true + shift + ;; + --) + shift + break + ;; esac done diff --git a/scripts/setup/generate-self-signed-cert b/scripts/setup/generate-self-signed-cert index 071679850d..dd1c0e8bd5 100755 --- a/scripts/setup/generate-self-signed-cert +++ b/scripts/setup/generate-self-signed-cert @@ -10,11 +10,20 @@ args="$(getopt -o '' --long help,force,exists-ok -- "$@")" eval "set -- $args" while true; do case "$1" in - --help) usage;; - --force) FORCE=1; shift;; - --exists-ok) EXISTS_OK=1; shift;; - --) shift; break;; - *) usage;; + --help) usage ;; + --force) + FORCE=1 + shift + ;; + --exists-ok) + EXISTS_OK=1 + shift + ;; + --) + shift + break + ;; + *) usage ;; esac done EXTERNAL_HOST="$1" @@ -51,9 +60,9 @@ fi rm -f "$KEYFILE" "$CERTFILE" if [[ "$EXTERNAL_HOST" =~ ^(([0-9]+\.){3}[0-9]+)(:[0-9]+)?$ ]]; then - subjectAltName="IP:${BASH_REMATCH[1]}" # IPv4 address + subjectAltName="IP:${BASH_REMATCH[1]}" # IPv4 address elif [[ "$EXTERNAL_HOST" =~ ^\[([^][]*)\](:[0-9]+)?$ ]]; then - subjectAltName="IP:${BASH_REMATCH[1]}" # IPv6 address + subjectAltName="IP:${BASH_REMATCH[1]}" # IPv6 address elif [[ "$EXTERNAL_HOST" =~ ^([^:]+)(:[0-9]+)?$ ]]; then subjectAltName="DNS:${BASH_REMATCH[1]}" else @@ -94,8 +103,8 @@ fi # Based on /usr/sbin/make-ssl-cert from Debian's `ssl-cert` package. openssl req -new -x509 \ - -config "$config" -days 3650 -nodes -sha256 \ - -out "$CERTFILE" -keyout "$KEYFILE" + -config "$config" -days 3650 -nodes -sha256 \ + -out "$CERTFILE" -keyout "$KEYFILE" chmod 644 "$CERTFILE" chmod 640 "$KEYFILE" diff --git a/scripts/setup/initialize-database b/scripts/setup/initialize-database index f5a8500889..47cbc2c7ab 100755 --- a/scripts/setup/initialize-database +++ b/scripts/setup/initialize-database @@ -10,10 +10,16 @@ args="$(getopt -o '' --long help,quiet -- "$@")" eval "set -- $args" while true; do case "$1" in - --help) usage;; - --quiet) QUIET=1; shift;; - --) shift; break;; - *) usage;; + --help) usage ;; + --quiet) + QUIET=1 + shift + ;; + --) + shift + break + ;; + *) usage ;; esac done diff --git a/scripts/setup/postgres-init-db b/scripts/setup/postgres-init-db index 96c3f29539..71a1303d34 100755 --- a/scripts/setup/postgres-init-db +++ b/scripts/setup/postgres-init-db @@ -13,15 +13,14 @@ POSTGRES_USER="${POSTGRES_USER:-postgres}" # This psql command may fail because the zulip database doesn’t exist, # hence the &&. if records="$( - cd / # Make sure the current working directory is readable by postgres + cd / # Make sure the current working directory is readable by postgres su "$POSTGRES_USER" -c "psql -v ON_ERROR_STOP=1 -Atc 'SELECT COUNT(*) FROM zulip.zerver_message;' zulip" )" && [ "$records" -gt 200 ]; then set +x echo "WARNING: This will delete your Zulip database which currently contains $records messages." read -p "Do you want to proceed? [y/N] " -r echo - if [[ ! $REPLY =~ ^[Yy]$ ]] - then + if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1 fi set -x @@ -35,12 +34,12 @@ fi # Drop any open connections to any old database. # Send the script via stdin in case the postgres user lacks permission to read it. su -s /usr/bin/env - -- "$POSTGRES_USER" \ - bash -s - zulip zulip_base < "$(dirname "$0")/terminate-psql-sessions" + bash -s - zulip zulip_base <"$(dirname "$0")/terminate-psql-sessions" ( - cd / # Make sure the current working directory is readable by postgres + cd / # Make sure the current working directory is readable by postgres su "$POSTGRES_USER" -c 'psql -v ON_ERROR_STOP=1 -e' -) < "$(dirname "$0")/create-db.sql" +) <"$(dirname "$0")/create-db.sql" # Clear memcached to avoid contamination from previous database state "$(dirname "$0")/flush-memcached" diff --git a/scripts/setup/setup-certbot b/scripts/setup/setup-certbot index 4a9efeb7f7..d5f75563c4 100755 --- a/scripts/setup/setup-certbot +++ b/scripts/setup/setup-certbot @@ -83,8 +83,14 @@ esac # Check for a supported OS release. if [ -f /etc/os-release ]; then - os_info="$(. /etc/os-release; printf '%s\n' "$ID" "$ID_LIKE")" - { read -r os_id; read -r os_id_like|| true; } <<< "$os_info" + os_info="$( + . /etc/os-release + printf '%s\n' "$ID" "$ID_LIKE" + )" + { + read -r os_id + read -r os_id_like || true + } <<<"$os_info" fi set -x @@ -104,10 +110,10 @@ esac # Passing --force-interactive suppresses a warning, but also brings up # an annoying prompt we stifle with --no-eff-email. certbot certonly "${method_args[@]}" \ - "${HOSTNAMES[@]}" -m "$EMAIL" \ - $agree_tos \ - "${deploy_hook[@]}" \ - --force-interactive --no-eff-email + "${HOSTNAMES[@]}" -m "$EMAIL" \ + $agree_tos \ + "${deploy_hook[@]}" \ + --force-interactive --no-eff-email symlink_with_backup() { if [ -e "$2" ]; then diff --git a/scripts/setup/sha256-tarball-to b/scripts/setup/sha256-tarball-to index b2e1b2a260..1edbbd1498 100755 --- a/scripts/setup/sha256-tarball-to +++ b/scripts/setup/sha256-tarball-to @@ -28,7 +28,7 @@ LOCALFILE="archive.tar.gz" wget -qO "$LOCALFILE" "$URL" # Check the hash against what was passed in -echo "$SHA256 $LOCALFILE" > "$LOCALFILE.sha256" +echo "$SHA256 $LOCALFILE" >"$LOCALFILE.sha256" sha256sum -c "$LOCALFILE.sha256" tar xzf "$LOCALFILE" diff --git a/tools/build-release-tarball b/tools/build-release-tarball index 19d7cd5792..a0594c800f 100755 --- a/tools/build-release-tarball +++ b/tools/build-release-tarball @@ -11,8 +11,11 @@ eval "set -- $args" while true; do case "$1" in - --) shift; break;; - *) usage;; + --) + shift + break + ;; + *) usage ;; esac done @@ -47,8 +50,8 @@ git archive -o "$TARBALL" "--prefix=$prefix/" HEAD cd "$TMPDIR" tar -xf "$TARBALL" while read -r i; do - rm -r --interactive=never "${TMPDIR:?}/$prefix/$i"; -done < "$TMPDIR/$prefix/tools/release-tarball-exclude.txt" + rm -r --interactive=never "${TMPDIR:?}/$prefix/$i" +done <"$TMPDIR/$prefix/tools/release-tarball-exclude.txt" tar -cf "$TARBALL" "$prefix" rm -rf "$prefix" @@ -78,10 +81,10 @@ mkdir -p "var/log" # TODO: Would be much better to instead run the below tools with some # sort of environment hack so that we don't need to create this dummy # secrets file. -cat >> zproject/prod_settings_template.py <>zproject/prod_settings_template.py <> zproject/dev-secrets.conf <>zproject/dev-secrets.conf < build_id -echo "$version" > version +echo "$GITID" >build_id +echo "$version" >version cd "$TMPDIR" diff --git a/tools/cache-zulip-git-version b/tools/cache-zulip-git-version index 82113f586a..330a6cd854 100755 --- a/tools/cache-zulip-git-version +++ b/tools/cache-zulip-git-version @@ -2,4 +2,4 @@ set -e cd "$(dirname "$0")/.." -git describe --tags --match='[0-9]*' > zulip-git-version || true +git describe --tags --match='[0-9]*' >zulip-git-version || true diff --git a/tools/ci/backend b/tools/ci/backend index 90327300a6..2bb1134c50 100755 --- a/tools/ci/backend +++ b/tools/ci/backend @@ -6,7 +6,7 @@ echo "Test suite is running under $(python --version)." set -e set -x -./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky +./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky ./tools/test-tools # We need to pass a parallel level to test-backend because CircleCI's # docker setup means the auto-detection logic sees the ~36 processes diff --git a/tools/ci/frontend b/tools/ci/frontend index 85b30280c3..87ea614b5a 100755 --- a/tools/ci/frontend +++ b/tools/ci/frontend @@ -5,7 +5,7 @@ source tools/ci/activate-venv set -e set -x -./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky +./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky # Run the node tests first, since they're fast and deterministic ./tools/test-js-with-node --coverage diff --git a/tools/ci/production-build b/tools/ci/production-build index 72323e5a7c..8e7f22bed9 100755 --- a/tools/ci/production-build +++ b/tools/ci/production-build @@ -33,11 +33,11 @@ fi mkdir /tmp/production-build mv /tmp/tmp.*/zulip-server-test.tar.gz /tmp/production-build cp -a \ - tools/ci/success-http-headers.template.txt \ - tools/ci/production-install \ - tools/ci/production-verify \ - tools/ci/production-upgrade-pg \ - tools/ci/production-extract-tarball \ - package.json yarn.lock \ - \ - /tmp/production-build + tools/ci/success-http-headers.template.txt \ + tools/ci/production-install \ + tools/ci/production-verify \ + tools/ci/production-upgrade-pg \ + tools/ci/production-extract-tarball \ + package.json yarn.lock \ + \ + /tmp/production-build diff --git a/tools/ci/production-install b/tools/ci/production-install index 739ab0b117..581b58aa16 100755 --- a/tools/ci/production-install +++ b/tools/ci/production-install @@ -14,8 +14,11 @@ APT_OPTIONS=(-o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-co apt-get update if [ -f /etc/os-release ]; then - os_info="$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME")" - { read -r os_version_codename || true; } <<< "$os_info" + os_info="$( + . /etc/os-release + printf '%s\n' "$VERSION_CODENAME" + )" + { read -r os_version_codename || true; } <<<"$os_info" fi if ! apt-get dist-upgrade -y "${APT_OPTIONS[@]}"; then diff --git a/tools/ci/production-verify b/tools/ci/production-verify index f6d53dfa60..99c24b8f8c 100755 --- a/tools/ci/production-verify +++ b/tools/ci/production-verify @@ -12,7 +12,9 @@ NOREPLY_EMAIL_ADDRESS = 'noreply@circleci.example.com' ALLOWED_HOSTS = [] EOF -echo; echo "Now testing that the supervisord jobs are running properly"; echo +echo +echo "Now testing that the supervisord jobs are running properly" +echo sleep 15 # Guaranteed to have a working supervisord process get an extra digit if supervisorctl status | grep -vq RUNNING || supervisorctl status | sed 's/^.*uptime //' | grep -q 0:00:0; then set +x @@ -33,16 +35,18 @@ if supervisorctl status | grep -vq RUNNING || supervisorctl status | sed 's/^.*u fi # TODO: Ideally this would test actually logging in, but this is a start. -echo; echo "Now testing that the newly installed server's homepage loads"; echo +echo +echo "Now testing that the newly installed server's homepage loads" +echo -wget https://localhost -O /tmp/index.html --no-check-certificate -S 2> /tmp/wget-output || true # || true so we see errors.log if this 500s -grep -vi '\(Vary\|Content-Language\|expires\|issued by\|modified\|saved\|[.][.][.]\|Date\|[-][-]\)' /tmp/wget-output > /tmp/http-headers-processed +wget https://localhost -O /tmp/index.html --no-check-certificate -S 2>/tmp/wget-output || true # || true so we see errors.log if this 500s +grep -vi '\(Vary\|Content-Language\|expires\|issued by\|modified\|saved\|[.][.][.]\|Date\|[-][-]\)' /tmp/wget-output >/tmp/http-headers-processed nginx_version="$(nginx -v 2>&1 | awk '{print $3, $4}')" # Simplify the diff by getting replacing 4-5 digit length numbers with . sed -i 's|Length: [0-9]\+\( [(][0-9]\+[.][0-9]K[)]\)\?|Length: |' /tmp/http-headers-processed -sed -i -e 's|Length: [0-9]\+\( [(][0-9]\+[.][0-9]K[)]\)\?|Length: |' -e "s|{nginx_version_string}|$nginx_version|g" /tmp/success-http-headers.template.txt +sed -i -e 's|Length: [0-9]\+\( [(][0-9]\+[.][0-9]K[)]\)\?|Length: |' -e "s|{nginx_version_string}|$nginx_version|g" /tmp/success-http-headers.template.txt if ! diff -ur /tmp/http-headers-processed /tmp/success-http-headers.template.txt; then set +x echo @@ -58,12 +62,14 @@ if ! diff -ur /tmp/http-headers-processed /tmp/success-http-headers.template.txt fi # Start the RabbitMQ queue worker related section -echo; echo "Now confirming all the RabbitMQ queue processors are correctly registered!"; echo +echo +echo "Now confirming all the RabbitMQ queue processors are correctly registered!" +echo # These hacky shell scripts just extract the sorted list of queue processors, running and expected -supervisorctl status | cut -f1 -dR | cut -f2- -d: | grep events | cut -f1 -d" " | cut -f3- -d_ | cut -f1 -d- | sort -u > /tmp/running_queue_processors.txt -su zulip -c /home/zulip/deployments/current/scripts/lib/queue_workers.py | sort -u > /tmp/all_queue_processors.txt -su zulip -c "/home/zulip/deployments/current/scripts/lib/queue_workers.py --queue-type test" | sort -u > /tmp/expected_test_queues.txt -grep -v -x -f /tmp/expected_test_queues.txt /tmp/all_queue_processors.txt > /tmp/expected_queue_processors.txt +supervisorctl status | cut -f1 -dR | cut -f2- -d: | grep events | cut -f1 -d" " | cut -f3- -d_ | cut -f1 -d- | sort -u >/tmp/running_queue_processors.txt +su zulip -c /home/zulip/deployments/current/scripts/lib/queue_workers.py | sort -u >/tmp/all_queue_processors.txt +su zulip -c "/home/zulip/deployments/current/scripts/lib/queue_workers.py --queue-type test" | sort -u >/tmp/expected_test_queues.txt +grep -v -x -f /tmp/expected_test_queues.txt /tmp/all_queue_processors.txt >/tmp/expected_queue_processors.txt if ! diff /tmp/expected_queue_processors.txt /tmp/running_queue_processors.txt >/dev/null; then set +x echo "FAILURE: Runnable queue processors declared in zerver/worker/queue_processors.py " @@ -74,7 +80,9 @@ if ! diff /tmp/expected_queue_processors.txt /tmp/running_queue_processors.txt > exit 1 fi -echo; echo "Now running RabbitMQ consumer Nagios tests"; echo +echo +echo "Now running RabbitMQ consumer Nagios tests" +echo # First run the check that usually runs in cron and populates the state files /home/zulip/deployments/current/scripts/nagios/check-rabbitmq-consumers @@ -95,10 +103,12 @@ done # Some of the Nagios tests have been temporarily disabled to work # around a Travis CI infrastructure issue. -echo; echo "Now running additional Nagios tests"; echo -if ! /usr/lib/nagios/plugins/zulip_app_frontend/check_queue_worker_errors || \ - ! su zulip -c /usr/lib/nagios/plugins/zulip_postgres_appdb/check_fts_update_log; then # || \ -# ! su zulip -c "/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time --site=https://127.0.0.1/api --nagios --insecure"; then +echo +echo "Now running additional Nagios tests" +echo +if ! /usr/lib/nagios/plugins/zulip_app_frontend/check_queue_worker_errors \ + || ! su zulip -c /usr/lib/nagios/plugins/zulip_postgres_appdb/check_fts_update_log; then # || \ + # ! su zulip -c "/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time --site=https://127.0.0.1/api --nagios --insecure"; then set +x echo echo "FAILURE: Nagios checks don't pass:" diff --git a/tools/clean-branches b/tools/clean-branches index 226307ee82..18451839d9 100755 --- a/tools/clean-branches +++ b/tools/clean-branches @@ -17,11 +17,11 @@ if [ $# -ne 0 ] && [ "$1" == "--reviews" ]; then fi push_args=() -function is_merged { +function is_merged() { ! git rev-list -n 1 origin/master.."$1" | grep -q . } -function clean_ref { +function clean_ref() { ref="$1" case "$ref" in */master | */HEAD) diff --git a/tools/commit-msg b/tools/commit-msg index bdc85d7335..12db6e0ab0 100755 --- a/tools/commit-msg +++ b/tools/commit-msg @@ -8,12 +8,12 @@ # Do not invoke gitlint if commit message is empty if grep -q '^[^#]' "$1"; then lint_cmd="cd ~/zulip && python -m gitlint.cli" - if \ - if [ -z "$VIRTUAL_ENV" ] && command -v vagrant > /dev/null && [ -e .vagrant ]; then + if + if [ -z "$VIRTUAL_ENV" ] && command -v vagrant >/dev/null && [ -e .vagrant ]; then ! vagrant ssh -c "$lint_cmd" else ! eval "$lint_cmd" - fi < "$1" + fi <"$1" then echo "WARNING: Your commit message does not match Zulip's style guide." fi diff --git a/tools/deploy-branch b/tools/deploy-branch index 6f00c7a26b..8649392b6e 100755 --- a/tools/deploy-branch +++ b/tools/deploy-branch @@ -1,6 +1,6 @@ #!/usr/bin/env bash -function error_out { +function error_out() { echo -en '\e[0;31m' echo "$1" echo -en '\e[0m' diff --git a/tools/pre-commit b/tools/pre-commit index 24eab091ec..3a8482c41d 100755 --- a/tools/pre-commit +++ b/tools/pre-commit @@ -18,7 +18,7 @@ if [ ${#changed_files} -eq 0 ]; then exit 0 fi -if [ -z "$VIRTUAL_ENV" ] && command -v vagrant > /dev/null && [ -e .vagrant ]; then +if [ -z "$VIRTUAL_ENV" ] && command -v vagrant >/dev/null && [ -e .vagrant ]; then vcmd="/srv/zulip/tools/lint --skip=gitlint --force $(printf '%q ' "${changed_files[@]}") || true" echo "Running lint using vagrant..." vagrant ssh -c "$vcmd" diff --git a/tools/provision b/tools/provision index 5efce99945..92bcc5ef8b 100755 --- a/tools/provision +++ b/tools/provision @@ -36,12 +36,15 @@ WARNING='\033[93m' ENDC='\033[0m' # Make the script independent of the location from where it is executed -PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +PARENT_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" + pwd -P +) cd "$PARENT_PATH" mkdir -p ../var/log LOG_PATH="../var/log/provision.log" -echo "PROVISIONING STARTING." >> $LOG_PATH +echo "PROVISIONING STARTING." >>$LOG_PATH # PYTHONUNBUFFERED is important to ensure that tracebacks don't get # lost far above where they should be in the output. diff --git a/tools/push-to-pull-request b/tools/push-to-pull-request index 6c77642607..90a30295a7 100755 --- a/tools/push-to-pull-request +++ b/tools/push-to-pull-request @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -usage () { +usage() { cat >&2 < zerver/tests/fixtures/messages.json + zerver.DefaultStream >zerver/tests/fixtures/messages.json # create pristine template database, for fast fixture restoration after tests are run. create_zulip_test_template diff --git a/tools/reset-to-pull-request b/tools/reset-to-pull-request index e525b94e13..0480d5535f 100755 --- a/tools/reset-to-pull-request +++ b/tools/reset-to-pull-request @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -usage () { +usage() { cat >&2 < /etc/hostname +echo "$HOSTNAME" >/etc/hostname hostname "$HOSTNAME" sed -i "s/localhost$/localhost $HOSTNAME $SERVER/" /etc/hosts @@ -57,15 +57,15 @@ EOF # function so we do can it again later with the zulip user function install_keys() { USERNAME="$1" - SSHDIR="$( getent passwd "$USERNAME" | cut -d: -f6 )/.ssh" + SSHDIR="$(getent passwd "$USERNAME" | cut -d: -f6)/.ssh" KEYDATA="$($AWS --output text \ - secretsmanager get-secret-value \ - --secret-id "$SSH_SECRET_ID" \ - --query SecretString)" + secretsmanager get-secret-value \ + --secret-id "$SSH_SECRET_ID" \ + --query SecretString)" mkdir -p "$SSHDIR" - echo "$KEYDATA" | jq -r .public | base64 -d > "$SSHDIR/id_rsa.pub" - echo "$KEYDATA" | jq -r .private | base64 -d > "$SSHDIR/id_rsa" - chown -R "$USERNAME:$USERNAME" "$SSHDIR" + echo "$KEYDATA" | jq -r .public | base64 -d >"$SSHDIR/id_rsa.pub" + echo "$KEYDATA" | jq -r .private | base64 -d >"$SSHDIR/id_rsa" + chown -R "$USERNAME:$USERNAME" "$SSHDIR" chmod 600 "$SSHDIR/id_rsa" } install_keys root diff --git a/tools/setup/bootstrap-awscli b/tools/setup/bootstrap-awscli index e797e59ad9..678df76f4c 100755 --- a/tools/setup/bootstrap-awscli +++ b/tools/setup/bootstrap-awscli @@ -8,7 +8,7 @@ if [ ! -d "/srv/zulip-aws-tools/v2/$AWS_CLI_VERSION" ]; then cd /srv/zulip-aws-tools || exit 1 rm -rf awscli.zip awscli.zip.sha256 aws/ wget -q "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -O awscli.zip - echo "$AWS_CLI_SHA awscli.zip" > awscli.zip.sha256 + echo "$AWS_CLI_SHA awscli.zip" >awscli.zip.sha256 sha256sum -c awscli.zip.sha256 unzip -q awscli.zip ( diff --git a/tools/setup/generate-fixtures b/tools/setup/generate-fixtures index 11a02894cc..e4a43700eb 100755 --- a/tools/setup/generate-fixtures +++ b/tools/setup/generate-fixtures @@ -1,11 +1,10 @@ #!/usr/bin/env bash set -e -run() -{ +run() { PGHOST=localhost PGUSER=zulip \ "$(dirname "$0")/../../scripts/setup/terminate-psql-sessions" zulip_test zulip_test_base zulip_test_template - psql -v ON_ERROR_STOP=1 -h localhost postgres zulip_test << EOF + psql -v ON_ERROR_STOP=1 -h localhost postgres zulip_test < var/puppeteer/test_credentials.js <var/puppeteer/test_credentials.js <> "$BOOTDATA" +} >>"$BOOTDATA" TAGS="[{Key=Name,Value=$SERVER},{Key=role,Value=\"$ROLES\"}]" INSTANCE_DATA=$($AWS ec2 run-instances \ - --iam-instance-profile 'Name="EC2ProdInstance"' \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --security-group-ids "$SECURITY_GROUPS" \ - --tag-specifications "ResourceType=instance,Tags=$TAGS" \ - --user-data "file://$BOOTDATA") + --iam-instance-profile 'Name="EC2ProdInstance"' \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --security-group-ids "$SECURITY_GROUPS" \ + --tag-specifications "ResourceType=instance,Tags=$TAGS" \ + --user-data "file://$BOOTDATA") INSTANCEID=$(echo "$INSTANCE_DATA" | jq -r .Instances[0].InstanceId) # Wait for public IP assignment @@ -89,12 +89,12 @@ PUBLIC_DNS_NAME="" while [ -z "$PUBLIC_DNS_NAME" ]; do sleep 1 PUBLIC_DNS_NAME=$($AWS ec2 describe-instances --instance-ids "$INSTANCEID" \ - | jq -r .Reservations[0].Instances[0].PublicDnsName ) + | jq -r .Reservations[0].Instances[0].PublicDnsName) done # Add the hostname to the zone ROUTE53_CHANGES=$(mktemp) -cat > "$ROUTE53_CHANGES" <"$ROUTE53_CHANGES" <&2 } loglevel=() -usage () { +usage() { cat <&2; exit 1; } +args="$(getopt -o hL: --long help,loglevel:,skip-check-links,skip-external-links -- "$@")" \ + || { + usage >&2 + exit 1 + } eval "set -- $args" while true; do case "$1" in - -h|--help) usage; exit 0;; - -L|--loglevel) loglevel=("$1" "$2"); shift 2;; - --skip-check-links) skip_check_links=1; shift;; - --skip-external-links) skip_external_links=1; shift;; - --) shift; break;; - *) exit 1;; + -h | --help) + usage + exit 0 + ;; + -L | --loglevel) + loglevel=("$1" "$2") + shift 2 + ;; + --skip-check-links) + skip_check_links=1 + shift + ;; + --skip-external-links) + skip_external_links=1 + shift + ;; + --) + shift + break + ;; + *) exit 1 ;; esac done diff --git a/tools/test-install/destroy-all b/tools/test-install/destroy-all index ac94576e9f..2eea9aaec5 100755 --- a/tools/test-install/destroy-all +++ b/tools/test-install/destroy-all @@ -10,10 +10,16 @@ args="$(getopt -o +f --long help,force -- "$@")" eval "set -- $args" while true; do case "$1" in - --help) usage;; - -f|--force) FORCE=1; shift;; - --) shift; break;; - *) usage;; + --help) usage ;; + -f | --force) + FORCE=1 + shift + ;; + --) + shift + break + ;; + *) usage ;; esac done @@ -27,9 +33,8 @@ if [ "$EUID" -ne 0 ]; then fi lxc-ls -f \ - | perl -lane '$_ = $F[0]; print if (/^zulip-install-/ && !/-base$/)' \ - | while read -r c -do - echo "$c" - lxc-destroy -f -n "$c" -done + | perl -lane '$_ = $F[0]; print if (/^zulip-install-/ && !/-base$/)' \ + | while read -r c; do + echo "$c" + lxc-destroy -f -n "$c" + done diff --git a/tools/test-install/install b/tools/test-install/install index d1f6ffab60..416f0fb763 100755 --- a/tools/test-install/install +++ b/tools/test-install/install @@ -10,14 +10,23 @@ args="$(getopt -o +r: --long help,release: -- "$@")" eval "set -- $args" while true; do case "$1" in - --help) usage;; - -r|--release) RELEASE="$2"; shift; shift;; - --) shift; break;; - *) usage;; + --help) usage ;; + -r | --release) + RELEASE="$2" + shift + shift + ;; + --) + shift + break + ;; + *) usage ;; esac done -INSTALLER="$1"; shift || usage -INSTALLER_ARGS=("$@"); set -- +INSTALLER="$1" +shift || usage +INSTALLER_ARGS=("$@") +set -- if [ -z "$RELEASE" ] || [ -z "$INSTALLER" ]; then usage @@ -42,7 +51,8 @@ while [ -z "$CONTAINER_NAME" ] || lxc-info -n "$CONTAINER_NAME" >/dev/null 2>&1; CONTAINER_NAME=zulip-install-"$(basename "$shared_dir")" done -message="$(cat <&2 runlevel="$(lxc-attach --clear-env -n "$CONTAINER_NAME" -- runlevel 2>/dev/null)" \ - || { sleep 1; continue; } + || { + sleep 1 + continue + } if [ "$runlevel" != "${0%[0-9]}" ]; then echo "lxc-wait: $CONTAINER_NAME: booted!" >&2 poll_network @@ -42,14 +52,16 @@ poll_runlevel() { exit 1 } - poll_network() { for _ in {1..60}; do echo "lxc-wait: $CONTAINER_NAME: polling for network..." >&2 # New hosts don't have `host` or `nslookup` lxc-attach --clear-env -n "$CONTAINER_NAME" -- \ - ping -q -c 1 archive.ubuntu.com 2>/dev/null >/dev/null \ - || { sleep 1; continue; } + ping -q -c 1 archive.ubuntu.com 2>/dev/null >/dev/null \ + || { + sleep 1 + continue + } echo "lxc-wait: $CONTAINER_NAME: network is up!" >&2 exit 0 done @@ -57,6 +69,4 @@ poll_network() { exit 1 } - poll_runlevel - diff --git a/tools/test-install/prepare-base b/tools/test-install/prepare-base index bb31c09d07..c2f1f5f9a3 100755 --- a/tools/test-install/prepare-base +++ b/tools/test-install/prepare-base @@ -7,13 +7,15 @@ if [ "$EUID" -ne 0 ]; then fi RELEASE="$1" -ARCH=amd64 # TODO: maybe i686 too +ARCH=amd64 # TODO: maybe i686 too case "$RELEASE" in - bionic) extra_packages=(python-pip) - ;; - focal) extra_packages=(python3-pip) - ;; + bionic) + extra_packages=(python-pip) + ;; + focal) + extra_packages=(python3-pip) + ;; *) echo "error: unsupported target release: $RELEASE" >&2 exit 1 @@ -46,16 +48,16 @@ run apt-get dist-upgrade -y # As an optimization, we install a bunch of packages the installer # would install for itself. run apt-get install -y --no-install-recommends \ - xvfb parallel unzip zip jq python3-pip wget curl eatmydata \ - git crudini openssl ssl-cert \ - build-essential python3-dev \ - memcached redis-server \ - hunspell-en-us supervisor libssl-dev puppet \ - gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \ - libldap2-dev \ - libxml2-dev libxslt1-dev libpq-dev \ - virtualenv \ - "${extra_packages[@]}" + xvfb parallel unzip zip jq python3-pip wget curl eatmydata \ + git crudini openssl ssl-cert \ + build-essential python3-dev \ + memcached redis-server \ + hunspell-en-us supervisor libssl-dev puppet \ + gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \ + libldap2-dev \ + libxml2-dev libxslt1-dev libpq-dev \ + virtualenv \ + "${extra_packages[@]}" run ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime run locale-gen en_US.UTF-8 || true diff --git a/tools/test-migrations b/tools/test-migrations index 8fde855351..724605d694 100755 --- a/tools/test-migrations +++ b/tools/test-migrations @@ -7,13 +7,13 @@ echo 'Testing whether migrations are consistent with models' new_auto_named_migrations=$(./manage.py showmigrations \ | grep -E ' [0-9]{4}_auto_' \ | grep -Eve ' [0-9]{4}_auto_201[67]' \ - -e ' 0052_auto_fix_realmalias_realm_nullable' \ - -e ' 0003_auto_20150817_1733' \ - -e ' 0002_auto_20150110_0810' \ - -e ' 0002_auto_20190420_0723' \ - -e ' 0009_auto_20191118_0520' \ + -e ' 0052_auto_fix_realmalias_realm_nullable' \ + -e ' 0003_auto_20150817_1733' \ + -e ' 0002_auto_20150110_0810' \ + -e ' 0002_auto_20190420_0723' \ + -e ' 0009_auto_20191118_0520' \ | sed 's/\[[x ]\] / /' \ - || true) + || true) if [ "$new_auto_named_migrations" != "" ]; then echo "ERROR: New migrations with unclear automatically generated names." echo "Please rename these migrations to have readable names:" diff --git a/tools/update-locked-requirements b/tools/update-locked-requirements index 43d56406d2..93b6499646 100755 --- a/tools/update-locked-requirements +++ b/tools/update-locked-requirements @@ -6,7 +6,7 @@ if [ ! -d /srv/zulip-py3-venv ] || [ ! -d /srv/zulip-thumbor-venv ]; then ./tools/setup/setup_venvs.py fi -compile_requirements () { +compile_requirements() { source="$1" output="$2"