zulip/puppet/kandra/files/nagios4/conf.d/services.cfg

### SSH

define service {
        use                             generic-service
        service_description             SSH - pageable
        hostgroup_name                  pageable_servers
        check_command                   check_ssh
        contact_groups                  page_admins
}

define service {
        use                             generic-service
        service_description             SSH
        hostgroup_name                  not_pageable_servers
        check_command                   check_ssh
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             SSH for flaky machines
        hostgroup_name                  flaky_servers
        check_command                   check_ssh
        normal_check_interval           2
        retry_check_interval            2
        max_check_attempts              5
        contact_groups                  admins
}


### Disk usage

define service {
        use                             generic-service
        service_description             Disk usage - pageable
        hostgroup_name                  pageable_servers
        check_command                   check_remote_disk!20%!10%
        contact_groups                  page_admins
}

define service {
        use                             generic-service
        service_description             Disk usage
        hostgroup_name                  not_pageable_servers
        check_command                   check_remote_disk!20%!10%
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Disk usage for flaky machines
        hostgroup_name                  flaky_servers
        check_command                   check_remote_disk!20%!10%
        normal_check_interval           2
        retry_check_interval            2
        max_check_attempts              5
        contact_groups                  admins
}


### System updates

define service {
        use                             generic-service
        service_description             Debian update availability
        hostgroup_name                  all
        check_command                   check_debian_packages!22
        contact_groups                  admins
}


### NTP

define service {
        use                             generic-service
        service_description             Check AWS NTP time
        hostgroup_name                  aws_host
        check_command                   check_ntp_time!22!169.254.169.123
        max_check_attempts              3
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check non-AWS NTP time
        hostgroup_name                  non_aws_host
        check_command                   check_ntp_time!22!pool.ntp.org
        max_check_attempts              5
        contact_groups                  admins
}





### Service groups

#### zmirror / zmirrorp

define service {
        use                             generic-service
        service_description             zephyr mirror forwarding
        hostgroup_name                  zmirror
        check_command                   check_zephyr_mirror_forwarding!22
        normal_check_interval           2
# Note: the actual check is run via cron, so retry_check_interval
# should always equal normal_check_interval.
        retry_check_interval            2
        max_check_attempts              5
        contact_groups                  page_admins
}

define service {
        use                             generic-service
        service_description             zmirror subscriptions syncing
        hostgroup_name                  zmirror
        check_command                   check_sync_public_streams!22
        normal_check_interval           2
        retry_check_interval            2
        max_check_attempts              5
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check personal zephyr mirrors
        hostgroup_name                  zmirrorp
        check_command                   check_personal_zephyr_mirrors!22
        contact_groups                  admins
}


#### Application frontends

define service {
        use                             generic-service
        service_description             HTTPS
        hostgroup_name                  frontends
        check_command                   check_https_status
        contact_groups                  page_admins
}

define service {
        use                             generic-service
        service_description             Check send receive time
        hostgroup_name                  frontends
        check_command                   check_send_receive_time!22
        max_check_attempts              2
        contact_groups                  page_admins
}

define service {
        use                             generic-service
        service_description             user zephyr mirror liveness
        hostgroup_name                  prod_frontends
        check_command                   check_user_zephyr_mirror_liveness!22
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check analytics state
        hostgroup_name                  prod_frontends
        check_command                   check_analytics_state!22
        max_check_attempts              2
        contact_groups                  admins
}


#### PostgreSQL

define service {
        use                             generic-service
        service_description             Check PostgreSQL connection
        hostgroup_name                  postgresql
        check_command                   check_postgres!zulip!nagios!connection
        contact_groups                  page_admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL autovac_freeze
        hostgroup_name                  postgresql
        check_command                   check_postgres_alert_args!zulip!nagios!autovac_freeze!101%!105%
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL backends
        hostgroup_name                  postgresql
        check_command                   check_postgres!zulip!nagios!backends
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL disabled triggers
        hostgroup_name                  postgresql
        check_command                   check_postgres!zulip!nagios!disabled_triggers
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL hitratio
        hostgroup_name                  postgresql
        check_command                   check_postgres!zulip!nagios!hitratio
        contact_groups                  admins
}

# define service {
#         use                             generic-service
#         service_description             Check PostgreSQL locks
#         hostgroup_name                  postgresql
#         check_command                   check_postgres_alert_args!zulip!nagios!locks!400!600
#         contact_groups                  admins
# }

define service {
        use                             generic-service
        service_description             Check PostgreSQL query_time
        hostgroup_name                  postgresql
        check_command                   check_postgres_alert_args!zulip!nagios!query_time!20 seconds!40 seconds
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL sequence
        hostgroup_name                  postgresql
        check_command                   check_postgres!zulip!nagios!sequence
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL timesync
        hostgroup_name                  postgresql
        check_command                   check_postgres!zulip!nagios!timesync
        contact_groups                  admins
}

# define service {
#         use                             generic-service
#         service_description             Check PostgreSQL txn_idle
#         hostgroup_name                  postgresql
#         check_command                   check_postgres_alert_args!zulip!nagios!txn_idle!20 seconds!40 seconds
#         contact_groups                  admins
# }

define service {
        use                             generic-service
        service_description             Check PostgreSQL txn_time
        hostgroup_name                  postgresql
        check_command                   check_postgres_alert_args!zulip!nagios!txn_time!20 seconds!40 seconds
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check FTS update log length
        hostgroup_name                  postgresql_primary
        check_command                   check_fts_update_log
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check PostgreSQL replication lag
        hostgroup_name                  postgresql_primary, postgresql_replica
        check_command                   check_postgresql_replication_lag
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check last PostgreSQL backup time
        hostgroup_name                  postgresql
        check_command                   check_postgresql_backup
        contact_groups                  admins
}


#### Redis

define service {
        use                             generic-service
        service_description             Check redis service
        # Both redis and frontends hostgroups, since frontends SSH proxy redis to themselves
        hostgroup_name                  frontends, redis
        check_command                   check_redis_ssh!22
        max_check_attempts              3
        contact_groups                  page_admins
}


#### RabbitMQ / queue workers

define service {
        use                             generic-service
        service_description             Check RabbitMQ queue sizes
        hostgroup_name                  frontends
        check_command                   check_rabbitmq_queues!22
        # Workaround weird checks 40s after first error causing alerts
        # from a single failure because cron hasn't run again yet
        max_check_attempts              3
        contact_groups                  page_admins
}

define service {
        name                            rabbitmq-consumer-service
        use                             generic-service
        service_description             RabbitMQ consumer check template
        hostgroup_name                  frontends
        # Workaround weird checks 40s after first error causing alerts
        # from a single failure because cron hasn't run again yet
        max_check_attempts              3
        contact_groups                  admins
        register                        0
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ notify_tornado consumers
        check_command                   check_rabbitmq_consumers!notify_tornado
        contact_groups                  page_admins
}


define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ deferred_work consumers
        check_command                   check_rabbitmq_consumers!deferred_work
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ digest digest_emails consumers
        check_command                   check_rabbitmq_consumers!digest_emails
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ email_mirror consumers
        check_command                   check_rabbitmq_consumers!email_mirror
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ email_senders consumers
        check_command                   check_rabbitmq_consumers!email_senders
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ embed_links consumers
        check_command                   check_rabbitmq_consumers!embed_links

}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ embedded_bots consumers
        check_command                   check_rabbitmq_consumers!embedded_bots
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ missedmessage_emails consumers
        check_command                   check_rabbitmq_consumers!missedmessage_emails
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ missedmessage_mobile_notifications consumers
        check_command                   check_rabbitmq_consumers!missedmessage_mobile_notifications
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ outgoing_webhooks consumers
        check_command                   check_rabbitmq_consumers!outgoing_webhooks
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ user_activity consumers
        check_command                   check_rabbitmq_consumers!user_activity
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ user_activity_interval consumers
        check_command                   check_rabbitmq_consumers!user_activity_interval
}

define service {
        use                             rabbitmq-consumer-service
        service_description             Check RabbitMQ user_presence consumers
        check_command                   check_rabbitmq_consumers!user_presence
}

define service {
        use                             generic-service
        service_description             Check worker memory usage
        hostgroup_name                  frontends
        check_command                   check_worker_memory
        contact_groups                  admins
}

define service {
        use                             generic-service
        service_description             Check for queue worker errors.
        hostgroup_name                  frontends
        check_command                   check_queue_worker_errors!22
        contact_groups                  admins
}



### Smokescreen

define service {
        use                             generic-service
        service_description             Check Smokescreen proxy
        hostgroup_name                  smokescreen
        check_command                   check_proxy_status
        contact_groups                  page_admins
}