2020-03-21 13:10:22 +01:00
|
|
|
import time
|
2020-06-11 00:54:34 +02:00
|
|
|
from unittest import TestCase, mock
|
|
|
|
|
|
|
|
from scripts.lib.check_rabbitmq_queue import CRITICAL, OK, UNKNOWN, WARNING, analyze_queue_stats
|
|
|
|
|
2020-03-21 13:10:22 +01:00
|
|
|
|
|
|
|
class AnalyzeQueueStatsTests(TestCase):
|
|
|
|
def test_no_stats_available(self) -> None:
|
2021-02-12 08:20:45 +01:00
|
|
|
result = analyze_queue_stats("name", {}, 0)
|
|
|
|
self.assertEqual(result["status"], UNKNOWN)
|
2020-03-21 13:10:22 +01:00
|
|
|
|
|
|
|
def test_queue_stuck(self) -> None:
|
2020-09-06 18:26:27 +02:00
|
|
|
"""Last update > 5 minutes ago and there's events in the queue."""
|
2020-03-21 13:10:22 +01:00
|
|
|
|
2021-02-12 08:20:45 +01:00
|
|
|
result = analyze_queue_stats("name", {"update_time": time.time() - 301}, 100)
|
|
|
|
self.assertEqual(result["status"], CRITICAL)
|
|
|
|
self.assertIn("queue appears to be stuck", result["message"])
|
2020-03-21 13:10:22 +01:00
|
|
|
|
|
|
|
def test_queue_just_started(self) -> None:
|
|
|
|
"""
|
|
|
|
We just started processing a burst of events, and haven't processed enough
|
|
|
|
to log productivity statistics yet.
|
|
|
|
"""
|
2021-02-12 08:19:30 +01:00
|
|
|
result = analyze_queue_stats(
|
2021-02-12 08:20:45 +01:00
|
|
|
"name",
|
2021-02-12 08:19:30 +01:00
|
|
|
{
|
2021-02-12 08:20:45 +01:00
|
|
|
"update_time": time.time(),
|
|
|
|
"current_queue_size": 10000,
|
|
|
|
"recent_average_consume_time": None,
|
2021-02-12 08:19:30 +01:00
|
|
|
},
|
|
|
|
10000,
|
|
|
|
)
|
2021-02-12 08:20:45 +01:00
|
|
|
self.assertEqual(result["status"], OK)
|
2020-03-21 13:10:22 +01:00
|
|
|
|
|
|
|
def test_queue_normal(self) -> None:
|
|
|
|
"""10000 events and each takes a second => it'll take a long time to empty."""
|
2021-02-12 08:19:30 +01:00
|
|
|
result = analyze_queue_stats(
|
2021-02-12 08:20:45 +01:00
|
|
|
"name",
|
2021-02-12 08:19:30 +01:00
|
|
|
{
|
2021-02-12 08:20:45 +01:00
|
|
|
"update_time": time.time(),
|
|
|
|
"current_queue_size": 10000,
|
|
|
|
"queue_last_emptied_timestamp": time.time() - 10000,
|
|
|
|
"recent_average_consume_time": 1,
|
2021-02-12 08:19:30 +01:00
|
|
|
},
|
|
|
|
10000,
|
|
|
|
)
|
2021-02-12 08:20:45 +01:00
|
|
|
self.assertEqual(result["status"], CRITICAL)
|
|
|
|
self.assertIn("clearing the backlog", result["message"])
|
2020-03-21 13:10:22 +01:00
|
|
|
|
|
|
|
# If we're doing 10K/sec, it's OK.
|
2021-02-12 08:19:30 +01:00
|
|
|
result = analyze_queue_stats(
|
2021-02-12 08:20:45 +01:00
|
|
|
"name",
|
2021-02-12 08:19:30 +01:00
|
|
|
{
|
2021-02-12 08:20:45 +01:00
|
|
|
"update_time": time.time(),
|
|
|
|
"current_queue_size": 10000,
|
|
|
|
"queue_last_emptied_timestamp": time.time() - 10000,
|
|
|
|
"recent_average_consume_time": 0.0001,
|
2021-02-12 08:19:30 +01:00
|
|
|
},
|
|
|
|
10000,
|
|
|
|
)
|
2021-02-12 08:20:45 +01:00
|
|
|
self.assertEqual(result["status"], OK)
|
2020-03-21 13:10:22 +01:00
|
|
|
|
2020-09-20 13:35:35 +02:00
|
|
|
# Verify logic around whether it'll take MAX_SECONDS_TO_CLEAR to clear queue.
|
2021-02-12 08:20:45 +01:00
|
|
|
with mock.patch.dict("scripts.lib.check_rabbitmq_queue.MAX_SECONDS_TO_CLEAR", {"name": 10}):
|
2021-02-12 08:19:30 +01:00
|
|
|
result = analyze_queue_stats(
|
2021-02-12 08:20:45 +01:00
|
|
|
"name",
|
2021-02-12 08:19:30 +01:00
|
|
|
{
|
2021-02-12 08:20:45 +01:00
|
|
|
"update_time": time.time(),
|
|
|
|
"current_queue_size": 11,
|
|
|
|
"queue_last_emptied_timestamp": time.time() - 10000,
|
|
|
|
"recent_average_consume_time": 1,
|
2021-02-12 08:19:30 +01:00
|
|
|
},
|
|
|
|
11,
|
|
|
|
)
|
2021-02-12 08:20:45 +01:00
|
|
|
self.assertEqual(result["status"], WARNING)
|
|
|
|
self.assertIn("clearing the backlog", result["message"])
|
2020-03-21 13:10:22 +01:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
result = analyze_queue_stats(
|
2021-02-12 08:20:45 +01:00
|
|
|
"name",
|
2021-02-12 08:19:30 +01:00
|
|
|
{
|
2021-02-12 08:20:45 +01:00
|
|
|
"update_time": time.time(),
|
|
|
|
"current_queue_size": 9,
|
|
|
|
"queue_last_emptied_timestamp": time.time() - 10000,
|
|
|
|
"recent_average_consume_time": 1,
|
2021-02-12 08:19:30 +01:00
|
|
|
},
|
|
|
|
9,
|
|
|
|
)
|
2021-02-12 08:20:45 +01:00
|
|
|
self.assertEqual(result["status"], OK)
|