py3: Switch almost all shebang lines to use `python3`.
This causes `upgrade-zulip-from-git`, as well as a no-option run of
`tools/build-release-tarball`, to produce a Zulip install running
Python 3, rather than Python 2. In particular this means that the
virtualenv we create, in which all application code runs, is Python 3.
One shebang line, on `zulip-ec2-configure-interfaces`, explicitly
keeps Python 2, and at least one external ops script, `wal-e`, also
still runs on Python 2. See discussion on the respective previous
commits that made those explicit. There may also be some other
third-party scripts we use, outside of this source tree and running
outside our virtualenv, that still run on Python 2.
2017-08-02 23:15:16 +02:00
|
|
|
#!/usr/bin/env python3
|
2017-11-09 08:13:07 +01:00
|
|
|
import argparse
|
2018-11-07 01:45:26 +01:00
|
|
|
import configparser
|
2013-10-28 15:54:32 +01:00
|
|
|
import os
|
2013-11-08 22:45:20 +01:00
|
|
|
import subprocess
|
2020-06-11 00:54:34 +02:00
|
|
|
import sys
|
|
|
|
import time
|
|
|
|
from collections import defaultdict
|
2019-07-23 23:58:11 +02:00
|
|
|
from typing import Dict
|
2017-03-05 00:35:26 +01:00
|
|
|
|
2020-04-03 20:25:28 +02:00
|
|
|
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
sys.path.append(ZULIP_PATH)
|
|
|
|
from scripts.lib.check_rabbitmq_queue import normal_queues
|
|
|
|
|
2013-04-16 20:07:53 +02:00
|
|
|
states = {
|
|
|
|
0: "OK",
|
|
|
|
1: "WARNING",
|
|
|
|
2: "CRITICAL",
|
python: Use trailing commas consistently.
Automatically generated by the following script, based on the output
of lint with flake8-comma:
import re
import sys
last_filename = None
last_row = None
lines = []
for msg in sys.stdin:
m = re.match(
r"\x1b\[35mflake8 \|\x1b\[0m \x1b\[1;31m(.+):(\d+):(\d+): (\w+)", msg
)
if m:
filename, row_str, col_str, err = m.groups()
row, col = int(row_str), int(col_str)
if filename == last_filename:
assert last_row != row
else:
if last_filename is not None:
with open(last_filename, "w") as f:
f.writelines(lines)
with open(filename) as f:
lines = f.readlines()
last_filename = filename
last_row = row
line = lines[row - 1]
if err in ["C812", "C815"]:
lines[row - 1] = line[: col - 1] + "," + line[col - 1 :]
elif err in ["C819"]:
assert line[col - 2] == ","
lines[row - 1] = line[: col - 2] + line[col - 1 :].lstrip(" ")
if last_filename is not None:
with open(last_filename, "w") as f:
f.writelines(lines)
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-10 05:23:40 +02:00
|
|
|
3: "UNKNOWN",
|
2013-04-16 20:07:53 +02:00
|
|
|
}
|
|
|
|
|
2013-10-28 15:54:32 +01:00
|
|
|
if 'USER' in os.environ and not os.environ['USER'] in ['root', 'rabbitmq']:
|
2016-03-10 17:15:34 +01:00
|
|
|
print("This script must be run as the root or rabbitmq user")
|
2013-04-16 20:07:53 +02:00
|
|
|
|
|
|
|
|
|
|
|
usage = """Usage: check-rabbitmq-consumers --queue=[queue-name] --min-threshold=[min-threshold]"""
|
|
|
|
|
2017-11-09 08:13:07 +01:00
|
|
|
parser = argparse.ArgumentParser(usage=usage)
|
|
|
|
parser.add_argument('--min-threshold',
|
|
|
|
dest='min_count',
|
|
|
|
type=int,
|
2020-09-02 21:10:51 +02:00
|
|
|
default=1)
|
2013-04-16 20:07:53 +02:00
|
|
|
|
2017-11-09 08:13:07 +01:00
|
|
|
options = parser.parse_args()
|
2013-04-16 20:07:53 +02:00
|
|
|
|
2018-11-07 01:45:26 +01:00
|
|
|
config_file = configparser.RawConfigParser()
|
|
|
|
config_file.read("/etc/zulip/zulip.conf")
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
def get_config(section: str, key: str, default_value: str) -> str:
|
2018-11-07 01:45:26 +01:00
|
|
|
if config_file.has_option(section, key):
|
|
|
|
return config_file.get(section, key)
|
|
|
|
return default_value
|
|
|
|
TORNADO_PROCESSES = int(get_config('application_server', 'tornado_processes', '1'))
|
|
|
|
|
2016-07-23 20:33:58 +02:00
|
|
|
output = subprocess.check_output(['/usr/sbin/rabbitmqctl', 'list_consumers'],
|
|
|
|
universal_newlines=True)
|
2013-04-16 20:07:53 +02:00
|
|
|
|
2020-04-22 01:09:50 +02:00
|
|
|
consumers: Dict[str, int] = defaultdict(int)
|
2013-04-16 20:07:53 +02:00
|
|
|
|
2020-09-02 02:50:08 +02:00
|
|
|
queues = {
|
|
|
|
*normal_queues,
|
2018-11-03 00:06:13 +01:00
|
|
|
# These queues may not be present if settings.TORNADO_PROCESSES > 1
|
|
|
|
'notify_tornado',
|
2020-09-02 02:50:08 +02:00
|
|
|
}
|
2016-08-12 23:09:36 +02:00
|
|
|
|
2017-02-23 07:56:03 +01:00
|
|
|
for queue_name in queues:
|
2017-02-21 06:47:56 +01:00
|
|
|
queue_name = queue_name.strip()
|
2016-08-12 23:09:36 +02:00
|
|
|
consumers[queue_name] = 0
|
|
|
|
|
2013-04-16 20:07:53 +02:00
|
|
|
for line in output.split('\n'):
|
|
|
|
parts = line.split('\t')
|
2016-08-12 23:09:36 +02:00
|
|
|
if len(parts) >= 2:
|
2018-11-07 01:45:26 +01:00
|
|
|
queue_name = parts[0]
|
|
|
|
if queue_name.startswith("notify_tornado_"):
|
|
|
|
queue_name = "notify_tornado"
|
|
|
|
consumers[queue_name] += 1
|
2013-04-16 20:07:53 +02:00
|
|
|
|
|
|
|
now = int(time.time())
|
|
|
|
|
2016-08-12 23:09:36 +02:00
|
|
|
for queue_name in consumers.keys():
|
|
|
|
state_file_path = "/var/lib/nagios_state/check-rabbitmq-consumers-" + queue_name
|
|
|
|
state_file_tmp = state_file_path + "-tmp"
|
2013-04-16 20:07:53 +02:00
|
|
|
|
2018-11-07 01:45:26 +01:00
|
|
|
target_count = options.min_count
|
dependencies: Remove WebSockets system for sending messages.
Zulip has had a small use of WebSockets (specifically, for the code
path of sending messages, via the webapp only) since ~2013. We
originally added this use of WebSockets in the hope that the latency
benefits of doing so would allow us to avoid implementing a markdown
local echo; they were not. Further, HTTP/2 may have eliminated the
latency difference we hoped to exploit by using WebSockets in any
case.
While we’d originally imagined using WebSockets for other endpoints,
there was never a good justification for moving more components to the
WebSockets system.
This WebSockets code path had a lot of downsides/complexity,
including:
* The messy hack involving constructing an emulated request object to
hook into doing Django requests.
* The `message_senders` queue processor system, which increases RAM
needs and must be provisioned independently from the rest of the
server).
* A duplicate check_send_receive_time Nagios test specific to
WebSockets.
* The requirement for users to have their firewalls/NATs allow
WebSocket connections, and a setting to disable them for networks
where WebSockets don’t work.
* Dependencies on the SockJS family of libraries, which has at times
been poorly maintained, and periodically throws random JavaScript
exceptions in our production environments without a deep enough
traceback to effectively investigate.
* A total of about 1600 lines of our code related to the feature.
* Increased load on the Tornado system, especially around a Zulip
server restart, and especially for large installations like
zulipchat.com, resulting in extra delay before messages can be sent
again.
As detailed in
https://github.com/zulip/zulip/pull/12862#issuecomment-536152397, it
appears that removing WebSockets moderately increases the time it
takes for the `send_message` API query to return from the server, but
does not significantly change the time between when a message is sent
and when it is received by clients. We don’t understand the reason
for that change (suggesting the possibility of a measurement error),
and even if it is a real change, we consider that potential small
latency regression to be acceptable.
If we later want WebSockets, we’ll likely want to just use Django
Channels.
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2019-07-23 01:43:40 +02:00
|
|
|
if queue_name == "notify_tornado":
|
2018-11-07 01:45:26 +01:00
|
|
|
target_count = TORNADO_PROCESSES
|
|
|
|
|
|
|
|
if consumers[queue_name] < target_count:
|
2016-08-12 23:09:36 +02:00
|
|
|
status = 2
|
|
|
|
else:
|
|
|
|
status = 0
|
|
|
|
with open(state_file_tmp, "w") as f:
|
2020-06-10 06:41:04 +02:00
|
|
|
f.write(f"{now}|{status}|{states[status]}|queue {queue_name} has {consumers[queue_name]} consumers, needs {target_count}\n")
|
2018-07-18 23:50:15 +02:00
|
|
|
os.rename(state_file_tmp, state_file_path)
|