mirror of https://github.com/zulip/zulip.git
110 lines
4.1 KiB
Python
110 lines
4.1 KiB
Python
import code
|
|
import gc
|
|
import logging
|
|
import os
|
|
import signal
|
|
import socket
|
|
import threading
|
|
import traceback
|
|
import tracemalloc
|
|
from types import FrameType
|
|
|
|
from django.conf import settings
|
|
from django.utils.timezone import now as timezone_now
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger('zulip.debug')
|
|
|
|
# Interactive debugging code from
|
|
# https://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
|
|
# (that link also points to code for an interactive remote debugger
|
|
# setup, which we might want if we move Tornado to run in a daemon
|
|
# rather than via screen).
|
|
def interactive_debug(sig: int, frame: FrameType) -> None:
|
|
"""Interrupt running process, and provide a python prompt for
|
|
interactive debugging."""
|
|
d = {'_frame': frame} # Allow access to frame object.
|
|
d.update(frame.f_globals) # Unless shadowed by global
|
|
d.update(frame.f_locals)
|
|
|
|
message = "Signal received : entering python shell.\nTraceback:\n"
|
|
message += ''.join(traceback.format_stack(frame))
|
|
i = code.InteractiveConsole(d)
|
|
i.interact(message)
|
|
|
|
# SIGUSR1 => Just print the stack
|
|
# SIGUSR2 => Print stack + open interactive debugging shell
|
|
def interactive_debug_listen() -> None:
|
|
signal.signal(signal.SIGUSR1, lambda sig, stack: traceback.print_stack(stack))
|
|
signal.signal(signal.SIGUSR2, interactive_debug)
|
|
|
|
def tracemalloc_dump() -> None:
|
|
if not tracemalloc.is_tracing():
|
|
logger.warning("pid {}: tracemalloc off, nothing to dump"
|
|
.format(os.getpid()))
|
|
return
|
|
# Despite our name for it, `timezone_now` always deals in UTC.
|
|
basename = "snap.{}.{}".format(os.getpid(),
|
|
timezone_now().strftime("%F-%T"))
|
|
path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
|
|
os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
|
|
|
|
gc.collect()
|
|
tracemalloc.take_snapshot().dump(path)
|
|
|
|
with open('/proc/{}/stat'.format(os.getpid()), 'rb') as f:
|
|
procstat = f.read().split()
|
|
rss_pages = int(procstat[23])
|
|
logger.info("tracemalloc dump: tracing {} MiB ({} MiB peak), using {} MiB; rss {} MiB; dumped {}"
|
|
.format(tracemalloc.get_traced_memory()[0] // 1048576,
|
|
tracemalloc.get_traced_memory()[1] // 1048576,
|
|
tracemalloc.get_tracemalloc_memory() // 1048576,
|
|
rss_pages // 256,
|
|
basename))
|
|
|
|
def tracemalloc_listen_sock(sock: socket.socket) -> None:
|
|
logger.debug('pid {}: tracemalloc_listen_sock started!'.format(os.getpid()))
|
|
while True:
|
|
sock.recv(1)
|
|
tracemalloc_dump()
|
|
|
|
listener_pid = None # type: Optional[int]
|
|
|
|
def tracemalloc_listen() -> None:
|
|
global listener_pid
|
|
if listener_pid == os.getpid():
|
|
# Already set up -- and in this process, not just its parent.
|
|
return
|
|
logger.debug('pid {}: tracemalloc_listen working...'.format(os.getpid()))
|
|
listener_pid = os.getpid()
|
|
|
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
|
path = "/tmp/tracemalloc.{}".format(os.getpid())
|
|
sock.bind(path)
|
|
thread = threading.Thread(target=lambda: tracemalloc_listen_sock(sock),
|
|
daemon=True)
|
|
thread.start()
|
|
logger.debug('pid {}: tracemalloc_listen done: {}'.format(
|
|
os.getpid(), path))
|
|
|
|
def maybe_tracemalloc_listen() -> None:
|
|
'''If tracemalloc tracing enabled, listen for requests to dump a snapshot.
|
|
|
|
To trigger once this is listening:
|
|
echo | socat -u stdin unix-sendto:/tmp/tracemalloc.$pid
|
|
|
|
To enable in the Zulip web server: edit /etc/zulip/uwsgi.ini ,
|
|
and add e.g. ` PYTHONTRACEMALLOC=5` to the `env=` line.
|
|
This function is called in middleware, so the process will
|
|
automatically start listening.
|
|
|
|
To enable in other contexts: see upstream docs
|
|
https://docs.python.org/3/library/tracemalloc .
|
|
You may also have to add a call to this function somewhere.
|
|
|
|
'''
|
|
if os.environ.get('PYTHONTRACEMALLOC'):
|
|
# If the server was started with `tracemalloc` tracing on, then
|
|
# listen for a signal to dump `tracemalloc` snapshots.
|
|
tracemalloc_listen()
|