zulip/zerver/lib/debug.py

import code
import gc
import logging
import os
import signal
import socket
import threading
import traceback
import tracemalloc
from types import FrameType

from django.conf import settings
from django.utils.timezone import now as timezone_now
from typing import Optional

logger = logging.getLogger('zulip.debug')

# Interactive debugging code from
# https://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
# (that link also points to code for an interactive remote debugger
# setup, which we might want if we move Tornado to run in a daemon
# rather than via screen).
def interactive_debug(sig: int, frame: FrameType) -> None:
    """Interrupt running process, and provide a python prompt for
    interactive debugging."""
    d = {'_frame': frame}      # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    message  = "Signal received : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    i = code.InteractiveConsole(d)
    i.interact(message)

# SIGUSR1 => Just print the stack
# SIGUSR2 => Print stack + open interactive debugging shell
def interactive_debug_listen() -> None:
    signal.signal(signal.SIGUSR1, lambda sig, stack: traceback.print_stack(stack))
    signal.signal(signal.SIGUSR2, interactive_debug)

def tracemalloc_dump() -> None:
    if not tracemalloc.is_tracing():
        logger.warning("pid %s: tracemalloc off, nothing to dump",
                       os.getpid())
        return
    # Despite our name for it, `timezone_now` always deals in UTC.
    basename = "snap.{}.{}".format(os.getpid(),
                                   timezone_now().strftime("%F-%T"))
    path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
    os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)

    gc.collect()
    tracemalloc.take_snapshot().dump(path)

    with open(f'/proc/{os.getpid()}/stat', 'rb') as f:
        procstat = f.read().split()
    rss_pages = int(procstat[23])
    logger.info("tracemalloc dump: tracing %s MiB (%s MiB peak), using %s MiB; rss %s MiB; dumped %s",
                tracemalloc.get_traced_memory()[0] // 1048576,
                tracemalloc.get_traced_memory()[1] // 1048576,
                tracemalloc.get_tracemalloc_memory() // 1048576,
                rss_pages // 256,
                basename)

def tracemalloc_listen_sock(sock: socket.socket) -> None:
    logger.debug('pid %s: tracemalloc_listen_sock started!', os.getpid())
    while True:
        sock.recv(1)
        tracemalloc_dump()

listener_pid: Optional[int] = None

def tracemalloc_listen() -> None:
    global listener_pid
    if listener_pid == os.getpid():
        # Already set up -- and in this process, not just its parent.
        return
    logger.debug('pid %s: tracemalloc_listen working...', os.getpid())
    listener_pid = os.getpid()

    sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
    path = f"/tmp/tracemalloc.{os.getpid()}"
    sock.bind(path)
    thread = threading.Thread(target=lambda: tracemalloc_listen_sock(sock),
                              daemon=True)
    thread.start()
    logger.debug('pid %s: tracemalloc_listen done: %s', os.getpid(), path)

def maybe_tracemalloc_listen() -> None:
    '''If tracemalloc tracing enabled, listen for requests to dump a snapshot.

    To trigger once this is listening:
      echo | socat -u stdin unix-sendto:/tmp/tracemalloc.$pid

    To enable in the Zulip web server: edit /etc/zulip/uwsgi.ini ,
    and add e.g. ` PYTHONTRACEMALLOC=5` to the `env=` line.
    This function is called in middleware, so the process will
    automatically start listening.

    To enable in other contexts: see upstream docs
    https://docs.python.org/3/library/tracemalloc .
    You may also have to add a call to this function somewhere.

    '''
    if os.environ.get('PYTHONTRACEMALLOC'):
        # If the server was started with `tracemalloc` tracing on, then
        # listen for a signal to dump `tracemalloc` snapshots.
        tracemalloc_listen()