diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2
index a33f4cae68..26877ebcfe 100755
--- a/scripts/lib/upgrade-zulip-stage-2
+++ b/scripts/lib/upgrade-zulip-stage-2
@@ -30,6 +30,7 @@ from scripts.lib.zulip_tools import (
     assert_running_as_root,
     get_config,
     get_config_file,
+    listening_publicly,
     parse_os_release,
     run_psql_as_postgres,
     start_arg_parser,
@@ -113,6 +114,21 @@ config_file = get_config_file()
 IS_SERVER_UP = True
 
 
+# Check if rabbitmq port 25672 is listening on anything except 127.0.0.1
+rabbitmq_dist_listen = listening_publicly(25672)
+if args.skip_puppet and rabbitmq_dist_listen:
+    logging.error(
+        "RabbitMQ is publicly-accessible on %s; this is a security vulnerability!",
+        ", ".join(rabbitmq_dist_listen),
+    )
+    logging.error(
+        "To fix the above security issue, re-run the upgrade without --skip-puppet "
+        "(which may be set in /etc/zulip/zulip.conf), in order to restart the "
+        "necessary services.  Running zulip-puppet-apply by itself is not sufficient."
+    )
+    sys.exit(1)
+
+
 def shutdown_server() -> None:
     global IS_SERVER_UP
 
@@ -283,6 +299,11 @@ if (not args.skip_puppet or migrations_needed) and IS_SERVER_UP:
     # state.
     shutdown_server()
 
+if rabbitmq_dist_listen:
+    shutdown_server()
+    logging.info("Shutting down rabbitmq to adjust its ports...")
+    subprocess.check_call(["/usr/sbin/service", "rabbitmq-server", "stop"])
+
 # Adjust Puppet class names for the manifest renames in the 4.0 release
 class_renames = {
     "zulip::app_frontend": "zulip::profile::app_frontend",
diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py
index 18274303f5..838ec3c3db 100755
--- a/scripts/lib/zulip_tools.py
+++ b/scripts/lib/zulip_tools.py
@@ -670,6 +670,24 @@ def start_arg_parser(action: str, add_help: bool = False) -> argparse.ArgumentPa
     return parser
 
 
+def listening_publicly(port: int) -> List[str]:
+    filter = f"sport = :{port} and not src 127.0.0.1:{port} and not src [::1]:{port}"
+    # Parse lines that look like this:
+    # tcp    LISTEN     0          128             0.0.0.0:25672        0.0.0.0:*
+    lines = (
+        subprocess.check_output(
+            ["/bin/ss", "-Hnl", filter],
+            universal_newlines=True,
+            # Hosts with IPv6 disabled will get "RTNETLINK answers: Invalid
+            # argument"; eat stderr to hide that
+            stderr=subprocess.DEVNULL,
+        )
+        .strip()
+        .splitlines()
+    )
+    return [line.split()[4] for line in lines]
+
+
 if __name__ == "__main__":
     cmd = sys.argv[1]
     if cmd == "make_deploy_path":