diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index cb9d6fb179..bb19eb81fc 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -15,11 +15,12 @@ All notable changes to the **Prowler API** are documented in this file. - Gunicorn worker timeout raised from the 30s default to 120s, so long-running requests are no longer killed prematurely [(#11631)](https://github.com/prowler-cloud/prowler/pull/11631) - Sentry now drops ASGI's `RequestAborted` errors from health-check probe disconnects on `/health/live` [(#11632)](https://github.com/prowler-cloud/prowler/pull/11632) +- Gunicorn keep-alive timeout now exceeds the load balancer idle timeout, stopping 502s from reused connections [(#11647)](https://github.com/prowler-cloud/prowler/pull/11647) +- API runs under the Uvicorn worker so keep-alive outlives the load balancer idle timeout, fixing Gunicorn's intermittent 502s [(#11663)](https://github.com/prowler-cloud/prowler/pull/11663) ### 🐞 Fixed - Database connections no longer leak under the ASGI worker, which previously exhausted the read replica's connection slots and caused 500s on read endpoints [(#11640)](https://github.com/prowler-cloud/prowler/pull/11640) -- Gunicorn keep-alive timeout now exceeds the load balancer idle timeout, stopping 502s from reused connections [(#11647)](https://github.com/prowler-cloud/prowler/pull/11647) ### 🔐 Security diff --git a/api/pyproject.toml b/api/pyproject.toml index 41014d3382..f678b908fc 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -62,7 +62,8 @@ dependencies = [ "gevent (==25.9.1)", "werkzeug (==3.1.7)", "sqlparse (==0.5.5)", - "fonttools (==4.62.1)" + "fonttools (==4.62.1)", + "uvicorn-worker (==0.4.0)", ] description = "Prowler's API (Django/DRF)" license = "Apache-2.0" @@ -422,6 +423,7 @@ constraint-dependencies = [ "uritemplate==4.2.0", "urllib3==2.7.0", "uuid6==2024.7.10", + "uvicorn==0.49.0", "uvloop==0.22.1", "vine==5.1.0", "vulture==2.14", diff --git a/api/src/backend/config/guniconf.py b/api/src/backend/config/guniconf.py index 3b8b7dbe8c..6f85a9e1bc 100644 --- a/api/src/backend/config/guniconf.py +++ b/api/src/backend/config/guniconf.py @@ -3,6 +3,8 @@ import multiprocessing import os import threading +from uvicorn_worker import UvicornWorker + from config.env import env # Ensure the environment variable for Django settings is set @@ -12,6 +14,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production") import django # noqa: E402 django.setup() + from api.compliance import warm_compliance_caches # noqa: E402 from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG # noqa: E402 from config.custom_logging import BackendLogger # noqa: E402 @@ -19,34 +22,28 @@ from config.custom_logging import BackendLogger # noqa: E402 BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1") PORT = env("DJANGO_PORT", default=8080) + +class ProwlerUvicornWorker(UvicornWorker): + CONFIG_KWARGS = { + # Keep-alive idle timeout. Must exceed the load balancer idle timeout. + "timeout_keep_alive": env.int("GUNICORN_KEEPALIVE", default=75), + "loop": "uvloop", + "lifespan": "off", # Django ASGIHandler doesn't handle lifespan scopes + } + + +# Required so SSE endpoints can keep the event loop alive while waiting for events +worker_class = env( + "DJANGO_WORKER_CLASS", + default="config.guniconf.ProwlerUvicornWorker", +) + # Server settings bind = f"{BIND_ADDRESS}:{PORT}" workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1) reload = DEBUG -# Native ASGI worker (gunicorn 24+). Required so SSE endpoints can keep the -# event loop alive while waiting for events. -worker_class = env("DJANGO_WORKER_CLASS", default="asgi") - -# Lifespan protocol. Django's ASGIHandler (config.asgi:application) serves only -# HTTP scopes and raises "Django can only handle ASGI/HTTP connections, not -# lifespan." gunicorn's default ("auto") probes the app with a lifespan scope -# to detect support, which triggers that error. We use no lifespan startup or -# shutdown hooks, so disable the protocol entirely. -asgi_lifespan = env("DJANGO_ASGI_LIFESPAN", default="off") - -# Event loop for the ASGI worker. "auto" uses uvloop when it is installed and -# falls back to the stdlib asyncio loop otherwise; uvloop gives the SSE event -# loop more headroom under many concurrent open streams. -asgi_loop = env("DJANGO_ASGI_LOOP", default="uvloop") - -# Max concurrent connections per ASGI worker. Each open SSE stream holds one -# connection for its whole lifetime, so this caps simultaneous SSE clients per -# worker (gunicorn's default is 1000). The sync-only `threads` option has no -# effect on ASGI workers. -worker_connections = env.int("DJANGO_WORKER_CONNECTIONS", default=1000) - # Preload the application before forking workers in production: the app is # imported once in the master and workers fork from it. In development, disable # preload so the server restarts on code changes. @@ -56,12 +53,6 @@ preload_app = not DEBUG # that may take longer, such as complex API operations. timeout = env.int("GUNICORN_TIMEOUT", default=120) -# HTTP keep-alive idle timeout. Must exceed the idle timeout of the proxy or load -# balancer in front of gunicorn, or it reuses a connection gunicorn just closed -# and returns a 502. Default clears the common 60s; raise `GUNICORN_KEEPALIVE` to -# stay above a longer one. -keepalive = env.int("GUNICORN_KEEPALIVE", default=75) - # Logging logconfig_dict = DJANGO_LOGGERS gunicorn_logger = logging.getLogger(BackendLogger.GUNICORN) diff --git a/api/uv.lock b/api/uv.lock index f96318c1f4..3b5cca9b6a 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -357,6 +357,7 @@ constraints = [ { name = "uritemplate", specifier = "==4.2.0" }, { name = "urllib3", specifier = "==2.7.0" }, { name = "uuid6", specifier = "==2024.7.10" }, + { name = "uvicorn", specifier = "==0.49.0" }, { name = "uvloop", specifier = "==0.22.1" }, { name = "vine", specifier = "==5.1.0" }, { name = "vulture", specifier = "==2.14" }, @@ -4575,6 +4576,7 @@ dependencies = [ { name = "sentry-sdk", extra = ["django"] }, { name = "sqlparse" }, { name = "uuid6" }, + { name = "uvicorn-worker" }, { name = "uvloop" }, { name = "werkzeug" }, { name = "xmlsec" }, @@ -4641,6 +4643,7 @@ requires-dist = [ { name = "sentry-sdk", extras = ["django"], specifier = "==2.56.0" }, { name = "sqlparse", specifier = "==0.5.5" }, { name = "uuid6", specifier = "==2024.7.10" }, + { name = "uvicorn-worker", specifier = "==0.4.0" }, { name = "uvloop", specifier = "==0.22.1" }, { name = "werkzeug", specifier = "==3.1.7" }, { name = "xmlsec", specifier = "==1.3.17" }, @@ -5846,6 +5849,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/3e/4ae6af487ce5781ed71d5fe10aca72e7cbc4d4f45afc31b120287082a8dd/uuid6-2024.7.10-py3-none-any.whl", hash = "sha256:93432c00ba403751f722829ad21759ff9db051dea140bf81493271e8e4dd18b7", size = 6376, upload-time = "2024-07-10T16:39:36.148Z" }, ] +[[package]] +name = "uvicorn" +version = "0.49.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", size = 91284, upload-time = "2026-06-03T22:01:30.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", size = 71376, upload-time = "2026-06-03T22:01:29.037Z" }, +] + +[[package]] +name = "uvicorn-worker" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gunicorn" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/59/9101b9c0680fd80e9d26c07deb822a5d18a324339fcf9cd017885ee808ad/uvicorn_worker-0.4.0.tar.gz", hash = "sha256:8ee5306070d8f38dce124adce488c3c0b50f20cf0c0222b12c66188da7214493", size = 9361, upload-time = "2025-09-20T10:47:01.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/25/09cd7a90c8bb7fb693be0d6704fccd5f9778d5513214b7a01cc4a94ff314/uvicorn_worker-0.4.0-py3-none-any.whl", hash = "sha256:e2ed952cef976f5e9e429d7269640bbcafbd36c80aa80f1003c8c77a6797abde", size = 5364, upload-time = "2025-09-20T10:46:59.776Z" }, +] + [[package]] name = "uvloop" version = "0.22.1"