feat(api): make Attack Paths sink selectable between Neo4j and Neptune (#11524)

This commit is contained in:
Josema Camacho authored and GitHub committed 2026-06-26 10:22:29 +02:00
1 parent 9b8b77cec0
commit 5793cd7e38
48 files changed
+9928 -3210

No files matched your search

+5
View File
@@ -311,6 +311,11 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
) # 48h
# Selects where the persistent attack-paths graph is stored. The scan
# temporary database is always Neo4j; only the sink is configurable.
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
ATTACK_PATHS_SINK_DATABASE = env.str("ATTACK_PATHS_SINK_DATABASE", default="neo4j")
# Orphan task recovery feature flags. The master switch is OFF by default, so task
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
# toggles default to enabled, so once the master is on every group recovers unless a
+6
View File
@@ -50,6 +50,12 @@ DATABASES = {
"USER": env.str("NEO4J_USER", "neo4j"),
"PASSWORD": env.str("NEO4J_PASSWORD", "neo4j_password"),
},
"neptune": {
"WRITER_ENDPOINT": env.str("NEPTUNE_WRITER_ENDPOINT", ""),
"READER_ENDPOINT": env.str("NEPTUNE_READER_ENDPOINT", ""),
"PORT": env.str("NEPTUNE_PORT", "8182"),
"REGION": env.str("AWS_REGION", ""),
},
}
DATABASES["default"] = DATABASES["prowler_user"]
@@ -49,12 +49,19 @@ DATABASES = {
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
},
# TODO: drop after Neptune cutover just loosen defaults to `""`
"neo4j": {
"HOST": env.str("NEO4J_HOST"),
"PORT": env.str("NEO4J_PORT"),
"USER": env.str("NEO4J_USER"),
"PASSWORD": env.str("NEO4J_PASSWORD"),
},
"neptune": {
"WRITER_ENDPOINT": env.str("NEPTUNE_WRITER_ENDPOINT", default=""),
"READER_ENDPOINT": env.str("NEPTUNE_READER_ENDPOINT", default=""),
"PORT": env.str("NEPTUNE_PORT", default="8182"),
"REGION": env.str("AWS_REGION", default=""),
},
}
DATABASES["default"] = DATABASES["prowler_user"]
+20 -4
View File
@@ -83,12 +83,28 @@ def _warm_compliance_caches_in_background():
def post_fork(_server, worker):
"""Warm compliance caches after each worker fork.
"""Re-initialize attack-paths drivers and warm compliance caches per worker.
Warm compliance caches in a background thread so the worker becomes ready
immediately. A request for a not-yet-warmed provider lazily loads just that
provider, which stays well under the worker timeout.
Neo4j / Neptune drivers spawn background IO threads that do not survive
``fork()``. When the gunicorn master runs with ``preload_app=True``, the
child inherits driver objects whose pool references dead threads and
hangs on the first ``pool.acquire`` call until the watchdog kills the
worker. Re-initializing per worker guarantees each child owns its own
live threads. See GUNICORN_WORKER_TIMEOUTS_ANALYSIS.md for detail.
Compliance caches are then warmed in a background thread so the worker
becomes ready immediately. A request for a not-yet-warmed provider lazily
loads just that provider, which stays well under the worker timeout.
"""
from api.attack_paths import database as graph_database
try:
graph_database.close_driver()
except Exception: # pragma: no cover - best-effort cleanup
pass
graph_database.init_driver()
gunicorn_logger.info(f"Attack-paths drivers initialized for worker {worker.pid}")
threading.Thread(
target=_warm_compliance_caches_in_background,
name="warm-compliance-caches",