mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-10-30 23:34:27 +00:00
feat(api): make Attack Paths sink selectable between Neo4j and Neptune (#11524)
This commit is contained in:
1 parent
9b8b77cec0
commit
5793cd7e38
48 files changed
+9928
-3210
No files matched your search
@@ -311,6 +311,11 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
|
||||
) # 48h
|
||||
|
||||
# Selects where the persistent attack-paths graph is stored. The scan
|
||||
# temporary database is always Neo4j; only the sink is configurable.
|
||||
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
|
||||
ATTACK_PATHS_SINK_DATABASE = env.str("ATTACK_PATHS_SINK_DATABASE", default="neo4j")
|
||||
|
||||
# Orphan task recovery feature flags. The master switch is OFF by default, so task
|
||||
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
|
||||
# toggles default to enabled, so once the master is on every group recovers unless a
|
||||
|
||||
@@ -50,6 +50,12 @@ DATABASES = {
|
||||
"USER": env.str("NEO4J_USER", "neo4j"),
|
||||
"PASSWORD": env.str("NEO4J_PASSWORD", "neo4j_password"),
|
||||
},
|
||||
"neptune": {
|
||||
"WRITER_ENDPOINT": env.str("NEPTUNE_WRITER_ENDPOINT", ""),
|
||||
"READER_ENDPOINT": env.str("NEPTUNE_READER_ENDPOINT", ""),
|
||||
"PORT": env.str("NEPTUNE_PORT", "8182"),
|
||||
"REGION": env.str("AWS_REGION", ""),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -49,12 +49,19 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
# TODO: drop after Neptune cutover just loosen defaults to `""`
|
||||
"neo4j": {
|
||||
"HOST": env.str("NEO4J_HOST"),
|
||||
"PORT": env.str("NEO4J_PORT"),
|
||||
"USER": env.str("NEO4J_USER"),
|
||||
"PASSWORD": env.str("NEO4J_PASSWORD"),
|
||||
},
|
||||
"neptune": {
|
||||
"WRITER_ENDPOINT": env.str("NEPTUNE_WRITER_ENDPOINT", default=""),
|
||||
"READER_ENDPOINT": env.str("NEPTUNE_READER_ENDPOINT", default=""),
|
||||
"PORT": env.str("NEPTUNE_PORT", default="8182"),
|
||||
"REGION": env.str("AWS_REGION", default=""),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -83,12 +83,28 @@ def _warm_compliance_caches_in_background():
|
||||
|
||||
|
||||
def post_fork(_server, worker):
|
||||
"""Warm compliance caches after each worker fork.
|
||||
"""Re-initialize attack-paths drivers and warm compliance caches per worker.
|
||||
|
||||
Warm compliance caches in a background thread so the worker becomes ready
|
||||
immediately. A request for a not-yet-warmed provider lazily loads just that
|
||||
provider, which stays well under the worker timeout.
|
||||
Neo4j / Neptune drivers spawn background IO threads that do not survive
|
||||
``fork()``. When the gunicorn master runs with ``preload_app=True``, the
|
||||
child inherits driver objects whose pool references dead threads and
|
||||
hangs on the first ``pool.acquire`` call until the watchdog kills the
|
||||
worker. Re-initializing per worker guarantees each child owns its own
|
||||
live threads. See GUNICORN_WORKER_TIMEOUTS_ANALYSIS.md for detail.
|
||||
|
||||
Compliance caches are then warmed in a background thread so the worker
|
||||
becomes ready immediately. A request for a not-yet-warmed provider lazily
|
||||
loads just that provider, which stays well under the worker timeout.
|
||||
"""
|
||||
from api.attack_paths import database as graph_database
|
||||
|
||||
try:
|
||||
graph_database.close_driver()
|
||||
except Exception: # pragma: no cover - best-effort cleanup
|
||||
pass
|
||||
graph_database.init_driver()
|
||||
gunicorn_logger.info(f"Attack-paths drivers initialized for worker {worker.pid}")
|
||||
|
||||
threading.Thread(
|
||||
target=_warm_compliance_caches_in_background,
|
||||
name="warm-compliance-caches",
|
||||
|
||||
Reference in new issue
Block a user