fix(api): log Celery worker failures and restart compose services (#12465)

Declare the Celery worker, kombu, billiard and amqp loggers so fatal worker errors are no longer silenced by disable_existing_loggers, and log task failures from celery.app.trace at WARNING. Add restart: unless-stopped to the long-running services in docker-compose.yml.

Refs #12461

api/changelog.d/worker-logging-restart-policy.fixed.md

@@ -0,0 +1 @@
+Celery loggers are now declared explicitly in `custom_logging.py` so fatal worker errors are no longer silenced by `disable_existing_loggers=True`. All long-running services in `docker-compose.yml` now have `restart: unless-stopped` so containers recover automatically after unexpected crashes.

api/src/backend/config/custom_logging.py

@@ -231,6 +231,62 @@ LOGGING = {
             "level": LEVEL,
             "propagate": False,
         },
+        # Celery loggers must be declared explicitly because
+        # disable_existing_loggers=True silences any logger that exists at
+        # dictConfig time but is not named here. Without these, fatal worker
+        # errors (e.g. celery.worker CRITICAL) produce no output.
+        # "celery" must keep propagating: get_task_logger() parents task
+        # loggers under celery.task, so blocking here hides them from root.
+        "celery": {
+            "level": LEVEL,
+            "propagate": True,
+        },
+        "celery.worker": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        "celery.worker.consumer": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        "celery.worker.consumer.consumer": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        "kombu": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        "kombu.transport.redis": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        "billiard": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        "amqp": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        # WARNING keeps task failures but skips one "succeeded" line per task.
+        "celery.app.trace": {
+            "handlers": ["tasks_console"],
+            "level": "WARNING",
+            "propagate": False,
+        },
+        "celery.beat": {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
     },
     # Gunicorn required configuration
     "root": {

docker-compose.yml

@@ -15,6 +15,7 @@ services:
   api:
     hostname: "prowler-api"
     image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
+    restart: unless-stopped
     env_file:
       - path: .env
         required: false
@@ -44,6 +45,7 @@ services:
 
   ui:
     image: prowlercloud/prowler-ui:${PROWLER_UI_VERSION:-stable}
+    restart: unless-stopped
     env_file:
       - path: .env
         required: false
@@ -61,6 +63,7 @@ services:
 
   postgres:
     image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
+    restart: unless-stopped
     hostname: "postgres-db"
     volumes:
       - ./_data/postgres:/var/lib/postgresql/data
@@ -81,6 +84,7 @@ services:
 
   valkey:
     image: valkey/valkey:8-alpine@sha256:a038175878d66b9d274fbf8be73c0305e93798b83917647f167e18cef3c71eec
+    restart: unless-stopped
     hostname: "valkey"
     volumes:
       - ./_data/valkey:/data
@@ -97,6 +101,7 @@ services:
 
   neo4j:
     image: graphstack/dozerdb:5.26.27.0@sha256:9b54d6b3a98a76c00bd23e8e78d8c82081ff168162aebd47b25c234e092cb0a0
+    restart: unless-stopped
     hostname: "neo4j"
     volumes:
       - ./_data/neo4j:/data
@@ -129,6 +134,7 @@ services:
 
   worker:
     image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
+    restart: unless-stopped
     # Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
     stop_grace_period: 120s
     env_file:
@@ -149,6 +155,7 @@ services:
 
   worker-beat:
     image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
+    restart: unless-stopped
     env_file:
       - path: ./.env
         required: false
@@ -165,6 +172,7 @@ services:
 
   mcp-server:
     image: prowlercloud/prowler-mcp:${PROWLER_MCP_VERSION:-stable}
+    restart: unless-stopped
     environment:
       - PROWLER_MCP_TRANSPORT_MODE=http
     env_file: