fix(api): add missing logging for Attack Paths query execution and scan error handling (#10297)

Co-authored-by: Josema Camacho <josema@prowler.com>

api/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to the **Prowler API** are documented in this file.
 
+## [1.21.0] (Prowler UNRELEASED)
+
+### 🔄 Changed
+
+- `POST /api/v1/providers` returns `409 Conflict` if already exists [(#10293)](https://github.com/prowler-cloud/prowler/pull/10293)
+
+---
+
+## [1.20.1] (Prowler UNRELEASED)
+
+### 🐞 Fixed
+
+- Attack Paths: Add missing logging for query execution and exception details in scan error handling [(#10269)](https://github.com/prowler-cloud/prowler/pull/10269)
+
+---
+
 ## [1.20.0] (Prowler v5.19.0)
 
 ### 🚀 Added

api/src/backend/api/v1/views.py

@@ -3,6 +3,7 @@ import glob
 import json
 import logging
 import os
+import time
 
 from collections import defaultdict
 from copy import deepcopy
@@ -2570,14 +2571,35 @@ class AttackPathsScanViewSet(BaseRLSViewSet):
             provider_id,
         )
 
+        start = time.monotonic()
         graph = attack_paths_views_helpers.execute_query(
             database_name,
             query_definition,
             parameters,
             provider_id,
         )
+        query_duration = time.monotonic() - start
         graph_database.clear_cache(database_name)
 
+        result_nodes = len(graph.get("nodes", []))
+        result_relationships = len(graph.get("relationships", []))
+        logger.info(
+            "attack_paths_query_run",
+            extra={
+                "user_id": str(request.user.id),
+                "tenant_id": str(attack_paths_scan.provider.tenant_id),
+                "metadata": {
+                    "query_id": query_definition.id,
+                    "provider": query_definition.provider,
+                    "scan_id": pk,
+                    "provider_id": provider_id,
+                    "result_nodes": result_nodes,
+                    "result_relationships": result_relationships,
+                    "query_duration": round(query_duration, 3),
+                },
+            },
+        )
+
         status_code = status.HTTP_200_OK
         if not graph.get("nodes"):
             status_code = status.HTTP_404_NOT_FOUND
@@ -2618,13 +2640,35 @@ class AttackPathsScanViewSet(BaseRLSViewSet):
         )
         provider_id = str(attack_paths_scan.provider_id)
 
+        start = time.monotonic()
         graph = attack_paths_views_helpers.execute_custom_query(
             database_name,
             serializer.validated_data["query"],
             provider_id,
         )
+        query_duration = time.monotonic() - start
         graph_database.clear_cache(database_name)
 
+        query_length = len(serializer.validated_data["query"])
+        result_nodes = len(graph.get("nodes", []))
+        result_relationships = len(graph.get("relationships", []))
+        logger.info(
+            "attack_paths_custom_query_run",
+            extra={
+                "user_id": str(request.user.id),
+                "tenant_id": str(attack_paths_scan.provider.tenant_id),
+                "metadata": {
+                    "provider": attack_paths_scan.provider.provider,
+                    "scan_id": pk,
+                    "provider_id": provider_id,
+                    "query_length": query_length,
+                    "result_nodes": result_nodes,
+                    "result_relationships": result_relationships,
+                    "query_duration": round(query_duration, 3),
+                },
+            },
+        )
+
         status_code = status.HTTP_200_OK
         if not graph.get("nodes"):
             status_code = status.HTTP_404_NOT_FOUND

api/src/backend/config/custom_logging.py

@@ -2,6 +2,7 @@ import json
 import logging
 from enum import StrEnum
 
+
 from config.env import env
 from django_guid.log_filters import CorrelationId
 
@@ -62,6 +63,8 @@ class NDJSONFormatter(logging.Formatter):
             log_record["duration"] = record.duration
         if hasattr(record, "status_code"):
             log_record["status_code"] = record.status_code
+        if hasattr(record, "metadata"):
+            log_record["metadata"] = record.metadata
 
         if record.exc_info:
             log_record["exc_info"] = self.formatException(record.exc_info)
@@ -107,6 +110,8 @@ class HumanReadableFormatter(logging.Formatter):
             log_components.append(f"done in {record.duration}s:")
         if hasattr(record, "status_code"):
             log_components.append(f"{record.status_code}")
+        if hasattr(record, "metadata"):
+            log_components.append(f"metadata={record.metadata}")
 
         if record.exc_info:
             log_components.append(self.formatException(record.exc_info))

api/src/backend/tasks/jobs/attack_paths/aws.py

@@ -239,8 +239,9 @@ def sync_aws_account(
                 failed_syncs[func_name] = exception_message
 
                 logger.warning(
-                    f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}. We "
-                    "are continuing on to the next AWS sync function.",
+                    f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}: {e}. "
+                    "Continuing to the next AWS sync function.",
+                    exc_info=True,
                 )
 
                 continue

api/src/backend/tasks/jobs/attack_paths/scan.py

@@ -212,18 +212,20 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
         try:
             graph_database.drop_database(tmp_cartography_config.neo4j_database)
 
-        except Exception:
+        except Exception as e:
             logger.error(
-                f"Failed to drop temporary Neo4j database {tmp_cartography_config.neo4j_database} during cleanup"
+                f"Failed to drop temporary Neo4j database {tmp_cartography_config.neo4j_database} during cleanup: {e}",
+                exc_info=True,
             )
 
         try:
             db_utils.finish_attack_paths_scan(
                 attack_paths_scan, StateChoices.FAILED, ingestion_exceptions
             )
-        except Exception:
-            logger.warning(
-                f"Could not mark attack paths scan {attack_paths_scan.id} as FAILED (row may have been deleted)"
+        except Exception as e:
+            logger.error(
+                f"Could not mark attack paths scan {attack_paths_scan.id} as FAILED (row may have been deleted): {e}",
+                exc_info=True,
             )
 
         raise