diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index cfaecfa2f2..9c3f278f79 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -10,6 +10,14 @@ All notable changes to the **Prowler API** are documented in this file. --- +## [1.20.1] (Prowler UNRELEASED) + +### 🐞 Fixed + +- Attack Paths: Add missing logging for query execution and exception details in scan error handling [(#10269)](https://github.com/prowler-cloud/prowler/pull/10269) + +--- + ## [1.20.0] (Prowler v5.19.0) ### 🚀 Added diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index b23c589262..75b03e2623 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -3,6 +3,7 @@ import glob import json import logging import os +import time from collections import defaultdict from copy import deepcopy @@ -2570,14 +2571,35 @@ class AttackPathsScanViewSet(BaseRLSViewSet): provider_id, ) + start = time.monotonic() graph = attack_paths_views_helpers.execute_query( database_name, query_definition, parameters, provider_id, ) + query_duration = time.monotonic() - start graph_database.clear_cache(database_name) + result_nodes = len(graph.get("nodes", [])) + result_relationships = len(graph.get("relationships", [])) + logger.info( + "attack_paths_query_run", + extra={ + "user_id": str(request.user.id), + "tenant_id": str(attack_paths_scan.provider.tenant_id), + "metadata": { + "query_id": query_definition.id, + "provider": query_definition.provider, + "scan_id": pk, + "provider_id": provider_id, + "result_nodes": result_nodes, + "result_relationships": result_relationships, + "query_duration": round(query_duration, 3), + }, + }, + ) + status_code = status.HTTP_200_OK if not graph.get("nodes"): status_code = status.HTTP_404_NOT_FOUND @@ -2618,13 +2640,35 @@ class AttackPathsScanViewSet(BaseRLSViewSet): ) provider_id = str(attack_paths_scan.provider_id) + start = time.monotonic() graph = attack_paths_views_helpers.execute_custom_query( database_name, serializer.validated_data["query"], provider_id, ) + query_duration = time.monotonic() - start graph_database.clear_cache(database_name) + query_length = len(serializer.validated_data["query"]) + result_nodes = len(graph.get("nodes", [])) + result_relationships = len(graph.get("relationships", [])) + logger.info( + "attack_paths_custom_query_run", + extra={ + "user_id": str(request.user.id), + "tenant_id": str(attack_paths_scan.provider.tenant_id), + "metadata": { + "provider": attack_paths_scan.provider.provider, + "scan_id": pk, + "provider_id": provider_id, + "query_length": query_length, + "result_nodes": result_nodes, + "result_relationships": result_relationships, + "query_duration": round(query_duration, 3), + }, + }, + ) + status_code = status.HTTP_200_OK if not graph.get("nodes"): status_code = status.HTTP_404_NOT_FOUND diff --git a/api/src/backend/config/custom_logging.py b/api/src/backend/config/custom_logging.py index fb04930679..fe2a090ca6 100644 --- a/api/src/backend/config/custom_logging.py +++ b/api/src/backend/config/custom_logging.py @@ -2,6 +2,7 @@ import json import logging from enum import StrEnum + from config.env import env from django_guid.log_filters import CorrelationId @@ -62,6 +63,8 @@ class NDJSONFormatter(logging.Formatter): log_record["duration"] = record.duration if hasattr(record, "status_code"): log_record["status_code"] = record.status_code + if hasattr(record, "metadata"): + log_record["metadata"] = record.metadata if record.exc_info: log_record["exc_info"] = self.formatException(record.exc_info) @@ -107,6 +110,8 @@ class HumanReadableFormatter(logging.Formatter): log_components.append(f"done in {record.duration}s:") if hasattr(record, "status_code"): log_components.append(f"{record.status_code}") + if hasattr(record, "metadata"): + log_components.append(f"metadata={record.metadata}") if record.exc_info: log_components.append(self.formatException(record.exc_info)) diff --git a/api/src/backend/tasks/jobs/attack_paths/aws.py b/api/src/backend/tasks/jobs/attack_paths/aws.py index 9242946181..160ad6ee3a 100644 --- a/api/src/backend/tasks/jobs/attack_paths/aws.py +++ b/api/src/backend/tasks/jobs/attack_paths/aws.py @@ -239,8 +239,9 @@ def sync_aws_account( failed_syncs[func_name] = exception_message logger.warning( - f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}. We " - "are continuing on to the next AWS sync function.", + f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}: {e}. " + "Continuing to the next AWS sync function.", + exc_info=True, ) continue diff --git a/api/src/backend/tasks/jobs/attack_paths/scan.py b/api/src/backend/tasks/jobs/attack_paths/scan.py index cd39700dd4..a7cf60b568 100644 --- a/api/src/backend/tasks/jobs/attack_paths/scan.py +++ b/api/src/backend/tasks/jobs/attack_paths/scan.py @@ -212,18 +212,20 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: try: graph_database.drop_database(tmp_cartography_config.neo4j_database) - except Exception: + except Exception as e: logger.error( - f"Failed to drop temporary Neo4j database {tmp_cartography_config.neo4j_database} during cleanup" + f"Failed to drop temporary Neo4j database {tmp_cartography_config.neo4j_database} during cleanup: {e}", + exc_info=True, ) try: db_utils.finish_attack_paths_scan( attack_paths_scan, StateChoices.FAILED, ingestion_exceptions ) - except Exception: - logger.warning( - f"Could not mark attack paths scan {attack_paths_scan.id} as FAILED (row may have been deleted)" + except Exception as e: + logger.error( + f"Could not mark attack paths scan {attack_paths_scan.id} as FAILED (row may have been deleted): {e}", + exc_info=True, ) raise