feat(overviews): Add API overview endpoints for findings and severity (#5910)

api/src/backend/tasks/jobs/scan.py

@@ -3,8 +3,7 @@ from copy import deepcopy
 from datetime import datetime, timezone
 
 from celery.utils.log import get_task_logger
-from prowler.lib.outputs.finding import Finding as ProwlerFinding
-from prowler.lib.scan.scan import Scan as ProwlerScan
+from django.db.models import Case, Count, IntegerField, Sum, When
 
 from api.compliance import (
     PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
@@ -12,17 +11,20 @@ from api.compliance import (
 )
 from api.db_utils import tenant_transaction
 from api.models import (
-    Provider,
-    Scan,
+    ComplianceOverview,
     Finding,
+    Provider,
     Resource,
     ResourceTag,
-    StatusChoices as FindingStatus,
+    Scan,
+    ScanSummary,
     StateChoices,
-    ComplianceOverview,
 )
+from api.models import StatusChoices as FindingStatus
 from api.utils import initialize_prowler_provider
 from api.v1.serializers import ScanTaskSerializer
+from prowler.lib.outputs.finding import Finding as ProwlerFinding
+from prowler.lib.scan.scan import Scan as ProwlerScan
 
 logger = get_task_logger(__name__)
 
@@ -268,7 +270,7 @@ def perform_prowler_scan(
             scan_instance.unique_resource_count = len(unique_resources)
             scan_instance.save()
 
-    if generate_compliance:
+    if exception is None and generate_compliance:
         try:
             regions = prowler_provider.get_regions()
         except AttributeError:
@@ -321,3 +323,155 @@ def perform_prowler_scan(
 
     serializer = ScanTaskSerializer(instance=scan_instance)
     return serializer.data
+
+
+def aggregate_findings(tenant_id: str, scan_id: str):
+    """
+    Aggregates findings for a given scan and stores the results in the ScanSummary table.
+
+    This function retrieves all findings associated with a given `scan_id` and calculates various
+    metrics such as counts of failed, passed, and muted findings, as well as their deltas (new,
+    changed, unchanged). The results are grouped by `check_id`, `service`, `severity`, and `region`.
+    These aggregated metrics are then stored in the `ScanSummary` table.
+
+    Args:
+        tenant_id (str): The ID of the tenant to which the scan belongs.
+        scan_id (str): The ID of the scan for which findings need to be aggregated.
+
+    Aggregated Metrics:
+        - fail: Total number of failed findings.
+        - _pass: Total number of passed findings.
+        - muted: Total number of muted findings.
+        - total: Total number of findings.
+        - new: Total number of new findings.
+        - changed: Total number of changed findings.
+        - unchanged: Total number of unchanged findings.
+        - fail_new: Failed findings with a delta of 'new'.
+        - fail_changed: Failed findings with a delta of 'changed'.
+        - pass_new: Passed findings with a delta of 'new'.
+        - pass_changed: Passed findings with a delta of 'changed'.
+        - muted_new: Muted findings with a delta of 'new'.
+        - muted_changed: Muted findings with a delta of 'changed'.
+    """
+    with tenant_transaction(tenant_id):
+        findings = Finding.objects.filter(scan_id=scan_id)
+
+        aggregation = findings.values(
+            "check_id",
+            "resources__service",
+            "severity",
+            "resources__region",
+        ).annotate(
+            fail=Sum(
+                Case(
+                    When(status="FAIL", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            _pass=Sum(
+                Case(
+                    When(status="PASS", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            muted=Sum(
+                Case(
+                    When(status="MUTED", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            total=Count("id"),
+            new=Sum(
+                Case(
+                    When(delta="new", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            changed=Sum(
+                Case(
+                    When(delta="changed", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            unchanged=Sum(
+                Case(
+                    When(delta__isnull=True, then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            fail_new=Sum(
+                Case(
+                    When(delta="new", status="FAIL", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            fail_changed=Sum(
+                Case(
+                    When(delta="changed", status="FAIL", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            pass_new=Sum(
+                Case(
+                    When(delta="new", status="PASS", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            pass_changed=Sum(
+                Case(
+                    When(delta="changed", status="PASS", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            muted_new=Sum(
+                Case(
+                    When(delta="new", status="MUTED", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+            muted_changed=Sum(
+                Case(
+                    When(delta="changed", status="MUTED", then=1),
+                    default=0,
+                    output_field=IntegerField(),
+                )
+            ),
+        )
+
+    with tenant_transaction(tenant_id):
+        scan_aggregations = {
+            ScanSummary(
+                tenant_id=tenant_id,
+                scan_id=scan_id,
+                check_id=agg["check_id"],
+                service=agg["resources__service"],
+                severity=agg["severity"],
+                region=agg["resources__region"],
+                fail=agg["fail"],
+                _pass=agg["_pass"],
+                muted=agg["muted"],
+                total=agg["total"],
+                new=agg["new"],
+                changed=agg["changed"],
+                unchanged=agg["unchanged"],
+                fail_new=agg["fail_new"],
+                fail_changed=agg["fail_changed"],
+                pass_new=agg["pass_new"],
+                pass_changed=agg["pass_changed"],
+                muted_new=agg["muted_new"],
+                muted_changed=agg["muted_changed"],
+            )
+            for agg in aggregation
+        }
+        ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000)

api/src/backend/tasks/tasks.py

@@ -1,12 +1,12 @@
 from celery import shared_task
+from config.celery import RLSTask
+from tasks.jobs.connection import check_provider_connection
+from tasks.jobs.deletion import delete_instance
+from tasks.jobs.scan import aggregate_findings, perform_prowler_scan
 
 from api.db_utils import tenant_transaction
 from api.decorators import set_tenant
 from api.models import Provider, Scan
-from config.celery import RLSTask
-from tasks.jobs.connection import check_provider_connection
-from tasks.jobs.deletion import delete_instance
-from tasks.jobs.scan import perform_prowler_scan
 
 
 @shared_task(base=RLSTask, name="provider-connection-check")
@@ -110,3 +110,8 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
         scan_id=str(scan_instance.id),
         provider_id=provider_id,
     )
+
+
+@shared_task(name="scan-summary")
+def perform_scan_summary_task(tenant_id: str, scan_id: str):
+    return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)

api/src/backend/tasks/tests/test_scan.py

@@ -1,19 +1,19 @@
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
-
-from api.models import (
-    StateChoices,
-    Severity,
-    Finding,
-    Resource,
-    StatusChoices,
-    Provider,
-)
 from tasks.jobs.scan import (
-    perform_prowler_scan,
     _create_finding_delta,
     _store_resources,
+    perform_prowler_scan,
+)
+
+from api.models import (
+    Finding,
+    Provider,
+    Resource,
+    Severity,
+    StateChoices,
+    StatusChoices,
 )
 
 
@@ -358,3 +358,6 @@ class TestPerformScan:
 
         assert resource == resource_instance
         assert resource_uid_tuple == (resource_instance.uid, resource_instance.region)
+
+
+# TODO Add tests for aggregations