feat(api): add resource group overview endpoint and filtering (#9694)

Co-authored-by: Alan Buscaglia <gentlemanprogramming@gmail.com>
Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>

api/src/backend/tasks/jobs/backfill.py

@@ -2,13 +2,13 @@ from collections import defaultdict
 from datetime import timedelta
 
 from celery.utils.log import get_task_logger
-from django.db.models import Sum
+from django.db.models import OuterRef, Subquery, Sum
 from django.utils import timezone
 from tasks.jobs.queries import (
     COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
     COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
 )
-from tasks.jobs.scan import aggregate_category_counts
+from tasks.jobs.scan import aggregate_category_counts, aggregate_resource_group_counts
 
 from api.db_router import READ_REPLICA_ALIAS, MainRouter
 from api.db_utils import (
@@ -28,6 +28,7 @@ from api.models import (
     ResourceScanSummary,
     Scan,
     ScanCategorySummary,
+    ScanGroupSummary,
     ScanSummary,
     StateChoices,
 )
@@ -356,6 +357,92 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
     return {"status": "backfilled", "categories_count": len(category_counts)}
 
 
+def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
+    """
+    Backfill ScanGroupSummary for a completed scan.
+
+    Aggregates resource group counts from all findings in the scan and creates
+    one ScanGroupSummary row per (resource_group, severity) combination.
+
+    Args:
+        tenant_id: Target tenant UUID
+        scan_id: Scan UUID to backfill
+
+    Returns:
+        dict: Status indicating whether backfill was performed
+    """
+    with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
+        if ScanGroupSummary.objects.filter(
+            tenant_id=tenant_id, scan_id=scan_id
+        ).exists():
+            return {"status": "already backfilled"}
+
+        if not Scan.objects.filter(
+            tenant_id=tenant_id,
+            id=scan_id,
+            state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
+        ).exists():
+            return {"status": "scan is not completed"}
+
+        resource_group_counts: dict[tuple[str, str], dict[str, int]] = {}
+        group_resources_cache: dict[str, set] = {}
+        # Get findings with their first resource UID via annotation
+        resource_uid_subquery = ResourceFindingMapping.objects.filter(
+            finding_id=OuterRef("id"), tenant_id=tenant_id
+        ).values("resource__uid")[:1]
+
+        for finding in (
+            Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
+            .annotate(resource_uid=Subquery(resource_uid_subquery))
+            .values(
+                "resource_groups",
+                "severity",
+                "status",
+                "delta",
+                "muted",
+                "resource_uid",
+            )
+        ):
+            aggregate_resource_group_counts(
+                resource_group=finding.get("resource_groups"),
+                severity=finding.get("severity"),
+                status=finding.get("status"),
+                delta=finding.get("delta"),
+                muted=finding.get("muted", False),
+                resource_uid=finding.get("resource_uid") or "",
+                cache=resource_group_counts,
+                group_resources_cache=group_resources_cache,
+            )
+
+        if not resource_group_counts:
+            return {"status": "no resource groups to backfill"}
+
+    # Compute group-level resource counts (same value for all severity rows in a group)
+    group_resource_counts = {
+        grp: len(uids) for grp, uids in group_resources_cache.items()
+    }
+    resource_group_summaries = [
+        ScanGroupSummary(
+            tenant_id=tenant_id,
+            scan_id=scan_id,
+            resource_group=grp,
+            severity=severity,
+            total_findings=counts["total"],
+            failed_findings=counts["failed"],
+            new_failed_findings=counts["new_failed"],
+            resources_count=group_resource_counts.get(grp, 0),
+        )
+        for (grp, severity), counts in resource_group_counts.items()
+    ]
+
+    with rls_transaction(tenant_id):
+        ScanGroupSummary.objects.bulk_create(
+            resource_group_summaries, batch_size=500, ignore_conflicts=True
+        )
+
+    return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)}
+
+
 def backfill_provider_compliance_scores(tenant_id: str) -> dict:
     """
     Backfill ProviderComplianceScore from latest completed scan per provider.

api/src/backend/tasks/jobs/scan.py

@@ -45,6 +45,7 @@ from api.models import (
     ResourceTag,
     Scan,
     ScanCategorySummary,
+    ScanGroupSummary,
     ScanSummary,
     StateChoices,
 )
@@ -127,6 +128,50 @@ def aggregate_category_counts(
             cache[key]["new_failed"] += 1
 
 
+def aggregate_resource_group_counts(
+    resource_group: str | None,
+    severity: str,
+    status: str,
+    delta: str | None,
+    muted: bool,
+    resource_uid: str,
+    cache: dict[tuple[str, str], dict[str, int]],
+    group_resources_cache: dict[str, set],
+) -> None:
+    """
+    Increment resource group counters in-place for a finding.
+
+    Args:
+        resource_group: Resource group from check metadata (e.g., "database", "compute").
+        severity: Severity level (e.g., "high", "medium").
+        status: Finding status as string ("FAIL", "PASS").
+        delta: Delta value as string ("new", "changed") or None.
+        muted: Whether the finding is muted.
+        resource_uid: Unique identifier for the resource to count distinct resources.
+        cache: Dict {(resource_group, severity): {"total", "failed", "new_failed"}} to update.
+        group_resources_cache: Dict {resource_group: set(resource_uids)} for group-level resource tracking.
+    """
+    if not resource_group:
+        return
+
+    is_failed = status == "FAIL" and not muted
+    is_new_failed = is_failed and delta == "new"
+
+    key = (resource_group, severity)
+    if key not in cache:
+        cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
+    if not muted:
+        cache[key]["total"] += 1
+    if is_failed:
+        cache[key]["failed"] += 1
+    if is_new_failed:
+        cache[key]["new_failed"] += 1
+
+    # Track resources at GROUP level (not per-severity) to avoid over-counting
+    if resource_uid and not muted:
+        group_resources_cache.setdefault(resource_group, set()).add(resource_uid)
+
+
 def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
     global _ATTACK_SURFACE_MAPPING_CACHE
 
@@ -438,6 +483,8 @@ def _process_finding_micro_batch(
     scan_resource_cache: set,
     mute_rules_cache: dict,
     scan_categories_cache: dict[tuple[str, str], dict[str, int]],
+    scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]],
+    group_resources_cache: dict[str, set],
 ) -> None:
     """
     Process a micro-batch of findings and persist them using bulk operations.
@@ -459,6 +506,8 @@ def _process_finding_micro_batch(
         scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
         mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
         scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
+        scan_resource_groups_cache: Dict tracking resource group counts {(resource_group, severity): {"total", "failed", "new_failed"}}.
+        group_resources_cache: Dict tracking unique resources per group {resource_group: set(resource_uids)}.
     """
     # Accumulate objects for bulk operations
     findings_to_create = []
@@ -499,6 +548,8 @@ def _process_finding_micro_batch(
                 with rls_transaction(tenant_id):
                     resource_uid = finding.resource_uid
                     if resource_uid not in resource_cache:
+                        check_metadata = finding.get_metadata()
+                        group = check_metadata.get("resourcegroup") or None
                         resource_instance, _ = Resource.objects.get_or_create(
                             tenant_id=tenant_id,
                             provider=provider_instance,
@@ -508,6 +559,7 @@ def _process_finding_micro_batch(
                                 "service": finding.service_name,
                                 "type": finding.resource_type,
                                 "name": finding.resource_name,
+                                "groups": [group] if group else None,
                             },
                         )
                         resource_cache[resource_uid] = resource_instance
@@ -528,6 +580,8 @@ def _process_finding_micro_batch(
 
         # Track resource field changes (defer save)
         updated = False
+        check_metadata = finding.get_metadata()
+        group = check_metadata.get("resourcegroup") or None
         if finding.region and resource_instance.region != finding.region:
             resource_instance.region = finding.region
             updated = True
@@ -548,6 +602,11 @@ def _process_finding_micro_batch(
         if resource_instance.partition != finding.partition:
             resource_instance.partition = finding.partition
             updated = True
+        if group and (
+            not resource_instance.groups or group not in resource_instance.groups
+        ):
+            resource_instance.groups = (resource_instance.groups or []) + [group]
+            updated = True
 
         if updated:
             dirty_resources[resource_uid] = resource_instance
@@ -633,6 +692,7 @@ def _process_finding_micro_batch(
             muted_reason=muted_reason,
             compliance=finding.compliance,
             categories=check_metadata.get("categories", []) or [],
+            resource_groups=check_metadata.get("resourcegroup") or None,
         )
         findings_to_create.append(finding_instance)
         resource_denormalized_data.append((finding_instance, resource_instance))
@@ -657,6 +717,18 @@ def _process_finding_micro_batch(
             cache=scan_categories_cache,
         )
 
+        # Track resource groups with counts for ScanGroupSummary
+        aggregate_resource_group_counts(
+            resource_group=check_metadata.get("resourcegroup") or None,
+            severity=finding.severity.value,
+            status=status.value,
+            delta=delta.value if delta else None,
+            muted=is_muted,
+            resource_uid=resource_instance.uid if resource_instance else "",
+            cache=scan_resource_groups_cache,
+            group_resources_cache=group_resources_cache,
+        )
+
     # Bulk operations within single transaction
     with rls_transaction(tenant_id):
         # Bulk create findings
@@ -714,7 +786,15 @@ def _process_finding_micro_batch(
             tenant_id=tenant_id,
             model=Resource,
             objects=list(dirty_resources.values()),
-            fields=["metadata", "details", "partition", "region", "service", "type"],
+            fields=[
+                "metadata",
+                "details",
+                "partition",
+                "region",
+                "service",
+                "type",
+                "groups",
+            ],
             batch_size=1000,
         )
 
@@ -757,6 +837,8 @@ def perform_prowler_scan(
     unique_resources = set()
     scan_resource_cache: set[tuple[str, str, str, str]] = set()
     scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
+    scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
+    group_resources_cache: dict[str, set] = {}
     start_time = time.time()
     exc = None
 
@@ -847,6 +929,8 @@ def perform_prowler_scan(
                     scan_resource_cache=scan_resource_cache,
                     mute_rules_cache=mute_rules_cache,
                     scan_categories_cache=scan_categories_cache,
+                    scan_resource_groups_cache=scan_resource_groups_cache,
+                    group_resources_cache=group_resources_cache,
                 )
 
             # Update scan progress
@@ -933,6 +1017,38 @@ def perform_prowler_scan(
         sentry_sdk.capture_exception(cat_exception)
         logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
 
+    try:
+        if scan_resource_groups_cache:
+            # Compute group-level resource counts (same value for all severity rows in a group)
+            group_resource_counts = {
+                grp: len(uids) for grp, uids in group_resources_cache.items()
+            }
+            resource_group_summaries = [
+                ScanGroupSummary(
+                    tenant_id=tenant_id,
+                    scan_id=scan_id,
+                    resource_group=grp,
+                    severity=severity,
+                    total_findings=counts["total"],
+                    failed_findings=counts["failed"],
+                    new_failed_findings=counts["new_failed"],
+                    resources_count=group_resource_counts.get(grp, 0),
+                )
+                for (
+                    grp,
+                    severity,
+                ), counts in scan_resource_groups_cache.items()
+            ]
+            with rls_transaction(tenant_id):
+                ScanGroupSummary.objects.bulk_create(
+                    resource_group_summaries, batch_size=500, ignore_conflicts=True
+                )
+    except Exception as rg_exception:
+        sentry_sdk.capture_exception(rg_exception)
+        logger.error(
+            f"Error storing resource groups for scan {scan_id}: {rg_exception}"
+        )
+
     serializer = ScanTaskSerializer(instance=scan_instance)
     return serializer.data
 

api/src/backend/tasks/tasks.py

@@ -14,6 +14,7 @@ from tasks.jobs.backfill import (
     backfill_provider_compliance_scores,
     backfill_resource_scan_summaries,
     backfill_scan_category_summaries,
+    backfill_scan_resource_group_summaries,
 )
 from tasks.jobs.connection import (
     check_integration_connection,
@@ -613,6 +614,21 @@ def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
     return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
 
 
+@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill")
+@handle_provider_deletion
+def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
+    """
+    Backfill ScanGroupSummary for a completed scan.
+
+    Aggregates unique resource groups from findings and creates a summary row.
+
+    Args:
+        tenant_id (str): The tenant identifier.
+        scan_id (str): The scan identifier.
+    """
+    return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
+
+
 @shared_task(name="backfill-provider-compliance-scores", queue="backfill")
 def backfill_provider_compliance_scores_task(tenant_id: str):
     """

api/src/backend/tasks/tests/test_backfill.py

@@ -8,6 +8,7 @@ from tasks.jobs.backfill import (
     backfill_provider_compliance_scores,
     backfill_resource_scan_summaries,
     backfill_scan_category_summaries,
+    backfill_scan_resource_group_summaries,
 )
 
 from api.models import (
@@ -16,6 +17,7 @@ from api.models import (
     ResourceScanSummary,
     Scan,
     ScanCategorySummary,
+    ScanGroupSummary,
     StateChoices,
 )
 from prowler.lib.check.models import Severity
@@ -265,6 +267,94 @@ class TestBackfillScanCategorySummaries:
             assert summary.new_failed_findings == 1
 
 
+@pytest.fixture(scope="function")
+def findings_with_group_fixture(scans_fixture, resources_fixture):
+    scan = scans_fixture[0]
+    resource = resources_fixture[0]
+
+    finding = Finding.objects.create(
+        tenant_id=scan.tenant_id,
+        uid="finding_with_group",
+        scan=scan,
+        delta="new",
+        status=Status.FAIL,
+        status_extended="test status",
+        impact=Severity.high,
+        impact_extended="test impact",
+        severity=Severity.high,
+        raw_result={"status": Status.FAIL},
+        check_id="test_check",
+        check_metadata={"CheckId": "test_check"},
+        resource_groups="ai_ml",
+        first_seen_at="2024-01-02T00:00:00Z",
+    )
+    finding.add_resources([resource])
+    return finding
+
+
+@pytest.fixture(scope="function")
+def scan_resource_group_summary_fixture(scans_fixture):
+    scan = scans_fixture[0]
+    return ScanGroupSummary.objects.create(
+        tenant_id=scan.tenant_id,
+        scan=scan,
+        resource_group="existing-group",
+        severity=Severity.high,
+        total_findings=1,
+        failed_findings=0,
+        new_failed_findings=0,
+        resources_count=1,
+    )
+
+
+@pytest.mark.django_db
+class TestBackfillScanGroupSummaries:
+    def test_already_backfilled(self, scan_resource_group_summary_fixture):
+        tenant_id = scan_resource_group_summary_fixture.tenant_id
+        scan_id = scan_resource_group_summary_fixture.scan_id
+
+        result = backfill_scan_resource_group_summaries(str(tenant_id), str(scan_id))
+
+        assert result == {"status": "already backfilled"}
+
+    def test_not_completed_scan(self, get_not_completed_scans):
+        for scan in get_not_completed_scans:
+            result = backfill_scan_resource_group_summaries(
+                str(scan.tenant_id), str(scan.id)
+            )
+            assert result == {"status": "scan is not completed"}
+
+    def test_no_resource_groups_to_backfill(self, scans_fixture):
+        scan = scans_fixture[1]  # Failed scan with no findings
+        result = backfill_scan_resource_group_summaries(
+            str(scan.tenant_id), str(scan.id)
+        )
+        assert result == {"status": "no resource groups to backfill"}
+
+    def test_successful_backfill(self, findings_with_group_fixture):
+        finding = findings_with_group_fixture
+        tenant_id = str(finding.tenant_id)
+        scan_id = str(finding.scan_id)
+
+        result = backfill_scan_resource_group_summaries(tenant_id, scan_id)
+
+        # 1 resource group × 1 severity = 1 row
+        assert result == {"status": "backfilled", "resource_groups_count": 1}
+
+        summaries = ScanGroupSummary.objects.filter(
+            tenant_id=tenant_id, scan_id=scan_id
+        )
+        assert summaries.count() == 1
+
+        summary = summaries.first()
+        assert summary.resource_group == "ai_ml"
+        assert summary.severity == Severity.high
+        assert summary.total_findings == 1
+        assert summary.failed_findings == 1
+        assert summary.new_failed_findings == 1
+        assert summary.resources_count == 1
+
+
 @pytest.mark.django_db
 class TestBackfillProviderComplianceScores:
     def test_no_completed_scans(self, tenants_fixture):

api/src/backend/tasks/tests/test_scan.py

@@ -1380,6 +1380,8 @@ class TestProcessFindingMicroBatch:
         scan_resource_cache: set[tuple[str, str, str, str]] = set()
         mute_rules_cache = {}
         scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
+        scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
+        group_resources_cache: dict[str, set] = {}
 
         with (
             patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1398,6 +1400,8 @@ class TestProcessFindingMicroBatch:
                 scan_resource_cache,
                 mute_rules_cache,
                 scan_categories_cache,
+                scan_resource_groups_cache,
+                group_resources_cache,
             )
 
         created_finding = Finding.objects.get(uid=finding.uid)
@@ -1491,6 +1495,8 @@ class TestProcessFindingMicroBatch:
         scan_resource_cache: set[tuple[str, str, str, str]] = set()
         mute_rules_cache = {finding.uid: "Muted via rule"}
         scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
+        scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
+        group_resources_cache: dict[str, set] = {}
 
         with (
             patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1509,6 +1515,8 @@ class TestProcessFindingMicroBatch:
                 scan_resource_cache,
                 mute_rules_cache,
                 scan_categories_cache,
+                scan_resource_groups_cache,
+                group_resources_cache,
             )
 
         existing_resource.refresh_from_db()
@@ -1617,6 +1625,8 @@ class TestProcessFindingMicroBatch:
         scan_resource_cache: set[tuple[str, str, str, str]] = set()
         mute_rules_cache = {}
         scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
+        scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
+        group_resources_cache: dict[str, set] = {}
 
         with (
             patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1636,6 +1646,8 @@ class TestProcessFindingMicroBatch:
                 scan_resource_cache,
                 mute_rules_cache,
                 scan_categories_cache,
+                scan_resource_groups_cache,
+                group_resources_cache,
             )
 
         # Verify the long UID finding was NOT created
@@ -1713,6 +1725,8 @@ class TestProcessFindingMicroBatch:
         scan_resource_cache: set[tuple[str, str, str, str]] = set()
         mute_rules_cache = {}
         scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
+        scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
+        group_resources_cache: dict[str, set] = {}
 
         with (
             patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1731,6 +1745,8 @@ class TestProcessFindingMicroBatch:
                 scan_resource_cache,
                 mute_rules_cache,
                 scan_categories_cache,
+                scan_resource_groups_cache,
+                group_resources_cache,
             )
 
         # finding1: PASS, severity=low, categories=["gen-ai", "security"]