perf(api): speed up finding-groups endpoint for finding-level filters (#11326)

api/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to the **Prowler API** are documented in this file.
 
+## [1.29.1] (Prowler UNRELEASED)
+
+### 🐞 Fixed
+
+- `finding-groups` slow response with finding-level filters such as `region`; check title and description are now read from the daily summaries, which drops sorting by `check_title` (not used by the UI) [(#11326)](https://github.com/prowler-cloud/prowler/pull/11326)
+
+---
+
 ## [1.29.0] (Prowler v5.28.0)
 
 ### 🚀 Added
@@ -29,7 +37,6 @@ All notable changes to the **Prowler API** are documented in this file.
 - `perform_scan_task` and `perform_scheduled_scan_task` now short-circuit with a warning and `return None` when the target provider no longer exists, instead of letting `handle_provider_deletion` raise `ProviderDeletedException`. `perform_scheduled_scan_task` also removes any orphan `PeriodicTask` it finds so beat stops re-firing scans for deleted providers. Prevents queued messages for deleted providers from being recorded as `FAILURE` [(#11185)](https://github.com/prowler-cloud/prowler/pull/11185)
 - Attack Paths: `BEDROCK-001` and `BEDROCK-002` now target roles trusting `bedrock-agentcore.amazonaws.com` instead of `bedrock.amazonaws.com`, eliminating false positives against regular Bedrock service roles (Agents, Knowledge Bases, model invocation) [(#11141)](https://github.com/prowler-cloud/prowler/pull/11141)
 
-
 ---
 
 ## [1.27.1] (Prowler v5.26.1)

api/src/backend/api/tests/test_views.py

@@ -15921,6 +15921,12 @@ class TestFindingGroupViewSet:
         assert attrs["fail_count"] == 0
         assert attrs["resources_total"] == 1
         assert attrs["resources_fail"] == 0
+        # check_title / check_description are resolved post-pagination from the
+        # summary table, not from the finding's check_metadata.
+        assert attrs["check_title"] == "Ensure EC2 instances do not have public IPs"
+        assert (
+            attrs["check_description"] == "EC2 instances should use private IPs only."
+        )
 
     def test_finding_groups_status_pass_when_no_fail(
         self, authenticated_client, finding_groups_fixture
@@ -17162,6 +17168,12 @@ class TestFindingGroupViewSet:
         assert attrs["fail_count"] == 0
         assert attrs["resources_total"] == 1
         assert attrs["resources_fail"] == 0
+        # check_title / check_description are resolved post-pagination from the
+        # summary table, not from the finding's check_metadata.
+        assert attrs["check_title"] == "Ensure EC2 instances do not have public IPs"
+        assert (
+            attrs["check_description"] == "EC2 instances should use private IPs only."
+        )
 
     def test_finding_groups_latest_status_in_filter(
         self, authenticated_client, finding_groups_fixture
@@ -17419,18 +17431,20 @@ class TestFindingGroupViewSet:
         check_ids = [item["id"] for item in data]
         assert check_ids == sorted(check_ids)
 
-    def test_finding_groups_latest_sort_by_check_title(
+    def test_finding_groups_latest_sort_by_check_title_not_supported(
         self, authenticated_client, finding_groups_fixture
     ):
-        """Test /latest supports sorting by check_title."""
+        """check_title is not a sortable field for finding groups.
+
+        Titles live in the TOASTed check_metadata blob and are resolved after
+        pagination from the summary table, so they cannot drive DB-level
+        ordering. Requesting that sort is rejected.
+        """
         response = authenticated_client.get(
             reverse("finding-group-latest"),
             {"sort": "check_title"},
         )
-        assert response.status_code == status.HTTP_200_OK
-        data = response.json()["data"]
-        check_titles = [item["attributes"]["check_title"] for item in data]
-        assert check_titles == sorted(check_titles)
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
 
     @pytest.mark.parametrize(
         "endpoint_name", ["finding-group-list", "finding-group-latest"]

api/src/backend/api/v1/views.py

@@ -7369,6 +7369,15 @@ class FindingGroupViewSet(BaseRLSViewSet):
             output_field=IntegerField(),
         )
 
+        # `check_title` / `check_description` are intentionally NOT resolved
+        # here. They live in the large JSONB `check_metadata` blob (TOASTed),
+        # so reading them per finding row is very expensive, and pulling them
+        # in via a correlated subquery makes Django add the subquery to GROUP
+        # BY, which re-evaluates it once per input row. They are identical for
+        # every finding of a `check_id`, so `_post_process_aggregation` fills
+        # them from the summary table's plain columns in a single batched
+        # lookup scoped to the paginated page.
+
         # `pass_count`, `fail_count` and `manual_count` only count non-muted
         # findings. Muted findings are tracked separately via the
         # `*_muted_count` fields.
@@ -7439,15 +7448,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
                 agg_failing_since=Min(
                     "first_seen_at", filter=Q(status="FAIL", muted=False)
                 ),
-                check_title=Coalesce(
-                    Max(KeyTextTransform("checktitle", "check_metadata")),
-                    Max(KeyTextTransform("CheckTitle", "check_metadata")),
-                    Max(KeyTextTransform("Checktitle", "check_metadata")),
-                ),
-                check_description=Coalesce(
-                    Max(KeyTextTransform("description", "check_metadata")),
-                    Max(KeyTextTransform("Description", "check_metadata")),
-                ),
             )
             .annotate(
                 # Group is muted only if it has zero non-muted findings.
@@ -7503,9 +7503,38 @@ class FindingGroupViewSet(BaseRLSViewSet):
         - Computes aggregated status (FAIL > PASS > MANUAL); the orthogonal
           ``muted`` boolean is already on the row from the SQL aggregation
         - Converts provider string to list
+        - Fills check_title / check_description for the findings path
         """
+        rows = list(aggregated_data)
+
+        # The findings-aggregation path omits check_title / check_description
+        # (they sit in TOASTed JSONB; see _aggregate_findings). Fill them from
+        # the summary table's plain columns in one query scoped to this page.
+        # The summary-aggregation path already carries them, so skip it there.
+        if rows and "check_title" not in rows[0]:
+            check_ids = [row["check_id"] for row in rows]
+            role = get_role(self.request.user, self.request.tenant_id)
+            summaries = FindingGroupDailySummary.objects.filter(
+                tenant_id=self.request.tenant_id,
+                check_id__in=check_ids,
+            )
+            # Scope to the user's providers, mirroring get_queryset(), so titles
+            # are read only from providers the user can see.
+            if not role.unlimited_visibility:
+                summaries = summaries.filter(provider__in=get_providers(role))
+            metadata_by_check = {
+                item["check_id"]: item
+                for item in summaries.order_by("check_id", "-inserted_at")
+                .distinct("check_id")
+                .values("check_id", "check_title", "check_description")
+            }
+            for row in rows:
+                metadata = metadata_by_check.get(row["check_id"], {})
+                row["check_title"] = metadata.get("check_title")
+                row["check_description"] = metadata.get("check_description")
+
         results = []
-        for row in aggregated_data:
+        for row in rows:
             # Convert severity order back to string
             severity_order = row.get("severity_order", 1)
             row["severity"] = SEVERITY_ORDER_REVERSE.get(
@@ -7551,7 +7580,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
 
     _FINDING_GROUP_SORT_MAP = {
         "check_id": "check_id",
-        "check_title": "check_title",
         "severity": "severity_order",
         "status": "status_order",
         "muted": "muted",