fix(scan-summaries): Improve efficiency on providers overview (#6716)

api/src/backend/api/migrations/0007_scan_and_scan_summaries_indexes.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.5 on 2025-01-28 15:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0006_findings_first_seen"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="scan",
+            index=models.Index(
+                fields=["tenant_id", "provider_id", "state", "inserted_at"],
+                name="scans_prov_state_insert_idx",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="scansummary",
+            constraint=models.Index(
+                fields=["tenant_id", "scan_id"], name="scan_summaries_tenant_scan_idx"
+            ),
+        ),
+    ]

api/src/backend/api/models.py

@@ -428,6 +428,10 @@ class Scan(RowLevelSecurityProtectedModel):
                 fields=["provider", "state", "trigger", "scheduled_at"],
                 name="scans_prov_state_trig_sche_idx",
             ),
+            models.Index(
+                fields=["tenant_id", "provider_id", "state", "inserted_at"],
+                name="scans_prov_state_insert_idx",
+            ),
         ]
 
     class JSONAPIMeta:
@@ -1094,6 +1098,10 @@ class ScanSummary(RowLevelSecurityProtectedModel):
                 fields=("tenant", "scan", "check_id", "service", "severity", "region"),
                 name="unique_scan_summary",
             ),
+            models.Index(
+                fields=["tenant_id", "scan_id"],
+                name="scan_summaries_tenant_scan_idx",
+            ),
             RowLevelSecurityConstraint(
                 field="tenant_id",
                 name="rls_on_%(class)s",

api/src/backend/api/specs/v1.yaml

@@ -714,8 +714,6 @@ paths:
           items:
             type: string
             enum:
-            - id
-            - -id
             - status
             - -status
             - severity
@@ -1242,8 +1240,6 @@ paths:
           items:
             type: string
             enum:
-            - id
-            - -id
             - status
             - -status
             - severity
@@ -1714,8 +1710,6 @@ paths:
           items:
             type: string
             enum:
-            - id
-            - -id
             - status
             - -status
             - severity
@@ -6753,7 +6747,7 @@ components:
                   type: integer
                 fail:
                   type: integer
-                manual:
+                muted:
                   type: integer
                 total:
                   type: integer

api/src/backend/api/tests/test_views.py

@@ -4280,18 +4280,15 @@ class TestOverviewViewSet:
         assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED
 
     def test_overview_providers_list(
-        self, authenticated_client, findings_fixture, resources_fixture
+        self, authenticated_client, scan_summaries_fixture, resources_fixture
     ):
         response = authenticated_client.get(reverse("overview-providers"))
         assert response.status_code == status.HTTP_200_OK
-        # Only findings from one provider
         assert len(response.json()["data"]) == 1
-        assert response.json()["data"][0]["attributes"]["findings"]["total"] == len(
-            findings_fixture
-        )
-        assert response.json()["data"][0]["attributes"]["findings"]["pass"] == 0
-        assert response.json()["data"][0]["attributes"]["findings"]["fail"] == 2
-        assert response.json()["data"][0]["attributes"]["findings"]["manual"] == 0
+        assert response.json()["data"][0]["attributes"]["findings"]["total"] == 4
+        assert response.json()["data"][0]["attributes"]["findings"]["pass"] == 2
+        assert response.json()["data"][0]["attributes"]["findings"]["fail"] == 1
+        assert response.json()["data"][0]["attributes"]["findings"]["muted"] == 1
         assert response.json()["data"][0]["attributes"]["resources"]["total"] == len(
             resources_fixture
         )

api/src/backend/api/v1/serializers.py

@@ -1735,7 +1735,7 @@ class OverviewProviderSerializer(serializers.Serializer):
             "properties": {
                 "pass": {"type": "integer"},
                 "fail": {"type": "integer"},
-                "manual": {"type": "integer"},
+                "muted": {"type": "integer"},
                 "total": {"type": "integer"},
             },
         }
@@ -1744,7 +1744,7 @@ class OverviewProviderSerializer(serializers.Serializer):
         return {
             "pass": obj["findings_passed"],
             "fail": obj["findings_failed"],
-            "manual": obj["findings_manual"],
+            "muted": obj["findings_muted"],
             "total": obj["total_findings"],
         }
 

api/src/backend/api/v1/views.py

@@ -4,7 +4,7 @@ from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.search import SearchQuery
 from django.db import transaction
 from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum
-from django.db.models.functions import JSONObject
+from django.db.models.functions import Coalesce, JSONObject
 from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_control
@@ -74,7 +74,6 @@ from api.models import (
     ScanSummary,
     SeverityChoices,
     StateChoices,
-    StatusChoices,
     Task,
     User,
     UserRoleRelationship,
@@ -1998,68 +1997,53 @@ class OverviewViewSet(BaseRLSViewSet):
     @action(detail=False, methods=["get"], url_name="providers")
     def providers(self, request):
         tenant_id = self.request.tenant_id
-        # Subquery to get the most recent finding for each uid
-        latest_finding_ids = (
-            Finding.objects.filter(
+
+        latest_scan_ids = (
+            Scan.objects.filter(
                 tenant_id=tenant_id,
-                uid=OuterRef("uid"),
-                scan__provider=OuterRef("scan__provider"),
+                state=StateChoices.COMPLETED,
             )
-            .order_by("-inserted_at")  # Most recent
-            .values("id")[:1]
+            .order_by("provider_id", "-inserted_at")
+            .distinct("provider_id")
+            .values_list("id", flat=True)
         )
 
-        # Filter findings to only include the most recent for each uid
-        recent_findings = Finding.objects.filter(
-            tenant_id=tenant_id, id__in=Subquery(latest_finding_ids)
-        )
-
-        # Aggregate findings by provider
         findings_aggregated = (
-            recent_findings.values("scan__provider__provider")
+            ScanSummary.objects.filter(tenant_id=tenant_id, scan_id__in=latest_scan_ids)
+            .values("scan__provider__provider")
             .annotate(
-                findings_passed=Count("id", filter=Q(status=StatusChoices.PASS.value)),
-                findings_failed=Count("id", filter=Q(status=StatusChoices.FAIL.value)),
-                findings_manual=Count(
-                    "id", filter=Q(status=StatusChoices.MANUAL.value)
-                ),
-                total_findings=Count("id"),
+                findings_passed=Coalesce(Sum("_pass"), 0),
+                findings_failed=Coalesce(Sum("fail"), 0),
+                findings_muted=Coalesce(Sum("muted"), 0),
+                total_findings=Coalesce(Sum("total"), 0),
             )
-            .order_by("-findings_failed")
         )
 
-        # Aggregate total resources by provider
         resources_aggregated = (
             Resource.objects.filter(tenant_id=tenant_id)
             .values("provider__provider")
             .annotate(total_resources=Count("id"))
         )
+        resources_dict = {
+            row["provider__provider"]: row["total_resources"]
+            for row in resources_aggregated
+        }
 
-        # Combine findings and resources data
         overview = []
-        for findings in findings_aggregated:
-            provider = findings["scan__provider__provider"]
-            total_resources = next(
-                (
-                    res["total_resources"]
-                    for res in resources_aggregated
-                    if res["provider__provider"] == provider
-                ),
-                0,
-            )
+        for row in findings_aggregated:
+            provider_type = row["scan__provider__provider"]
             overview.append(
                 {
-                    "provider": provider,
-                    "total_resources": total_resources,
-                    "total_findings": findings["total_findings"],
-                    "findings_passed": findings["findings_passed"],
-                    "findings_failed": findings["findings_failed"],
-                    "findings_manual": findings["findings_manual"],
+                    "provider": provider_type,
+                    "total_resources": resources_dict.get(provider_type, 0),
+                    "total_findings": row["total_findings"],
+                    "findings_passed": row["findings_passed"],
+                    "findings_failed": row["findings_failed"],
+                    "findings_muted": row["findings_muted"],
                 }
             )
 
         serializer = OverviewProviderSerializer(overview, many=True)
-
         return Response(serializer.data, status=status.HTTP_200_OK)
 
     @action(detail=False, methods=["get"], url_name="findings")