feat(resources): Optimize findings prefetching during resource views (#8336)

api/src/backend/api/migrations/0040_rfm_tenant_resource_index_partitions.py

@@ -0,0 +1,30 @@
+from functools import partial
+
+from django.db import migrations
+
+from api.db_utils import create_index_on_partitions, drop_index_on_partitions
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("api", "0039_resource_resources_failed_findings_idx"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            partial(
+                create_index_on_partitions,
+                parent_table="resource_finding_mappings",
+                index_name="rfm_tenant_resource_idx",
+                columns="tenant_id, resource_id",
+                method="BTREE",
+            ),
+            reverse_code=partial(
+                drop_index_on_partitions,
+                parent_table="resource_finding_mappings",
+                index_name="rfm_tenant_resource_idx",
+            ),
+        ),
+    ]

api/src/backend/api/migrations/0041_rfm_tenant_resource_parent_partitions.py

@@ -0,0 +1,17 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0040_rfm_tenant_resource_index_partitions"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="resourcefindingmapping",
+            index=models.Index(
+                fields=["tenant_id", "resource_id"],
+                name="rfm_tenant_resource_idx",
+            ),
+        ),
+    ]

api/src/backend/api/migrations/0042_scan_scans_prov_ins_desc_idx.py

@@ -0,0 +1,23 @@
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("api", "0041_rfm_tenant_resource_parent_partitions"),
+        ("django_celery_beat", "0019_alter_periodictasks_options"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="scan",
+            index=models.Index(
+                condition=models.Q(("state", "completed")),
+                fields=["tenant_id", "provider_id", "-inserted_at"],
+                include=("id",),
+                name="scans_prov_ins_desc_idx",
+            ),
+        ),
+    ]

api/src/backend/api/models.py

@@ -476,6 +476,13 @@ class Scan(RowLevelSecurityProtectedModel):
                 condition=Q(state=StateChoices.COMPLETED),
                 name="scans_prov_state_ins_desc_idx",
             ),
+            # TODO This might replace `scans_prov_state_ins_desc_idx` completely. Review usage
+            models.Index(
+                fields=["tenant_id", "provider_id", "-inserted_at"],
+                condition=Q(state=StateChoices.COMPLETED),
+                include=["id"],
+                name="scans_prov_ins_desc_idx",
+            ),
         ]
 
     class JSONAPIMeta:
@@ -860,6 +867,10 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected
                 fields=["tenant_id", "finding_id"],
                 name="rfm_tenant_finding_idx",
             ),
+            models.Index(
+                fields=["tenant_id", "resource_id"],
+                name="rfm_tenant_resource_idx",
+            ),
         ]
         constraints = [
             models.UniqueConstraint(

api/src/backend/api/specs/v1.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: Prowler API
-  version: 1.10.1
+  version: 1.10.2
   description: |-
     Prowler API specification.
 

api/src/backend/api/v1/views.py

@@ -22,7 +22,7 @@ from django.conf import settings as django_settings
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.search import SearchQuery
 from django.db import transaction
-from django.db.models import Count, F, Prefetch, Q, Sum
+from django.db.models import Count, F, Prefetch, Q, Subquery, Sum
 from django.db.models.functions import Coalesce
 from django.http import HttpResponse
 from django.shortcuts import redirect
@@ -292,7 +292,7 @@ class SchemaView(SpectacularAPIView):
 
     def get(self, request, *args, **kwargs):
         spectacular_settings.TITLE = "Prowler API"
-        spectacular_settings.VERSION = "1.10.1"
+        spectacular_settings.VERSION = "1.10.2"
         spectacular_settings.DESCRIPTION = (
             "Prowler API specification.\n\nThis file is auto-generated."
         )
@@ -1994,6 +1994,21 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
             )
         )
 
+    def _should_prefetch_findings(self) -> bool:
+        fields_param = self.request.query_params.get("fields[resources]", "")
+        include_param = self.request.query_params.get("include", "")
+        return (
+            fields_param == ""
+            or "findings" in fields_param.split(",")
+            or "findings" in include_param.split(",")
+        )
+
+    def _get_findings_prefetch(self):
+        findings_queryset = Finding.all_objects.defer("scan", "resources").filter(
+            tenant_id=self.request.tenant_id
+        )
+        return [Prefetch("findings", queryset=findings_queryset)]
+
     def get_serializer_class(self):
         if self.action in ["metadata", "metadata_latest"]:
             return ResourceMetadataSerializer
@@ -2017,7 +2032,11 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
             filtered_queryset,
             manager=Resource.all_objects,
             select_related=["provider"],
-            prefetch_related=["findings"],
+            prefetch_related=(
+                self._get_findings_prefetch()
+                if self._should_prefetch_findings()
+                else []
+            ),
         )
 
     def retrieve(self, request, *args, **kwargs):
@@ -2042,14 +2061,18 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
         tenant_id = request.tenant_id
         filtered_queryset = self.filter_queryset(self.get_queryset())
 
-        latest_scan_ids = (
-            Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
+        latest_scans = (
+            Scan.all_objects.filter(
+                tenant_id=tenant_id,
+                state=StateChoices.COMPLETED,
+            )
             .order_by("provider_id", "-inserted_at")
             .distinct("provider_id")
-            .values_list("id", flat=True)
+            .values("provider_id")
         )
+
         filtered_queryset = filtered_queryset.filter(
-            tenant_id=tenant_id, provider__scan__in=latest_scan_ids
+            provider_id__in=Subquery(latest_scans)
         )
 
         return self.paginate_by_pk(
@@ -2057,7 +2080,11 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
             filtered_queryset,
             manager=Resource.all_objects,
             select_related=["provider"],
-            prefetch_related=["findings"],
+            prefetch_related=(
+                self._get_findings_prefetch()
+                if self._should_prefetch_findings()
+                else []
+            ),
         )
 
     @action(detail=False, methods=["get"], url_name="metadata")