diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 87d82df6f5..6afe7abb1b 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to the **Prowler API** are documented in this file. +## [1.10.2] (Prowler v5.9.2) + +### Changed +- Optimized queries for resources views [(#8336)](https://github.com/prowler-cloud/prowler/pull/8336) + +--- + ## [v1.10.1] (Prowler v5.9.1) ### Fixed diff --git a/api/pyproject.toml b/api/pyproject.toml index f7cd0d1808..dac2c9d8b1 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -38,7 +38,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.10.1" +version = "1.10.2" [project.scripts] celery = "src.backend.config.settings.celery" diff --git a/api/src/backend/api/migrations/0040_rfm_tenant_resource_index_partitions.py b/api/src/backend/api/migrations/0040_rfm_tenant_resource_index_partitions.py new file mode 100644 index 0000000000..431d656376 --- /dev/null +++ b/api/src/backend/api/migrations/0040_rfm_tenant_resource_index_partitions.py @@ -0,0 +1,30 @@ +from functools import partial + +from django.db import migrations + +from api.db_utils import create_index_on_partitions, drop_index_on_partitions + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0039_resource_resources_failed_findings_idx"), + ] + + operations = [ + migrations.RunPython( + partial( + create_index_on_partitions, + parent_table="resource_finding_mappings", + index_name="rfm_tenant_resource_idx", + columns="tenant_id, resource_id", + method="BTREE", + ), + reverse_code=partial( + drop_index_on_partitions, + parent_table="resource_finding_mappings", + index_name="rfm_tenant_resource_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0041_rfm_tenant_resource_parent_partitions.py b/api/src/backend/api/migrations/0041_rfm_tenant_resource_parent_partitions.py new file mode 100644 index 0000000000..cd4b54d61b --- /dev/null +++ b/api/src/backend/api/migrations/0041_rfm_tenant_resource_parent_partitions.py @@ -0,0 +1,17 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0040_rfm_tenant_resource_index_partitions"), + ] + + operations = [ + migrations.AddIndex( + model_name="resourcefindingmapping", + index=models.Index( + fields=["tenant_id", "resource_id"], + name="rfm_tenant_resource_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0042_scan_scans_prov_ins_desc_idx.py b/api/src/backend/api/migrations/0042_scan_scans_prov_ins_desc_idx.py new file mode 100644 index 0000000000..da2db905c6 --- /dev/null +++ b/api/src/backend/api/migrations/0042_scan_scans_prov_ins_desc_idx.py @@ -0,0 +1,23 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0041_rfm_tenant_resource_parent_partitions"), + ("django_celery_beat", "0019_alter_periodictasks_options"), + ] + + operations = [ + AddIndexConcurrently( + model_name="scan", + index=models.Index( + condition=models.Q(("state", "completed")), + fields=["tenant_id", "provider_id", "-inserted_at"], + include=("id",), + name="scans_prov_ins_desc_idx", + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index f81ebafaf3..632e3b5dd3 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -476,6 +476,13 @@ class Scan(RowLevelSecurityProtectedModel): condition=Q(state=StateChoices.COMPLETED), name="scans_prov_state_ins_desc_idx", ), + # TODO This might replace `scans_prov_state_ins_desc_idx` completely. Review usage + models.Index( + fields=["tenant_id", "provider_id", "-inserted_at"], + condition=Q(state=StateChoices.COMPLETED), + include=["id"], + name="scans_prov_ins_desc_idx", + ), ] class JSONAPIMeta: @@ -860,6 +867,10 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected fields=["tenant_id", "finding_id"], name="rfm_tenant_finding_idx", ), + models.Index( + fields=["tenant_id", "resource_id"], + name="rfm_tenant_resource_idx", + ), ] constraints = [ models.UniqueConstraint( diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 3b500422dc..3ea3610e29 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.10.1 + version: 1.10.2 description: |- Prowler API specification. diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 6219609717..e81808ae8f 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -22,7 +22,7 @@ from django.conf import settings as django_settings from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction -from django.db.models import Count, F, Prefetch, Q, Sum +from django.db.models import Count, F, Prefetch, Q, Subquery, Sum from django.db.models.functions import Coalesce from django.http import HttpResponse from django.shortcuts import redirect @@ -292,7 +292,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.10.1" + spectacular_settings.VERSION = "1.10.2" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -1994,6 +1994,21 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet): ) ) + def _should_prefetch_findings(self) -> bool: + fields_param = self.request.query_params.get("fields[resources]", "") + include_param = self.request.query_params.get("include", "") + return ( + fields_param == "" + or "findings" in fields_param.split(",") + or "findings" in include_param.split(",") + ) + + def _get_findings_prefetch(self): + findings_queryset = Finding.all_objects.defer("scan", "resources").filter( + tenant_id=self.request.tenant_id + ) + return [Prefetch("findings", queryset=findings_queryset)] + def get_serializer_class(self): if self.action in ["metadata", "metadata_latest"]: return ResourceMetadataSerializer @@ -2017,7 +2032,11 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet): filtered_queryset, manager=Resource.all_objects, select_related=["provider"], - prefetch_related=["findings"], + prefetch_related=( + self._get_findings_prefetch() + if self._should_prefetch_findings() + else [] + ), ) def retrieve(self, request, *args, **kwargs): @@ -2042,14 +2061,18 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet): tenant_id = request.tenant_id filtered_queryset = self.filter_queryset(self.get_queryset()) - latest_scan_ids = ( - Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + latest_scans = ( + Scan.all_objects.filter( + tenant_id=tenant_id, + state=StateChoices.COMPLETED, + ) .order_by("provider_id", "-inserted_at") .distinct("provider_id") - .values_list("id", flat=True) + .values("provider_id") ) + filtered_queryset = filtered_queryset.filter( - tenant_id=tenant_id, provider__scan__in=latest_scan_ids + provider_id__in=Subquery(latest_scans) ) return self.paginate_by_pk( @@ -2057,7 +2080,11 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet): filtered_queryset, manager=Resource.all_objects, select_related=["provider"], - prefetch_related=["findings"], + prefetch_related=( + self._get_findings_prefetch() + if self._should_prefetch_findings() + else [] + ), ) @action(detail=False, methods=["get"], url_name="metadata")