From 52bb1de4c106e603aa3cc84b22936360cc74ff48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Fern=C3=A1ndez=20Poyatos?= Date: Wed, 14 May 2025 09:50:00 +0200 Subject: [PATCH] feat(findings): add /latest endpoints and filters --- api/src/backend/api/filters.py | 232 ++++++++++++++++++------------- api/src/backend/api/v1/mixins.py | 33 +++++ api/src/backend/api/v1/views.py | 151 +++++++++++++++++--- 3 files changed, 301 insertions(+), 115 deletions(-) create mode 100644 api/src/backend/api/v1/mixins.py diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 6d9512f2e2..9e95016e1d 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -81,6 +81,114 @@ class ChoiceInFilter(BaseInFilter, ChoiceFilter): pass +class CommonFindingFilters(FilterSet): + # We filter providers from the scan in findings + provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" + ) + provider_type__in = ChoiceInFilter( + choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" + ) + provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact") + provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in") + provider_uid__icontains = CharFilter( + field_name="scan__provider__uid", lookup_expr="icontains" + ) + provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact") + provider_alias__in = CharInFilter( + field_name="scan__provider__alias", lookup_expr="in" + ) + provider_alias__icontains = CharFilter( + field_name="scan__provider__alias", lookup_expr="icontains" + ) + + updated_at = DateFilter(field_name="updated_at", lookup_expr="date") + + uid = CharFilter(field_name="uid") + delta = ChoiceFilter(choices=Finding.DeltaChoices.choices) + status = ChoiceFilter(choices=StatusChoices.choices) + severity = ChoiceFilter(choices=SeverityChoices) + impact = ChoiceFilter(choices=SeverityChoices) + muted = BooleanFilter( + help_text="If this filter is not provided, muted and non-muted findings will be returned." + ) + + resources = UUIDInFilter(field_name="resource__id", lookup_expr="in") + + region = CharFilter(method="filter_resource_region") + region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap") + region__icontains = CharFilter( + field_name="resource_regions", lookup_expr="icontains" + ) + + service = CharFilter(method="filter_resource_service") + service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap") + service__icontains = CharFilter( + field_name="resource_services", lookup_expr="icontains" + ) + + resource_uid = CharFilter(field_name="resources__uid") + resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in") + resource_uid__icontains = CharFilter( + field_name="resources__uid", lookup_expr="icontains" + ) + + resource_name = CharFilter(field_name="resources__name") + resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in") + resource_name__icontains = CharFilter( + field_name="resources__name", lookup_expr="icontains" + ) + + resource_type = CharFilter(method="filter_resource_type") + resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap") + resource_type__icontains = CharFilter( + field_name="resources__type", lookup_expr="icontains" + ) + + # Temporarily disabled until we implement tag filtering in the UI + # resource_tag_key = CharFilter(field_name="resources__tags__key") + # resource_tag_key__in = CharInFilter( + # field_name="resources__tags__key", lookup_expr="in" + # ) + # resource_tag_key__icontains = CharFilter( + # field_name="resources__tags__key", lookup_expr="icontains" + # ) + # resource_tag_value = CharFilter(field_name="resources__tags__value") + # resource_tag_value__in = CharInFilter( + # field_name="resources__tags__value", lookup_expr="in" + # ) + # resource_tag_value__icontains = CharFilter( + # field_name="resources__tags__value", lookup_expr="icontains" + # ) + # resource_tags = CharInFilter( + # method="filter_resource_tag", + # lookup_expr="in", + # help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be " + # "separated by commas.", + # ) + + def filter_resource_service(self, queryset, name, value): + return queryset.filter(resource_services__contains=[value]) + + def filter_resource_region(self, queryset, name, value): + return queryset.filter(resource_regions__contains=[value]) + + def filter_resource_type(self, queryset, name, value): + return queryset.filter(resource_types__contains=[value]) + + def filter_resource_tag(self, queryset, name, value): + overall_query = Q() + for key_value_pair in value: + tag_key, tag_value = key_value_pair.split(":", 1) + overall_query |= Q( + resources__tags__key__icontains=tag_key, + resources__tags__value__icontains=tag_value, + ) + return queryset.filter(overall_query).distinct() + + class TenantFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") updated_at = DateFilter(field_name="updated_at", lookup_expr="date") @@ -257,94 +365,7 @@ class ResourceFilter(ProviderRelationshipFilterSet): return queryset.filter(tags__text_search=value) -class FindingFilter(FilterSet): - # We filter providers from the scan in findings - provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") - provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") - provider_type = ChoiceFilter( - choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" - ) - provider_type__in = ChoiceInFilter( - choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" - ) - provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact") - provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in") - provider_uid__icontains = CharFilter( - field_name="scan__provider__uid", lookup_expr="icontains" - ) - provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact") - provider_alias__in = CharInFilter( - field_name="scan__provider__alias", lookup_expr="in" - ) - provider_alias__icontains = CharFilter( - field_name="scan__provider__alias", lookup_expr="icontains" - ) - - updated_at = DateFilter(field_name="updated_at", lookup_expr="date") - - uid = CharFilter(field_name="uid") - delta = ChoiceFilter(choices=Finding.DeltaChoices.choices) - status = ChoiceFilter(choices=StatusChoices.choices) - severity = ChoiceFilter(choices=SeverityChoices) - impact = ChoiceFilter(choices=SeverityChoices) - muted = BooleanFilter( - help_text="If this filter is not provided, muted and non-muted findings will be returned." - ) - - resources = UUIDInFilter(field_name="resource__id", lookup_expr="in") - - region = CharFilter(method="filter_resource_region") - region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap") - region__icontains = CharFilter( - field_name="resource_regions", lookup_expr="icontains" - ) - - service = CharFilter(method="filter_resource_service") - service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap") - service__icontains = CharFilter( - field_name="resource_services", lookup_expr="icontains" - ) - - resource_uid = CharFilter(field_name="resources__uid") - resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in") - resource_uid__icontains = CharFilter( - field_name="resources__uid", lookup_expr="icontains" - ) - - resource_name = CharFilter(field_name="resources__name") - resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in") - resource_name__icontains = CharFilter( - field_name="resources__name", lookup_expr="icontains" - ) - - resource_type = CharFilter(method="filter_resource_type") - resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap") - resource_type__icontains = CharFilter( - field_name="resources__type", lookup_expr="icontains" - ) - - # Temporarily disabled until we implement tag filtering in the UI - # resource_tag_key = CharFilter(field_name="resources__tags__key") - # resource_tag_key__in = CharInFilter( - # field_name="resources__tags__key", lookup_expr="in" - # ) - # resource_tag_key__icontains = CharFilter( - # field_name="resources__tags__key", lookup_expr="icontains" - # ) - # resource_tag_value = CharFilter(field_name="resources__tags__value") - # resource_tag_value__in = CharInFilter( - # field_name="resources__tags__value", lookup_expr="in" - # ) - # resource_tag_value__icontains = CharFilter( - # field_name="resources__tags__value", lookup_expr="icontains" - # ) - # resource_tags = CharInFilter( - # method="filter_resource_tag", - # lookup_expr="in", - # help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be " - # "separated by commas.", - # ) - +class FindingFilter(CommonFindingFilters): scan = UUIDFilter(method="filter_scan_id") scan__in = UUIDInFilter(method="filter_scan_id_in") @@ -512,16 +533,6 @@ class FindingFilter(FilterSet): return queryset.filter(id__lt=end) - def filter_resource_tag(self, queryset, name, value): - overall_query = Q() - for key_value_pair in value: - tag_key, tag_value = key_value_pair.split(":", 1) - overall_query |= Q( - resources__tags__key__icontains=tag_key, - resources__tags__value__icontains=tag_value, - ) - return queryset.filter(overall_query).distinct() - @staticmethod def maybe_date_to_datetime(value): dt = value @@ -530,6 +541,31 @@ class FindingFilter(FilterSet): return dt +class LatestFindingFilter(CommonFindingFilters): + class Meta: + model = Finding + fields = { + "id": ["exact", "in"], + "uid": ["exact", "in"], + "delta": ["exact", "in"], + "status": ["exact", "in"], + "severity": ["exact", "in"], + "impact": ["exact", "in"], + "check_id": ["exact", "in", "icontains"], + } + filter_overrides = { + FindingDeltaEnumField: { + "filter_class": CharFilter, + }, + StatusEnumField: { + "filter_class": CharFilter, + }, + SeverityEnumField: { + "filter_class": CharFilter, + }, + } + + class ProviderSecretFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") updated_at = DateFilter(field_name="updated_at", lookup_expr="date") diff --git a/api/src/backend/api/v1/mixins.py b/api/src/backend/api/v1/mixins.py new file mode 100644 index 0000000000..85250c0eef --- /dev/null +++ b/api/src/backend/api/v1/mixins.py @@ -0,0 +1,33 @@ +from rest_framework.response import Response + + +class PaginateByPkMixin: + """ + Mixin to paginate on a list of PKs (cheaper than heavy JOINs), + re-fetch the full objects with the desired select/prefetch, + re-sort them to preserve DB ordering, then serialize + return. + """ + + def paginate_by_pk( + self, + request, # noqa: F841 + base_queryset, + manager, + select_related: list[str] | None = None, + prefetch_related: list[str] | None = None, + ) -> Response: + pk_list = base_queryset.values_list("id", flat=True) + page = self.paginate_queryset(pk_list) + if page is None: + return Response(self.get_serializer(base_queryset, many=True).data) + + queryset = manager.filter(id__in=page) + if select_related: + queryset = queryset.select_related(*select_related) + if prefetch_related: + queryset = queryset.prefetch_related(*prefetch_related) + + queryset = sorted(queryset, key=lambda obj: page.index(obj.id)) + + serialized = self.get_serializer(queryset, many=True).data + return self.get_paginated_response(serialized) diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 19b21df92e..4bf8cfe2f6 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -65,6 +65,7 @@ from api.filters import ( FindingFilter, IntegrationFilter, InvitationFilter, + LatestFindingFilter, MembershipFilter, ProviderFilter, ProviderGroupFilter, @@ -110,6 +111,7 @@ from api.utils import ( validate_invitation, ) from api.uuid_utils import datetime_to_uuid7, uuid7_start +from api.v1.mixins import PaginateByPkMixin from api.v1.serializers import ( ComplianceOverviewFullSerializer, ComplianceOverviewMetadataSerializer, @@ -1671,10 +1673,24 @@ class ResourceViewSet(BaseRLSViewSet): ], filters=True, ), + latest=extend_schema( + tags=["Finding"], + summary="List the latest findings", + description="Retrieve a list of the latest findings from the latest scans for each provider with options for " + "filtering by various criteria.", + filters=True, + ), + metadata_latest=extend_schema( + tags=["Finding"], + summary="Retrieve metadata values from the latest findings", + description="Fetch unique metadata values from a set of findings from the latest scans for each provider. " + "This is useful for dynamic filtering.", + filters=True, + ), ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") -class FindingViewSet(BaseRLSViewSet): +class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): queryset = Finding.all_objects.all() serializer_class = FindingSerializer filterset_class = FindingFilter @@ -1706,11 +1722,16 @@ class FindingViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "findings_services_regions": return FindingDynamicFilterSerializer - elif self.action == "metadata": + elif self.action in ["metadata", "metadata_latest"]: return FindingMetadataSerializer return super().get_serializer_class() + def get_filterset_class(self): + if self.action in ["latest", "metadata_latest"]: + return LatestFindingFilter + return FindingFilter + def get_queryset(self): tenant_id = self.request.tenant_id user_roles = get_role(self.request.user) @@ -1750,21 +1771,14 @@ class FindingViewSet(BaseRLSViewSet): return super().filter_queryset(queryset) def list(self, request, *args, **kwargs): - base_qs = self.filter_queryset(self.get_queryset()) - paginated_ids = self.paginate_queryset(base_qs.values_list("id", flat=True)) - if paginated_ids is not None: - ids = list(paginated_ids) - findings = ( - Finding.all_objects.filter(tenant_id=self.request.tenant_id, id__in=ids) - .select_related("scan") - .prefetch_related("resources") - ) - # Re-sort in Python to preserve ordering: - findings = sorted(findings, key=lambda x: ids.index(x.id)) - serializer = self.get_serializer(findings, many=True) - return self.get_paginated_response(serializer.data) - serializer = self.get_serializer(base_qs, many=True) - return Response(serializer.data) + filtered_queryset = self.filter_queryset(self.get_queryset()) + return self.paginate_by_pk( + request, + filtered_queryset, + manager=Finding.all_objects, + select_related=["scan"], + prefetch_related=["resources"], + ) @action(detail=False, methods=["get"], url_name="findings_services_regions") def findings_services_regions(self, request): @@ -1897,6 +1911,109 @@ class FindingViewSet(BaseRLSViewSet): serializer.is_valid(raise_exception=True) return Response(serializer.data) + @action(detail=False, methods=["get"], url_name="latest") + def latest(self, request): + tenant_id = request.tenant_id + filtered_queryset = self.filter_queryset(self.get_queryset()) + + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + filtered_queryset = filtered_queryset.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + + return self.paginate_by_pk( + request, + filtered_queryset, + manager=Finding.all_objects, + select_related=["scan"], + prefetch_related=["resources"], + ) + + @action( + detail=False, + methods=["get"], + url_name="metadata_latest", + url_path="metadata/latest", + ) + def metadata_latest(self, request): + # Force filter validation + filtered_queryset = self.filter_queryset(self.get_queryset()) + + tenant_id = request.tenant_id + query_params = request.query_params + + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + + queryset = ResourceScanSummary.objects.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + # ToRemove: Temporary fallback mechanism + scans_with_flag = latest_scan_ids.annotate( + has_summary=Exists( + ResourceScanSummary.objects.filter( + tenant_id=tenant_id, + scan_id=OuterRef("pk"), + ) + ) + ) + if missing_scan_ids := scans_with_flag.filter(has_summary=False).values_list( + "id", flat=True + ): + for scan_id in missing_scan_ids: + backfill_scan_resource_summaries_task.apply_async( + kwargs={"tenant_id": tenant_id, "scan_id": scan_id} + ) + return Response( + get_findings_metadata_no_aggregations(tenant_id, filtered_queryset) + ) + + if service_filter := query_params.get("filter[service]") or query_params.get( + "filter[service__in]" + ): + queryset = queryset.filter(service__in=service_filter.split(",")) + if region_filter := query_params.get("filter[region]") or query_params.get( + "filter[region__in]" + ): + queryset = queryset.filter(region__in=region_filter.split(",")) + if resource_type_filter := query_params.get( + "filter[resource_type]" + ) or query_params.get("filter[resource_type__in]"): + queryset = queryset.filter( + resource_type__in=resource_type_filter.split(",") + ) + + services = list( + queryset.values_list("service", flat=True).distinct().order_by("service") + ) + regions = list( + queryset.values_list("region", flat=True).distinct().order_by("region") + ) + resource_types = list( + queryset.values_list("resource_type", flat=True) + .distinct() + .order_by("resource_type") + ) + + result = { + "services": services, + "regions": regions, + "resource_types": resource_types, + } + + serializer = self.get_serializer(data=result) + serializer.is_valid(raise_exception=True) + return Response(serializer.data) + @extend_schema_view( list=extend_schema(