feat(findings): add /latest endpoints and filters

2026-07-23 12:31:54 +00:00 · 2025-05-14 09:50:00 +02:00
parent 02d7eaf268
commit 52bb1de4c1
3 changed files with 301 additions and 115 deletions
@@ -81,6 +81,114 @@ class ChoiceInFilter(BaseInFilter, ChoiceFilter):
    pass


+class CommonFindingFilters(FilterSet):
+    # We filter providers from the scan in findings
+    provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
+    provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
+    provider_type = ChoiceFilter(
+        choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
+    )
+    provider_type__in = ChoiceInFilter(
+        choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
+    )
+    provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact")
+    provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in")
+    provider_uid__icontains = CharFilter(
+        field_name="scan__provider__uid", lookup_expr="icontains"
+    )
+    provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact")
+    provider_alias__in = CharInFilter(
+        field_name="scan__provider__alias", lookup_expr="in"
+    )
+    provider_alias__icontains = CharFilter(
+        field_name="scan__provider__alias", lookup_expr="icontains"
+    )
+
+    updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
+
+    uid = CharFilter(field_name="uid")
+    delta = ChoiceFilter(choices=Finding.DeltaChoices.choices)
+    status = ChoiceFilter(choices=StatusChoices.choices)
+    severity = ChoiceFilter(choices=SeverityChoices)
+    impact = ChoiceFilter(choices=SeverityChoices)
+    muted = BooleanFilter(
+        help_text="If this filter is not provided, muted and non-muted findings will be returned."
+    )
+
+    resources = UUIDInFilter(field_name="resource__id", lookup_expr="in")
+
+    region = CharFilter(method="filter_resource_region")
+    region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap")
+    region__icontains = CharFilter(
+        field_name="resource_regions", lookup_expr="icontains"
+    )
+
+    service = CharFilter(method="filter_resource_service")
+    service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap")
+    service__icontains = CharFilter(
+        field_name="resource_services", lookup_expr="icontains"
+    )
+
+    resource_uid = CharFilter(field_name="resources__uid")
+    resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in")
+    resource_uid__icontains = CharFilter(
+        field_name="resources__uid", lookup_expr="icontains"
+    )
+
+    resource_name = CharFilter(field_name="resources__name")
+    resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in")
+    resource_name__icontains = CharFilter(
+        field_name="resources__name", lookup_expr="icontains"
+    )
+
+    resource_type = CharFilter(method="filter_resource_type")
+    resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap")
+    resource_type__icontains = CharFilter(
+        field_name="resources__type", lookup_expr="icontains"
+    )
+
+    # Temporarily disabled until we implement tag filtering in the UI
+    # resource_tag_key = CharFilter(field_name="resources__tags__key")
+    # resource_tag_key__in = CharInFilter(
+    #     field_name="resources__tags__key", lookup_expr="in"
+    # )
+    # resource_tag_key__icontains = CharFilter(
+    #     field_name="resources__tags__key", lookup_expr="icontains"
+    # )
+    # resource_tag_value = CharFilter(field_name="resources__tags__value")
+    # resource_tag_value__in = CharInFilter(
+    #     field_name="resources__tags__value", lookup_expr="in"
+    # )
+    # resource_tag_value__icontains = CharFilter(
+    #     field_name="resources__tags__value", lookup_expr="icontains"
+    # )
+    # resource_tags = CharInFilter(
+    #     method="filter_resource_tag",
+    #     lookup_expr="in",
+    #     help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be "
+    #     "separated by commas.",
+    # )
+
+    def filter_resource_service(self, queryset, name, value):
+        return queryset.filter(resource_services__contains=[value])
+
+    def filter_resource_region(self, queryset, name, value):
+        return queryset.filter(resource_regions__contains=[value])
+
+    def filter_resource_type(self, queryset, name, value):
+        return queryset.filter(resource_types__contains=[value])
+
+    def filter_resource_tag(self, queryset, name, value):
+        overall_query = Q()
+        for key_value_pair in value:
+            tag_key, tag_value = key_value_pair.split(":", 1)
+            overall_query |= Q(
+                resources__tags__key__icontains=tag_key,
+                resources__tags__value__icontains=tag_value,
+            )
+        return queryset.filter(overall_query).distinct()
+
+
 class TenantFilter(FilterSet):
    inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
    updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
@@ -257,94 +365,7 @@ class ResourceFilter(ProviderRelationshipFilterSet):
        return queryset.filter(tags__text_search=value)


-class FindingFilter(FilterSet):
-    # We filter providers from the scan in findings
-    provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
-    provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
-    provider_type = ChoiceFilter(
-        choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
-    )
-    provider_type__in = ChoiceInFilter(
-        choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
-    )
-    provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact")
-    provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in")
-    provider_uid__icontains = CharFilter(
-        field_name="scan__provider__uid", lookup_expr="icontains"
-    )
-    provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact")
-    provider_alias__in = CharInFilter(
-        field_name="scan__provider__alias", lookup_expr="in"
-    )
-    provider_alias__icontains = CharFilter(
-        field_name="scan__provider__alias", lookup_expr="icontains"
-    )
-
-    updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
-
-    uid = CharFilter(field_name="uid")
-    delta = ChoiceFilter(choices=Finding.DeltaChoices.choices)
-    status = ChoiceFilter(choices=StatusChoices.choices)
-    severity = ChoiceFilter(choices=SeverityChoices)
-    impact = ChoiceFilter(choices=SeverityChoices)
-    muted = BooleanFilter(
-        help_text="If this filter is not provided, muted and non-muted findings will be returned."
-    )
-
-    resources = UUIDInFilter(field_name="resource__id", lookup_expr="in")
-
-    region = CharFilter(method="filter_resource_region")
-    region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap")
-    region__icontains = CharFilter(
-        field_name="resource_regions", lookup_expr="icontains"
-    )
-
-    service = CharFilter(method="filter_resource_service")
-    service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap")
-    service__icontains = CharFilter(
-        field_name="resource_services", lookup_expr="icontains"
-    )
-
-    resource_uid = CharFilter(field_name="resources__uid")
-    resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in")
-    resource_uid__icontains = CharFilter(
-        field_name="resources__uid", lookup_expr="icontains"
-    )
-
-    resource_name = CharFilter(field_name="resources__name")
-    resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in")
-    resource_name__icontains = CharFilter(
-        field_name="resources__name", lookup_expr="icontains"
-    )
-
-    resource_type = CharFilter(method="filter_resource_type")
-    resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap")
-    resource_type__icontains = CharFilter(
-        field_name="resources__type", lookup_expr="icontains"
-    )
-
-    # Temporarily disabled until we implement tag filtering in the UI
-    # resource_tag_key = CharFilter(field_name="resources__tags__key")
-    # resource_tag_key__in = CharInFilter(
-    #     field_name="resources__tags__key", lookup_expr="in"
-    # )
-    # resource_tag_key__icontains = CharFilter(
-    #     field_name="resources__tags__key", lookup_expr="icontains"
-    # )
-    # resource_tag_value = CharFilter(field_name="resources__tags__value")
-    # resource_tag_value__in = CharInFilter(
-    #     field_name="resources__tags__value", lookup_expr="in"
-    # )
-    # resource_tag_value__icontains = CharFilter(
-    #     field_name="resources__tags__value", lookup_expr="icontains"
-    # )
-    # resource_tags = CharInFilter(
-    #     method="filter_resource_tag",
-    #     lookup_expr="in",
-    #     help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be "
-    #     "separated by commas.",
-    # )
-
+class FindingFilter(CommonFindingFilters):
    scan = UUIDFilter(method="filter_scan_id")
    scan__in = UUIDInFilter(method="filter_scan_id_in")

@@ -512,16 +533,6 @@ class FindingFilter(FilterSet):

        return queryset.filter(id__lt=end)

-    def filter_resource_tag(self, queryset, name, value):
-        overall_query = Q()
-        for key_value_pair in value:
-            tag_key, tag_value = key_value_pair.split(":", 1)
-            overall_query |= Q(
-                resources__tags__key__icontains=tag_key,
-                resources__tags__value__icontains=tag_value,
-            )
-        return queryset.filter(overall_query).distinct()
-
    @staticmethod
    def maybe_date_to_datetime(value):
        dt = value
@@ -530,6 +541,31 @@ class FindingFilter(FilterSet):
        return dt


+class LatestFindingFilter(CommonFindingFilters):
+    class Meta:
+        model = Finding
+        fields = {
+            "id": ["exact", "in"],
+            "uid": ["exact", "in"],
+            "delta": ["exact", "in"],
+            "status": ["exact", "in"],
+            "severity": ["exact", "in"],
+            "impact": ["exact", "in"],
+            "check_id": ["exact", "in", "icontains"],
+        }
+        filter_overrides = {
+            FindingDeltaEnumField: {
+                "filter_class": CharFilter,
+            },
+            StatusEnumField: {
+                "filter_class": CharFilter,
+            },
+            SeverityEnumField: {
+                "filter_class": CharFilter,
+            },
+        }
+
+
 class ProviderSecretFilter(FilterSet):
    inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
    updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
@@ -0,0 +1,33 @@
+from rest_framework.response import Response
+
+
+class PaginateByPkMixin:
+    """
+    Mixin to paginate on a list of PKs (cheaper than heavy JOINs),
+    re-fetch the full objects with the desired select/prefetch,
+    re-sort them to preserve DB ordering, then serialize + return.
+    """
+
+    def paginate_by_pk(
+        self,
+        request,  # noqa: F841
+        base_queryset,
+        manager,
+        select_related: list[str] | None = None,
+        prefetch_related: list[str] | None = None,
+    ) -> Response:
+        pk_list = base_queryset.values_list("id", flat=True)
+        page = self.paginate_queryset(pk_list)
+        if page is None:
+            return Response(self.get_serializer(base_queryset, many=True).data)
+
+        queryset = manager.filter(id__in=page)
+        if select_related:
+            queryset = queryset.select_related(*select_related)
+        if prefetch_related:
+            queryset = queryset.prefetch_related(*prefetch_related)
+
+        queryset = sorted(queryset, key=lambda obj: page.index(obj.id))
+
+        serialized = self.get_serializer(queryset, many=True).data
+        return self.get_paginated_response(serialized)
@@ -65,6 +65,7 @@ from api.filters import (
    FindingFilter,
    IntegrationFilter,
    InvitationFilter,
+    LatestFindingFilter,
    MembershipFilter,
    ProviderFilter,
    ProviderGroupFilter,
@@ -110,6 +111,7 @@ from api.utils import (
    validate_invitation,
 )
 from api.uuid_utils import datetime_to_uuid7, uuid7_start
+from api.v1.mixins import PaginateByPkMixin
 from api.v1.serializers import (
    ComplianceOverviewFullSerializer,
    ComplianceOverviewMetadataSerializer,
@@ -1671,10 +1673,24 @@ class ResourceViewSet(BaseRLSViewSet):
        ],
        filters=True,
    ),
+    latest=extend_schema(
+        tags=["Finding"],
+        summary="List the latest findings",
+        description="Retrieve a list of the latest findings from the latest scans for each provider with options for "
+        "filtering by various criteria.",
+        filters=True,
+    ),
+    metadata_latest=extend_schema(
+        tags=["Finding"],
+        summary="Retrieve metadata values from the latest findings",
+        description="Fetch unique metadata values from a set of findings from the latest scans for each provider. "
+        "This is useful for dynamic filtering.",
+        filters=True,
+    ),
 )
@method_decorator(CACHE_DECORATOR, name="list")
@method_decorator(CACHE_DECORATOR, name="retrieve")
-class FindingViewSet(BaseRLSViewSet):
+class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
    queryset = Finding.all_objects.all()
    serializer_class = FindingSerializer
    filterset_class = FindingFilter
@@ -1706,11 +1722,16 @@ class FindingViewSet(BaseRLSViewSet):
    def get_serializer_class(self):
        if self.action == "findings_services_regions":
            return FindingDynamicFilterSerializer
-        elif self.action == "metadata":
+        elif self.action in ["metadata", "metadata_latest"]:
            return FindingMetadataSerializer

        return super().get_serializer_class()

+    def get_filterset_class(self):
+        if self.action in ["latest", "metadata_latest"]:
+            return LatestFindingFilter
+        return FindingFilter
+
    def get_queryset(self):
        tenant_id = self.request.tenant_id
        user_roles = get_role(self.request.user)
@@ -1750,21 +1771,14 @@ class FindingViewSet(BaseRLSViewSet):
        return super().filter_queryset(queryset)

    def list(self, request, *args, **kwargs):
-        base_qs = self.filter_queryset(self.get_queryset())
-        paginated_ids = self.paginate_queryset(base_qs.values_list("id", flat=True))
-        if paginated_ids is not None:
-            ids = list(paginated_ids)
-            findings = (
-                Finding.all_objects.filter(tenant_id=self.request.tenant_id, id__in=ids)
-                .select_related("scan")
-                .prefetch_related("resources")
-            )
-            # Re-sort in Python to preserve ordering:
-            findings = sorted(findings, key=lambda x: ids.index(x.id))
-            serializer = self.get_serializer(findings, many=True)
-            return self.get_paginated_response(serializer.data)
-        serializer = self.get_serializer(base_qs, many=True)
-        return Response(serializer.data)
+        filtered_queryset = self.filter_queryset(self.get_queryset())
+        return self.paginate_by_pk(
+            request,
+            filtered_queryset,
+            manager=Finding.all_objects,
+            select_related=["scan"],
+            prefetch_related=["resources"],
+        )

    @action(detail=False, methods=["get"], url_name="findings_services_regions")
    def findings_services_regions(self, request):
@@ -1897,6 +1911,109 @@ class FindingViewSet(BaseRLSViewSet):
        serializer.is_valid(raise_exception=True)
        return Response(serializer.data)

+    @action(detail=False, methods=["get"], url_name="latest")
+    def latest(self, request):
+        tenant_id = request.tenant_id
+        filtered_queryset = self.filter_queryset(self.get_queryset())
+
+        latest_scan_ids = (
+            Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
+            .order_by("provider_id", "-inserted_at")
+            .distinct("provider_id")
+            .values_list("id", flat=True)
+        )
+        filtered_queryset = filtered_queryset.filter(
+            tenant_id=tenant_id, scan_id__in=latest_scan_ids
+        )
+
+        return self.paginate_by_pk(
+            request,
+            filtered_queryset,
+            manager=Finding.all_objects,
+            select_related=["scan"],
+            prefetch_related=["resources"],
+        )
+
+    @action(
+        detail=False,
+        methods=["get"],
+        url_name="metadata_latest",
+        url_path="metadata/latest",
+    )
+    def metadata_latest(self, request):
+        # Force filter validation
+        filtered_queryset = self.filter_queryset(self.get_queryset())
+
+        tenant_id = request.tenant_id
+        query_params = request.query_params
+
+        latest_scan_ids = (
+            Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
+            .order_by("provider_id", "-inserted_at")
+            .distinct("provider_id")
+            .values_list("id", flat=True)
+        )
+
+        queryset = ResourceScanSummary.objects.filter(
+            tenant_id=tenant_id, scan_id__in=latest_scan_ids
+        )
+        # ToRemove: Temporary fallback mechanism
+        scans_with_flag = latest_scan_ids.annotate(
+            has_summary=Exists(
+                ResourceScanSummary.objects.filter(
+                    tenant_id=tenant_id,
+                    scan_id=OuterRef("pk"),
+                )
+            )
+        )
+        if missing_scan_ids := scans_with_flag.filter(has_summary=False).values_list(
+            "id", flat=True
+        ):
+            for scan_id in missing_scan_ids:
+                backfill_scan_resource_summaries_task.apply_async(
+                    kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
+                )
+            return Response(
+                get_findings_metadata_no_aggregations(tenant_id, filtered_queryset)
+            )
+
+        if service_filter := query_params.get("filter[service]") or query_params.get(
+            "filter[service__in]"
+        ):
+            queryset = queryset.filter(service__in=service_filter.split(","))
+        if region_filter := query_params.get("filter[region]") or query_params.get(
+            "filter[region__in]"
+        ):
+            queryset = queryset.filter(region__in=region_filter.split(","))
+        if resource_type_filter := query_params.get(
+            "filter[resource_type]"
+        ) or query_params.get("filter[resource_type__in]"):
+            queryset = queryset.filter(
+                resource_type__in=resource_type_filter.split(",")
+            )
+
+        services = list(
+            queryset.values_list("service", flat=True).distinct().order_by("service")
+        )
+        regions = list(
+            queryset.values_list("region", flat=True).distinct().order_by("region")
+        )
+        resource_types = list(
+            queryset.values_list("resource_type", flat=True)
+            .distinct()
+            .order_by("resource_type")
+        )
+
+        result = {
+            "services": services,
+            "regions": regions,
+            "resource_types": resource_types,
+        }
+
+        serializer = self.get_serializer(data=result)
+        serializer.is_valid(raise_exception=True)
+        return Response(serializer.data)
+

@extend_schema_view(
    list=extend_schema(