diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 5fe9c28259..615ba1d91e 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to the **Prowler API** are documented in this file. - Added huge improvements to `/findings/metadata` and resource related filters for findings [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690). - Added improvements to `/overviews` endpoints [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690). - Added new queue to perform backfill background tasks [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690). +- Added new endpoints to retrieve latest findings and metadata [(#7743)](https://github.com/prowler-cloud/prowler/pull/7743). --- diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 6d9512f2e2..9e95016e1d 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -81,6 +81,114 @@ class ChoiceInFilter(BaseInFilter, ChoiceFilter): pass +class CommonFindingFilters(FilterSet): + # We filter providers from the scan in findings + provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" + ) + provider_type__in = ChoiceInFilter( + choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" + ) + provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact") + provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in") + provider_uid__icontains = CharFilter( + field_name="scan__provider__uid", lookup_expr="icontains" + ) + provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact") + provider_alias__in = CharInFilter( + field_name="scan__provider__alias", lookup_expr="in" + ) + provider_alias__icontains = CharFilter( + field_name="scan__provider__alias", lookup_expr="icontains" + ) + + updated_at = DateFilter(field_name="updated_at", lookup_expr="date") + + uid = CharFilter(field_name="uid") + delta = ChoiceFilter(choices=Finding.DeltaChoices.choices) + status = ChoiceFilter(choices=StatusChoices.choices) + severity = ChoiceFilter(choices=SeverityChoices) + impact = ChoiceFilter(choices=SeverityChoices) + muted = BooleanFilter( + help_text="If this filter is not provided, muted and non-muted findings will be returned." + ) + + resources = UUIDInFilter(field_name="resource__id", lookup_expr="in") + + region = CharFilter(method="filter_resource_region") + region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap") + region__icontains = CharFilter( + field_name="resource_regions", lookup_expr="icontains" + ) + + service = CharFilter(method="filter_resource_service") + service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap") + service__icontains = CharFilter( + field_name="resource_services", lookup_expr="icontains" + ) + + resource_uid = CharFilter(field_name="resources__uid") + resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in") + resource_uid__icontains = CharFilter( + field_name="resources__uid", lookup_expr="icontains" + ) + + resource_name = CharFilter(field_name="resources__name") + resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in") + resource_name__icontains = CharFilter( + field_name="resources__name", lookup_expr="icontains" + ) + + resource_type = CharFilter(method="filter_resource_type") + resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap") + resource_type__icontains = CharFilter( + field_name="resources__type", lookup_expr="icontains" + ) + + # Temporarily disabled until we implement tag filtering in the UI + # resource_tag_key = CharFilter(field_name="resources__tags__key") + # resource_tag_key__in = CharInFilter( + # field_name="resources__tags__key", lookup_expr="in" + # ) + # resource_tag_key__icontains = CharFilter( + # field_name="resources__tags__key", lookup_expr="icontains" + # ) + # resource_tag_value = CharFilter(field_name="resources__tags__value") + # resource_tag_value__in = CharInFilter( + # field_name="resources__tags__value", lookup_expr="in" + # ) + # resource_tag_value__icontains = CharFilter( + # field_name="resources__tags__value", lookup_expr="icontains" + # ) + # resource_tags = CharInFilter( + # method="filter_resource_tag", + # lookup_expr="in", + # help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be " + # "separated by commas.", + # ) + + def filter_resource_service(self, queryset, name, value): + return queryset.filter(resource_services__contains=[value]) + + def filter_resource_region(self, queryset, name, value): + return queryset.filter(resource_regions__contains=[value]) + + def filter_resource_type(self, queryset, name, value): + return queryset.filter(resource_types__contains=[value]) + + def filter_resource_tag(self, queryset, name, value): + overall_query = Q() + for key_value_pair in value: + tag_key, tag_value = key_value_pair.split(":", 1) + overall_query |= Q( + resources__tags__key__icontains=tag_key, + resources__tags__value__icontains=tag_value, + ) + return queryset.filter(overall_query).distinct() + + class TenantFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") updated_at = DateFilter(field_name="updated_at", lookup_expr="date") @@ -257,94 +365,7 @@ class ResourceFilter(ProviderRelationshipFilterSet): return queryset.filter(tags__text_search=value) -class FindingFilter(FilterSet): - # We filter providers from the scan in findings - provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") - provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") - provider_type = ChoiceFilter( - choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" - ) - provider_type__in = ChoiceInFilter( - choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" - ) - provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact") - provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in") - provider_uid__icontains = CharFilter( - field_name="scan__provider__uid", lookup_expr="icontains" - ) - provider_alias = CharFilter(field_name="scan__provider__alias", lookup_expr="exact") - provider_alias__in = CharInFilter( - field_name="scan__provider__alias", lookup_expr="in" - ) - provider_alias__icontains = CharFilter( - field_name="scan__provider__alias", lookup_expr="icontains" - ) - - updated_at = DateFilter(field_name="updated_at", lookup_expr="date") - - uid = CharFilter(field_name="uid") - delta = ChoiceFilter(choices=Finding.DeltaChoices.choices) - status = ChoiceFilter(choices=StatusChoices.choices) - severity = ChoiceFilter(choices=SeverityChoices) - impact = ChoiceFilter(choices=SeverityChoices) - muted = BooleanFilter( - help_text="If this filter is not provided, muted and non-muted findings will be returned." - ) - - resources = UUIDInFilter(field_name="resource__id", lookup_expr="in") - - region = CharFilter(method="filter_resource_region") - region__in = CharInFilter(field_name="resource_regions", lookup_expr="overlap") - region__icontains = CharFilter( - field_name="resource_regions", lookup_expr="icontains" - ) - - service = CharFilter(method="filter_resource_service") - service__in = CharInFilter(field_name="resource_services", lookup_expr="overlap") - service__icontains = CharFilter( - field_name="resource_services", lookup_expr="icontains" - ) - - resource_uid = CharFilter(field_name="resources__uid") - resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in") - resource_uid__icontains = CharFilter( - field_name="resources__uid", lookup_expr="icontains" - ) - - resource_name = CharFilter(field_name="resources__name") - resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in") - resource_name__icontains = CharFilter( - field_name="resources__name", lookup_expr="icontains" - ) - - resource_type = CharFilter(method="filter_resource_type") - resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap") - resource_type__icontains = CharFilter( - field_name="resources__type", lookup_expr="icontains" - ) - - # Temporarily disabled until we implement tag filtering in the UI - # resource_tag_key = CharFilter(field_name="resources__tags__key") - # resource_tag_key__in = CharInFilter( - # field_name="resources__tags__key", lookup_expr="in" - # ) - # resource_tag_key__icontains = CharFilter( - # field_name="resources__tags__key", lookup_expr="icontains" - # ) - # resource_tag_value = CharFilter(field_name="resources__tags__value") - # resource_tag_value__in = CharInFilter( - # field_name="resources__tags__value", lookup_expr="in" - # ) - # resource_tag_value__icontains = CharFilter( - # field_name="resources__tags__value", lookup_expr="icontains" - # ) - # resource_tags = CharInFilter( - # method="filter_resource_tag", - # lookup_expr="in", - # help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be " - # "separated by commas.", - # ) - +class FindingFilter(CommonFindingFilters): scan = UUIDFilter(method="filter_scan_id") scan__in = UUIDInFilter(method="filter_scan_id_in") @@ -512,16 +533,6 @@ class FindingFilter(FilterSet): return queryset.filter(id__lt=end) - def filter_resource_tag(self, queryset, name, value): - overall_query = Q() - for key_value_pair in value: - tag_key, tag_value = key_value_pair.split(":", 1) - overall_query |= Q( - resources__tags__key__icontains=tag_key, - resources__tags__value__icontains=tag_value, - ) - return queryset.filter(overall_query).distinct() - @staticmethod def maybe_date_to_datetime(value): dt = value @@ -530,6 +541,31 @@ class FindingFilter(FilterSet): return dt +class LatestFindingFilter(CommonFindingFilters): + class Meta: + model = Finding + fields = { + "id": ["exact", "in"], + "uid": ["exact", "in"], + "delta": ["exact", "in"], + "status": ["exact", "in"], + "severity": ["exact", "in"], + "impact": ["exact", "in"], + "check_id": ["exact", "in", "icontains"], + } + filter_overrides = { + FindingDeltaEnumField: { + "filter_class": CharFilter, + }, + StatusEnumField: { + "filter_class": CharFilter, + }, + SeverityEnumField: { + "filter_class": CharFilter, + }, + } + + class ProviderSecretFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") updated_at = DateFilter(field_name="updated_at", lookup_expr="date") diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 534a232ca8..4a5bdb4662 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1238,6 +1238,416 @@ paths: schema: $ref: '#/components/schemas/FindingDynamicFilterResponse' description: '' + /api/v1/findings/latest: + get: + operationId: findings_latest_retrieve + description: Retrieve a list of the latest findings from the latest scans for + each provider with options for filtering by various criteria. + summary: List the latest findings + parameters: + - in: query + name: fields[findings] + schema: + type: array + items: + type: string + enum: + - uid + - delta + - status + - status_extended + - severity + - check_id + - check_metadata + - raw_result + - inserted_at + - updated_at + - first_seen_at + - muted + - url + - scan + - resources + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[check_id] + schema: + type: string + - in: query + name: filter[check_id__icontains] + schema: + type: string + - in: query + name: filter[check_id__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[delta] + schema: + type: string + nullable: true + enum: + - changed + - new + description: |- + * `new` - New + * `changed` - Changed + - in: query + name: filter[delta__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[id] + schema: + type: string + format: uuid + - in: query + name: filter[id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[impact] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[impact__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[muted] + schema: + type: boolean + description: If this filter is not provided, muted and non-muted findings + will be returned. + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_name] + schema: + type: string + - in: query + name: filter[resource_name__icontains] + schema: + type: string + - in: query + name: filter[resource_name__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_type] + schema: + type: string + - in: query + name: filter[resource_type__icontains] + schema: + type: string + - in: query + name: filter[resource_type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_uid] + schema: + type: string + - in: query + name: filter[resource_uid__icontains] + schema: + type: string + - in: query + name: filter[resource_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resources] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[severity] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[severity__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[status] + schema: + type: string + enum: + - FAIL + - MANUAL + - PASS + description: |- + * `FAIL` - Fail + * `PASS` - Pass + * `MANUAL` - Manual + - in: query + name: filter[status__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - scan + - resources + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - status + - -status + - severity + - -severity + - check_id + - -check_id + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Finding + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/FindingResponse' + description: '' /api/v1/findings/metadata: get: operationId: findings_metadata_retrieve @@ -1674,6 +2084,392 @@ paths: schema: $ref: '#/components/schemas/FindingMetadataResponse' description: '' + /api/v1/findings/metadata/latest: + get: + operationId: findings_metadata_latest_retrieve + description: Fetch unique metadata values from a set of findings from the latest + scans for each provider. This is useful for dynamic filtering. + summary: Retrieve metadata values from the latest findings + parameters: + - in: query + name: fields[findings-metadata] + schema: + type: array + items: + type: string + enum: + - services + - regions + - resource_types + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[check_id] + schema: + type: string + - in: query + name: filter[check_id__icontains] + schema: + type: string + - in: query + name: filter[check_id__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[delta] + schema: + type: string + nullable: true + enum: + - changed + - new + description: |- + * `new` - New + * `changed` - Changed + - in: query + name: filter[delta__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[id] + schema: + type: string + format: uuid + - in: query + name: filter[id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[impact] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[impact__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[muted] + schema: + type: boolean + description: If this filter is not provided, muted and non-muted findings + will be returned. + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_name] + schema: + type: string + - in: query + name: filter[resource_name__icontains] + schema: + type: string + - in: query + name: filter[resource_name__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_type] + schema: + type: string + - in: query + name: filter[resource_type__icontains] + schema: + type: string + - in: query + name: filter[resource_type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_uid] + schema: + type: string + - in: query + name: filter[resource_uid__icontains] + schema: + type: string + - in: query + name: filter[resource_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resources] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[severity] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[severity__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[status] + schema: + type: string + enum: + - FAIL + - MANUAL + - PASS + description: |- + * `FAIL` - Fail + * `PASS` - Pass + * `MANUAL` - Manual + - in: query + name: filter[status__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - status + - -status + - severity + - -severity + - check_id + - -check_id + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Finding + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/FindingMetadataResponse' + description: '' /api/v1/integrations: get: operationId: integrations_list diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 0608c4a378..a82348fe87 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -3185,6 +3185,29 @@ class TestFindingViewSet: ] } + def test_findings_latest(self, authenticated_client, latest_scan_finding): + response = authenticated_client.get( + reverse("finding-latest"), + ) + assert response.status_code == status.HTTP_200_OK + # The latest scan only has one finding, in comparison with `GET /findings` + assert len(response.json()["data"]) == 1 + assert ( + response.json()["data"][0]["attributes"]["status"] + == latest_scan_finding.status + ) + + def test_findings_metadata_latest(self, authenticated_client, latest_scan_finding): + response = authenticated_client.get( + reverse("finding-metadata_latest"), + ) + assert response.status_code == status.HTTP_200_OK + attributes = response.json()["data"]["attributes"] + + assert attributes["services"] == latest_scan_finding.resource_services + assert attributes["regions"] == latest_scan_finding.resource_regions + assert attributes["resource_types"] == latest_scan_finding.resource_types + @pytest.mark.django_db class TestJWTFields: diff --git a/api/src/backend/api/v1/mixins.py b/api/src/backend/api/v1/mixins.py new file mode 100644 index 0000000000..85250c0eef --- /dev/null +++ b/api/src/backend/api/v1/mixins.py @@ -0,0 +1,33 @@ +from rest_framework.response import Response + + +class PaginateByPkMixin: + """ + Mixin to paginate on a list of PKs (cheaper than heavy JOINs), + re-fetch the full objects with the desired select/prefetch, + re-sort them to preserve DB ordering, then serialize + return. + """ + + def paginate_by_pk( + self, + request, # noqa: F841 + base_queryset, + manager, + select_related: list[str] | None = None, + prefetch_related: list[str] | None = None, + ) -> Response: + pk_list = base_queryset.values_list("id", flat=True) + page = self.paginate_queryset(pk_list) + if page is None: + return Response(self.get_serializer(base_queryset, many=True).data) + + queryset = manager.filter(id__in=page) + if select_related: + queryset = queryset.select_related(*select_related) + if prefetch_related: + queryset = queryset.prefetch_related(*prefetch_related) + + queryset = sorted(queryset, key=lambda obj: page.index(obj.id)) + + serialized = self.get_serializer(queryset, many=True).data + return self.get_paginated_response(serialized) diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 19b21df92e..4bf8cfe2f6 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -65,6 +65,7 @@ from api.filters import ( FindingFilter, IntegrationFilter, InvitationFilter, + LatestFindingFilter, MembershipFilter, ProviderFilter, ProviderGroupFilter, @@ -110,6 +111,7 @@ from api.utils import ( validate_invitation, ) from api.uuid_utils import datetime_to_uuid7, uuid7_start +from api.v1.mixins import PaginateByPkMixin from api.v1.serializers import ( ComplianceOverviewFullSerializer, ComplianceOverviewMetadataSerializer, @@ -1671,10 +1673,24 @@ class ResourceViewSet(BaseRLSViewSet): ], filters=True, ), + latest=extend_schema( + tags=["Finding"], + summary="List the latest findings", + description="Retrieve a list of the latest findings from the latest scans for each provider with options for " + "filtering by various criteria.", + filters=True, + ), + metadata_latest=extend_schema( + tags=["Finding"], + summary="Retrieve metadata values from the latest findings", + description="Fetch unique metadata values from a set of findings from the latest scans for each provider. " + "This is useful for dynamic filtering.", + filters=True, + ), ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") -class FindingViewSet(BaseRLSViewSet): +class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): queryset = Finding.all_objects.all() serializer_class = FindingSerializer filterset_class = FindingFilter @@ -1706,11 +1722,16 @@ class FindingViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "findings_services_regions": return FindingDynamicFilterSerializer - elif self.action == "metadata": + elif self.action in ["metadata", "metadata_latest"]: return FindingMetadataSerializer return super().get_serializer_class() + def get_filterset_class(self): + if self.action in ["latest", "metadata_latest"]: + return LatestFindingFilter + return FindingFilter + def get_queryset(self): tenant_id = self.request.tenant_id user_roles = get_role(self.request.user) @@ -1750,21 +1771,14 @@ class FindingViewSet(BaseRLSViewSet): return super().filter_queryset(queryset) def list(self, request, *args, **kwargs): - base_qs = self.filter_queryset(self.get_queryset()) - paginated_ids = self.paginate_queryset(base_qs.values_list("id", flat=True)) - if paginated_ids is not None: - ids = list(paginated_ids) - findings = ( - Finding.all_objects.filter(tenant_id=self.request.tenant_id, id__in=ids) - .select_related("scan") - .prefetch_related("resources") - ) - # Re-sort in Python to preserve ordering: - findings = sorted(findings, key=lambda x: ids.index(x.id)) - serializer = self.get_serializer(findings, many=True) - return self.get_paginated_response(serializer.data) - serializer = self.get_serializer(base_qs, many=True) - return Response(serializer.data) + filtered_queryset = self.filter_queryset(self.get_queryset()) + return self.paginate_by_pk( + request, + filtered_queryset, + manager=Finding.all_objects, + select_related=["scan"], + prefetch_related=["resources"], + ) @action(detail=False, methods=["get"], url_name="findings_services_regions") def findings_services_regions(self, request): @@ -1897,6 +1911,109 @@ class FindingViewSet(BaseRLSViewSet): serializer.is_valid(raise_exception=True) return Response(serializer.data) + @action(detail=False, methods=["get"], url_name="latest") + def latest(self, request): + tenant_id = request.tenant_id + filtered_queryset = self.filter_queryset(self.get_queryset()) + + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + filtered_queryset = filtered_queryset.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + + return self.paginate_by_pk( + request, + filtered_queryset, + manager=Finding.all_objects, + select_related=["scan"], + prefetch_related=["resources"], + ) + + @action( + detail=False, + methods=["get"], + url_name="metadata_latest", + url_path="metadata/latest", + ) + def metadata_latest(self, request): + # Force filter validation + filtered_queryset = self.filter_queryset(self.get_queryset()) + + tenant_id = request.tenant_id + query_params = request.query_params + + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + + queryset = ResourceScanSummary.objects.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + # ToRemove: Temporary fallback mechanism + scans_with_flag = latest_scan_ids.annotate( + has_summary=Exists( + ResourceScanSummary.objects.filter( + tenant_id=tenant_id, + scan_id=OuterRef("pk"), + ) + ) + ) + if missing_scan_ids := scans_with_flag.filter(has_summary=False).values_list( + "id", flat=True + ): + for scan_id in missing_scan_ids: + backfill_scan_resource_summaries_task.apply_async( + kwargs={"tenant_id": tenant_id, "scan_id": scan_id} + ) + return Response( + get_findings_metadata_no_aggregations(tenant_id, filtered_queryset) + ) + + if service_filter := query_params.get("filter[service]") or query_params.get( + "filter[service__in]" + ): + queryset = queryset.filter(service__in=service_filter.split(",")) + if region_filter := query_params.get("filter[region]") or query_params.get( + "filter[region__in]" + ): + queryset = queryset.filter(region__in=region_filter.split(",")) + if resource_type_filter := query_params.get( + "filter[resource_type]" + ) or query_params.get("filter[resource_type__in]"): + queryset = queryset.filter( + resource_type__in=resource_type_filter.split(",") + ) + + services = list( + queryset.values_list("service", flat=True).distinct().order_by("service") + ) + regions = list( + queryset.values_list("region", flat=True).distinct().order_by("region") + ) + resource_types = list( + queryset.values_list("resource_type", flat=True) + .distinct() + .order_by("resource_type") + ) + + result = { + "services": services, + "regions": regions, + "resource_types": resource_types, + } + + serializer = self.get_serializer(data=result) + serializer.is_valid(raise_exception=True) + return Response(serializer.data) + @extend_schema_view( list=extend_schema( diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 2fba4cebb7..8f58c447de 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -929,6 +929,47 @@ def backfill_scan_metadata_fixture(scans_fixture, findings_fixture): backfill_resource_scan_summaries(tenant_id=tenant_id, scan_id=scan_id) +@pytest.fixture(scope="function") +def latest_scan_finding(authenticated_client, providers_fixture, resources_fixture): + provider = providers_fixture[0] + tenant_id = str(providers_fixture[0].tenant_id) + resource = resources_fixture[0] + scan = Scan.objects.create( + name="latest completed scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant_id, + ) + finding = Finding.objects.create( + tenant_id=tenant_id, + uid="test_finding_uid_1", + scan=scan, + delta="new", + status=Status.FAIL, + status_extended="test status extended ", + impact=Severity.critical, + impact_extended="test impact extended one", + severity=Severity.critical, + raw_result={ + "status": Status.FAIL, + "impact": Severity.critical, + "severity": Severity.critical, + }, + tags={"test": "dev-qa"}, + check_id="test_check_id", + check_metadata={ + "CheckId": "test_check_id", + "Description": "test description apple sauce", + }, + first_seen_at="2024-01-02T00:00:00Z", + ) + + finding.add_resources([resource]) + backfill_resource_scan_summaries(tenant_id, str(scan.id)) + return finding + + def get_authorization_header(access_token: str) -> dict: return {"Authorization": f"Bearer {access_token}"}