feat(findings): add /findings/metadata to retrieve dynamic filters information (#6503)

This commit is contained in:
Víctor Fernández Poyatos
2025-01-14 15:30:03 +01:00
committed by GitHub
parent d7d9118b9b
commit 1846535d8d
4 changed files with 604 additions and 112 deletions

View File

@@ -1,7 +1,7 @@
openapi: 3.0.3
info:
title: Prowler API
version: 1.1.0
version: 1.2.0
description: |-
Prowler API specification.
@@ -1167,12 +1167,439 @@ paths:
- Finding
security:
- jwtAuth: []
deprecated: true
responses:
'201':
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/OpenApiResponseResponse'
$ref: '#/components/schemas/FindingDynamicFilterResponse'
description: ''
/api/v1/findings/metadata:
get:
operationId: findings_metadata_retrieve
description: Fetch unique metadata values from a set of findings. This is useful
for dynamic filtering.
summary: Retrieve metadata values from findings
parameters:
- in: query
name: fields[findings-metadata]
schema:
type: array
items:
type: string
enum:
- services
- regions
- resource_types
- tags
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[check_id]
schema:
type: string
- in: query
name: filter[check_id__icontains]
schema:
type: string
- in: query
name: filter[check_id__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[delta]
schema:
type: string
nullable: true
enum:
- changed
- new
description: |-
* `new` - New
* `changed` - Changed
- in: query
name: filter[delta__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[id]
schema:
type: string
format: uuid
- in: query
name: filter[id__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[impact]
schema:
type: string
enum:
- critical
- high
- informational
- low
- medium
description: |-
* `critical` - Critical
* `high` - High
* `medium` - Medium
* `low` - Low
* `informational` - Informational
- in: query
name: filter[impact__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[inserted_at]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__date]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__gte]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__lte]
schema:
type: string
format: date
- in: query
name: filter[provider]
schema:
type: string
format: uuid
- in: query
name: filter[provider__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_alias]
schema:
type: string
- in: query
name: filter[provider_alias__icontains]
schema:
type: string
- in: query
name: filter[provider_alias__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_type]
schema:
type: string
enum:
- aws
- azure
- gcp
- kubernetes
description: |-
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
- in: query
name: filter[provider_type__in]
schema:
type: array
items:
type: string
enum:
- aws
- azure
- gcp
- kubernetes
description: |-
Multiple values may be separated by commas.
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
explode: false
style: form
- in: query
name: filter[provider_uid]
schema:
type: string
- in: query
name: filter[provider_uid__icontains]
schema:
type: string
- in: query
name: filter[provider_uid__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[region]
schema:
type: string
- in: query
name: filter[region__icontains]
schema:
type: string
- in: query
name: filter[region__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_name]
schema:
type: string
- in: query
name: filter[resource_name__icontains]
schema:
type: string
- in: query
name: filter[resource_name__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_type]
schema:
type: string
- in: query
name: filter[resource_type__icontains]
schema:
type: string
- in: query
name: filter[resource_type__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resource_uid]
schema:
type: string
- in: query
name: filter[resource_uid__icontains]
schema:
type: string
- in: query
name: filter[resource_uid__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[resources]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[scan]
schema:
type: string
format: uuid
- in: query
name: filter[scan__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- name: filter[search]
required: false
in: query
description: A search term.
schema:
type: string
- in: query
name: filter[service]
schema:
type: string
- in: query
name: filter[service__icontains]
schema:
type: string
- in: query
name: filter[service__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[severity]
schema:
type: string
enum:
- critical
- high
- informational
- low
- medium
description: |-
* `critical` - Critical
* `high` - High
* `medium` - Medium
* `low` - Low
* `informational` - Informational
- in: query
name: filter[severity__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[status]
schema:
type: string
enum:
- FAIL
- MANUAL
- MUTED
- PASS
description: |-
* `FAIL` - Fail
* `PASS` - Pass
* `MANUAL` - Manual
* `MUTED` - Muted
- in: query
name: filter[status__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[uid]
schema:
type: string
- in: query
name: filter[uid__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[updated_at]
schema:
type: string
format: date
- in: query
name: filter[updated_at__gte]
schema:
type: string
format: date-time
- in: query
name: filter[updated_at__lte]
schema:
type: string
format: date-time
- name: sort
required: false
in: query
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
schema:
type: array
items:
type: string
enum:
- id
- -id
- status
- -status
- severity
- -severity
- check_id
- -check_id
- inserted_at
- -inserted_at
- updated_at
- -updated_at
explode: false
tags:
- Finding
security:
- jwtAuth: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/FindingMetadataResponse'
description: ''
/api/v1/invitations/accept:
post:
@@ -2948,9 +3375,7 @@ paths:
- name
- manage_users
- manage_account
- manage_billing
- manage_providers
- manage_integrations
- manage_scans
- permission_state
- unlimited_visibility
@@ -3068,12 +3493,8 @@ paths:
- -manage_users
- manage_account
- -manage_account
- manage_billing
- -manage_billing
- manage_providers
- -manage_providers
- manage_integrations
- -manage_integrations
- manage_scans
- -manage_scans
- permission_state
@@ -3147,9 +3568,7 @@ paths:
- name
- manage_users
- manage_account
- manage_billing
- manage_providers
- manage_integrations
- manage_scans
- permission_state
- unlimited_visibility
@@ -5458,6 +5877,92 @@ components:
readOnly: true
required:
- scan
FindingDynamicFilter:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/FindingDynamicFilterTypeEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id: {}
attributes:
type: object
properties:
services:
type: array
items:
type: string
regions:
type: array
items:
type: string
required:
- services
- regions
FindingDynamicFilterResponse:
type: object
properties:
data:
$ref: '#/components/schemas/FindingDynamicFilter'
required:
- data
FindingDynamicFilterTypeEnum:
type: string
enum:
- finding-dynamic-filters
FindingMetadata:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/FindingMetadataTypeEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id: {}
attributes:
type: object
properties:
services:
type: array
items:
type: string
regions:
type: array
items:
type: string
resource_types:
type: array
items:
type: string
tags:
description: Tags are described as key-value pairs.
required:
- services
- regions
- resource_types
- tags
FindingMetadataResponse:
type: object
properties:
data:
$ref: '#/components/schemas/FindingMetadata'
required:
- data
FindingMetadataTypeEnum:
type: string
enum:
- findings-metadata
FindingResponse:
type: object
properties:
@@ -5902,8 +6407,6 @@ components:
- data
description: A related resource object from type roles
title: roles
required:
- roles
InvitationUpdateResponse:
type: object
properties:
@@ -5915,7 +6418,6 @@ components:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
@@ -5924,9 +6426,6 @@ components:
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id:
type: string
format: uuid
attributes:
type: object
properties:
@@ -6437,8 +6936,6 @@ components:
- data
description: A related resource object from type roles
title: roles
required:
- roles
required:
- data
PatchedProviderGroupMembershipRequest:
@@ -6850,12 +7347,8 @@ components:
type: boolean
manage_account:
type: boolean
manage_billing:
type: boolean
manage_providers:
type: boolean
manage_integrations:
type: boolean
manage_scans:
type: boolean
permission_state:
@@ -7131,37 +7624,6 @@ components:
required:
- name
- email
relationships:
type: object
properties:
roles:
type: object
properties:
data:
type: array
items:
type: object
properties:
id:
type: string
format: uuid
title: Resource Identifier
description: The identifier of the related object.
type:
type: string
enum:
- roles
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share
common attributes and relationships.
required:
- id
- type
required:
- data
description: A related resource object from type roles
title: roles
required:
- data
Provider:
@@ -8537,12 +8999,8 @@ components:
type: boolean
manage_account:
type: boolean
manage_billing:
type: boolean
manage_providers:
type: boolean
manage_integrations:
type: boolean
manage_scans:
type: boolean
permission_state:
@@ -8670,12 +9128,8 @@ components:
type: boolean
manage_account:
type: boolean
manage_billing:
type: boolean
manage_providers:
type: boolean
manage_integrations:
type: boolean
manage_scans:
type: boolean
permission_state:
@@ -8808,12 +9262,8 @@ components:
type: boolean
manage_account:
type: boolean
manage_billing:
type: boolean
manage_providers:
type: boolean
manage_integrations:
type: boolean
manage_scans:
type: boolean
permission_state:
@@ -9877,37 +10327,6 @@ components:
required:
- name
- email
relationships:
type: object
properties:
roles:
type: object
properties:
data:
type: array
items:
type: object
properties:
id:
type: string
format: uuid
title: Resource Identifier
description: The identifier of the related object.
type:
type: string
enum:
- roles
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
attributes and relationships.
required:
- id
- type
required:
- data
description: A related resource object from type roles
title: roles
UserUpdateResponse:
type: object
properties:

View File

@@ -2582,30 +2582,34 @@ class TestFindingViewSet:
)
assert response.status_code == status.HTTP_404_NOT_FOUND
def test_findings_services_regions_retrieve(
self, authenticated_client, findings_fixture
):
def test_findings_metadata_retrieve(self, authenticated_client, findings_fixture):
finding_1, *_ = findings_fixture
response = authenticated_client.get(
reverse("finding-findings_services_regions"),
reverse("finding-metadata"),
{"filter[inserted_at]": finding_1.updated_at.strftime("%Y-%m-%d")},
)
data = response.json()
expected_services = {"ec2", "s3"}
expected_regions = {"eu-west-1", "us-east-1"}
expected_tags = {"key": "value", "key2": "value2"}
expected_resource_types = {"prowler-test"}
assert data["data"]["type"] == "finding-dynamic-filters"
assert data["data"]["type"] == "findings-metadata"
assert data["data"]["id"] is None
assert set(data["data"]["attributes"]["services"]) == expected_services
assert set(data["data"]["attributes"]["regions"]) == expected_regions
assert (
set(data["data"]["attributes"]["resource_types"]) == expected_resource_types
)
assert data["data"]["attributes"]["tags"] == expected_tags
def test_findings_services_regions_severity_retrieve(
def test_findings_metadata_severity_retrieve(
self, authenticated_client, findings_fixture
):
finding_1, *_ = findings_fixture
response = authenticated_client.get(
reverse("finding-findings_services_regions"),
reverse("finding-metadata"),
{
"filter[severity__in]": ["low", "medium"],
"filter[inserted_at]": finding_1.updated_at.strftime("%Y-%m-%d"),
@@ -2615,26 +2619,34 @@ class TestFindingViewSet:
expected_services = {"s3"}
expected_regions = {"eu-west-1"}
expected_tags = {"key": "value", "key2": "value2"}
expected_resource_types = {"prowler-test"}
assert data["data"]["type"] == "finding-dynamic-filters"
assert data["data"]["type"] == "findings-metadata"
assert data["data"]["id"] is None
assert set(data["data"]["attributes"]["services"]) == expected_services
assert set(data["data"]["attributes"]["regions"]) == expected_regions
assert (
set(data["data"]["attributes"]["resource_types"]) == expected_resource_types
)
assert data["data"]["attributes"]["tags"] == expected_tags
def test_findings_services_regions_future_date(self, authenticated_client):
def test_findings_metadata_future_date(self, authenticated_client):
response = authenticated_client.get(
reverse("finding-findings_services_regions"),
reverse("finding-metadata"),
{"filter[inserted_at]": "2048-01-01"},
)
data = response.json()
assert data["data"]["type"] == "finding-dynamic-filters"
assert data["data"]["type"] == "findings-metadata"
assert data["data"]["id"] is None
assert data["data"]["attributes"]["services"] == []
assert data["data"]["attributes"]["regions"] == []
assert data["data"]["attributes"]["tags"] == {}
assert data["data"]["attributes"]["resource_types"] == []
def test_findings_services_regions_invalid_date(self, authenticated_client):
def test_findings_metadata_invalid_date(self, authenticated_client):
response = authenticated_client.get(
reverse("finding-findings_services_regions"),
reverse("finding-metadata"),
{"filter[inserted_at]": "2048-01-011"},
)
assert response.json() == {

View File

@@ -917,6 +917,7 @@ class FindingSerializer(RLSSerializer):
}
# To be removed when the related endpoint is removed as well
class FindingDynamicFilterSerializer(serializers.Serializer):
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
@@ -925,6 +926,18 @@ class FindingDynamicFilterSerializer(serializers.Serializer):
resource_name = "finding-dynamic-filters"
class FindingMetadataSerializer(serializers.Serializer):
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
resource_types = serializers.ListField(
child=serializers.CharField(), allow_empty=True
)
tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
class Meta:
resource_name = "findings-metadata"
# Provider secrets
class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
@staticmethod

View File

@@ -4,6 +4,7 @@ from django.contrib.postgres.aggregates import ArrayAgg
from django.contrib.postgres.search import SearchQuery
from django.db import transaction
from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum
from django.db.models.functions import JSONObject
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_control
@@ -87,6 +88,7 @@ from api.v1.serializers import (
ComplianceOverviewFullSerializer,
ComplianceOverviewSerializer,
FindingDynamicFilterSerializer,
FindingMetadataSerializer,
FindingSerializer,
InvitationAcceptSerializer,
InvitationCreateSerializer,
@@ -192,7 +194,7 @@ class SchemaView(SpectacularAPIView):
def get(self, request, *args, **kwargs):
spectacular_settings.TITLE = "Prowler API"
spectacular_settings.VERSION = "1.1.1"
spectacular_settings.VERSION = "1.2.0"
spectacular_settings.DESCRIPTION = (
"Prowler API specification.\n\nThis file is auto-generated."
)
@@ -1274,7 +1276,13 @@ class ResourceViewSet(BaseRLSViewSet):
tags=["Finding"],
summary="Retrieve the services and regions that are impacted by findings",
description="Fetch services and regions affected in findings.",
responses={201: OpenApiResponse(response=MembershipSerializer)},
filters=True,
deprecated=True,
),
metadata=extend_schema(
tags=["Finding"],
summary="Retrieve metadata values from findings",
description="Fetch unique metadata values from a set of findings. This is useful for dynamic filtering.",
filters=True,
),
)
@@ -1308,6 +1316,8 @@ class FindingViewSet(BaseRLSViewSet):
def get_serializer_class(self):
if self.action == "findings_services_regions":
return FindingDynamicFilterSerializer
elif self.action == "metadata":
return FindingMetadataSerializer
return super().get_serializer_class()
@@ -1376,6 +1386,44 @@ class FindingViewSet(BaseRLSViewSet):
return Response(data=serializer.data, status=status.HTTP_200_OK)
@action(detail=False, methods=["get"], url_name="metadata")
def metadata(self, request):
queryset = self.get_queryset()
filtered_queryset = self.filter_queryset(queryset)
result = filtered_queryset.aggregate(
services=ArrayAgg("resources__service", flat=True, distinct=True),
regions=ArrayAgg("resources__region", flat=True, distinct=True),
tags=ArrayAgg(
JSONObject(
key=F("resources__tags__key"), value=F("resources__tags__value")
),
distinct=True,
filter=Q(resources__tags__key__isnull=False),
),
resource_types=ArrayAgg("resources__type", flat=True, distinct=True),
)
if result["services"] is None:
result["services"] = []
if result["regions"] is None:
result["regions"] = []
if result["regions"] is None:
result["regions"] = []
if result["resource_types"] is None:
result["resource_types"] = []
if result["tags"] is None:
result["tags"] = []
result["tags"] = {t["key"]: t["value"] for t in result["tags"]}
serializer = self.get_serializer(
data=result,
)
serializer.is_valid(raise_exception=True)
return Response(data=serializer.data, status=status.HTTP_200_OK)
@extend_schema_view(
list=extend_schema(