diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 6c43a2e582..814d3b4d0d 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.15.0] (Prowler UNRELEASED) ### Added +- Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) +- New endpoint to retrieve the number of providers grouped by provider type [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) - Support for configuring multiple LLM providers [(#8772)](https://github.com/prowler-cloud/prowler/pull/8772) ## [1.14.0] (Prowler 5.13.0) diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 26e5baa782..ef1387f0bd 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -247,6 +247,14 @@ class ProviderFilter(FilterSet): choices=Provider.ProviderChoices.choices, lookup_expr="in", ) + provider_type = ChoiceFilter( + choices=Provider.ProviderChoices.choices, field_name="provider" + ) + provider_type__in = ChoiceInFilter( + field_name="provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) class Meta: model = Provider diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index d3f68719a1..b6596768d5 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -4536,6 +4536,37 @@ paths: schema: $ref: '#/components/schemas/OverviewProviderResponse' description: '' + /api/v1/overviews/providers/count: + get: + operationId: overviews_providers_count_retrieve + description: Retrieve the number of providers grouped by provider type. This + endpoint counts every provider in the tenant, including those without completed + scans. + summary: Get provider counts grouped by type + parameters: + - in: query + name: fields[providers-count-overview] + schema: + type: array + items: + type: string + enum: + - id + - count + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewProviderCountResponse' + description: '' /api/v1/overviews/services: get: operationId: overviews_services_retrieve @@ -5362,6 +5393,50 @@ paths: * `github` - GitHub explode: false style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 4c1e219dad1cc0e7 + enum: + - aws + - azure + - gcp + - github + - kubernetes + - m365 + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 4c1e219dad1cc0e7 + enum: + - aws + - azure + - gcp + - github + - kubernetes + - m365 + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + explode: false + style: form - name: filter[search] required: false in: query @@ -11975,6 +12050,38 @@ components: readOnly: true required: - id + OverviewProviderCount: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - providers-count-overview + id: {} + attributes: + type: object + properties: + id: + type: string + count: + type: integer + required: + - id + - count + OverviewProviderCountResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewProviderCount' + required: + - data OverviewProviderResponse: type: object properties: diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 94ce2d0ed2..eec52347c1 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -23,6 +23,7 @@ from conftest import ( today_after_n_days, ) from django.conf import settings +from django.db.models import Count from django.http import JsonResponse from django.test import RequestFactory from django.urls import reverse @@ -946,6 +947,74 @@ class TestProviderViewSet: assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(providers_fixture) + def test_providers_filter_provider_type( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type]": "aws"} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert all(item["attributes"]["provider"] == "aws" for item in data) + + def test_providers_filter_provider_type_in( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type__in]": "aws,gcp"} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 3 + assert {"aws", "gcp"} >= {item["attributes"]["provider"] for item in data} + + def test_providers_filter_provider_type_invalid( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type]": "invalid"} + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_providers_disable_pagination( + self, authenticated_client, providers_fixture, tenants_fixture + ): + tenant, *_ = tenants_fixture + existing_count = Provider.objects.filter(tenant_id=tenant.id).count() + target_total = settings.REST_FRAMEWORK["PAGE_SIZE"] + 1 + additional_needed = max(0, target_total - existing_count) + + base_uid = 200000000000 + for index in range(additional_needed): + Provider.objects.create( + tenant_id=tenant.id, + provider=Provider.ProviderChoices.AWS, + uid=f"{base_uid + index:012d}", + alias=f"aws_extra_{index}", + ) + + total_providers = Provider.objects.filter(tenant_id=tenant.id).count() + + paginated_response = authenticated_client.get(reverse("provider-list")) + assert paginated_response.status_code == status.HTTP_200_OK + paginated_data = paginated_response.json()["data"] + assert len(paginated_data) == min( + settings.REST_FRAMEWORK["PAGE_SIZE"], total_providers + ) + paginated_meta = paginated_response.json().get("meta", {}) + assert "pagination" in paginated_meta + assert paginated_meta["pagination"]["count"] == total_providers + + unpaginated_response = authenticated_client.get( + reverse("provider-list"), {"page[disable]": "true"} + ) + assert unpaginated_response.status_code == status.HTTP_200_OK + unpaginated_data = unpaginated_response.json()["data"] + assert len(unpaginated_data) == total_providers + unpaginated_meta = unpaginated_response.json().get("meta", {}) + assert "pagination" not in unpaginated_meta + @pytest.mark.parametrize( "include_values, expected_resources", [ @@ -5787,6 +5856,40 @@ class TestOverviewViewSet: # Since we rely on completed scans, there are only 2 resources now assert response.json()["data"][0]["attributes"]["resources"]["total"] == 2 + def test_overview_providers_count( + self, + authenticated_client, + scan_summaries_fixture, + resources_fixture, + providers_fixture, + tenants_fixture, + ): + tenant = tenants_fixture[0] + + default_response = authenticated_client.get(reverse("overview-providers")) + assert default_response.status_code == status.HTTP_200_OK + default_data = default_response.json()["data"] + assert len(default_data) == 1 + assert all("count" not in item["attributes"] for item in default_data) + grouped_response = authenticated_client.get(reverse("overview-providers-count")) + assert grouped_response.status_code == status.HTTP_200_OK + grouped_data = grouped_response.json()["data"] + assert len(grouped_data) >= 1 + + aggregated = { + entry["id"]: entry["attributes"]["count"] for entry in grouped_data + } + db_counts = ( + Provider.objects.filter(tenant_id=tenant.id, is_deleted=False) + .values("provider") + .annotate(count=Count("id")) + ) + expected = {row["provider"]: row["count"] for row in db_counts} + + assert aggregated == expected + for entry in grouped_data: + assert "findings" not in entry["attributes"] + def test_overview_services_list_no_required_filters( self, authenticated_client, scan_summaries_fixture ): diff --git a/api/src/backend/api/v1/mixins.py b/api/src/backend/api/v1/mixins.py index 1cef76a84c..e1a5d3470f 100644 --- a/api/src/backend/api/v1/mixins.py +++ b/api/src/backend/api/v1/mixins.py @@ -12,6 +12,24 @@ from api.models import StateChoices, Task from api.v1.serializers import TaskSerializer +class DisablePaginationMixin: + disable_pagination_query_param = "page[disable]" + disable_pagination_truthy_values = {"true"} + + def should_disable_pagination(self) -> bool: + if not hasattr(self, "request"): + return False + value = self.request.query_params.get(self.disable_pagination_query_param) + if value is None: + return False + return str(value).lower() in self.disable_pagination_truthy_values + + def paginate_queryset(self, queryset): + if self.should_disable_pagination(): + return None + return super().paginate_queryset(queryset) + + class PaginateByPkMixin: """ Mixin to paginate on a list of PKs (cheaper than heavy JOINs), diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 511b1f89e4..d99fd8de58 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -2105,6 +2105,17 @@ class OverviewProviderSerializer(serializers.Serializer): } +class OverviewProviderCountSerializer(serializers.Serializer): + id = serializers.CharField(source="provider") + count = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "providers-count-overview" + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + class OverviewFindingSerializer(serializers.Serializer): id = serializers.CharField(default="n/a") new = serializers.IntegerField() diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 2e8a78207e..d304ae6b64 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -147,7 +147,7 @@ from api.utils import ( validate_invitation, ) from api.uuid_utils import datetime_to_uuid7, uuid7_start -from api.v1.mixins import PaginateByPkMixin, TaskManagementMixin +from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin from api.v1.serializers import ( ComplianceOverviewAttributesSerializer, ComplianceOverviewDetailSerializer, @@ -176,6 +176,7 @@ from api.v1.serializers import ( LighthouseTenantConfigUpdateSerializer, MembershipSerializer, OverviewFindingSerializer, + OverviewProviderCountSerializer, OverviewProviderSerializer, OverviewServiceSerializer, OverviewSeveritySerializer, @@ -1431,7 +1432,7 @@ class ProviderGroupProvidersRelationshipView(RelationshipView, BaseRLSViewSet): ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") -class ProviderViewSet(BaseRLSViewSet): +class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): queryset = Provider.objects.all() serializer_class = ProviderSerializer http_method_names = ["get", "post", "patch", "delete"] @@ -3691,6 +3692,13 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): "each provider are considered in the aggregation to ensure accurate and up-to-date insights." ), ), + providers_count=extend_schema( + summary="Get provider counts grouped by type", + description=( + "Retrieve the number of providers grouped by provider type. " + "This endpoint counts every provider in the tenant, including those without completed scans." + ), + ), findings=extend_schema( summary="Get aggregated findings data", description=( @@ -3742,6 +3750,8 @@ class OverviewViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "providers": return OverviewProviderSerializer + elif self.action == "providers_count": + return OverviewProviderCountSerializer elif self.action == "findings": return OverviewFindingSerializer elif self.action == "findings_severity": @@ -3829,6 +3839,36 @@ class OverviewViewSet(BaseRLSViewSet): status=status.HTTP_200_OK, ) + @action( + detail=False, + methods=["get"], + url_path="providers/count", + url_name="providers-count", + ) + def providers_count(self, request): + tenant_id = self.request.tenant_id + providers_qs = Provider.objects.filter(tenant_id=tenant_id) + + if hasattr(self, "allowed_providers"): + allowed_ids = list(self.allowed_providers.values_list("id", flat=True)) + if not allowed_ids: + overview = [] + return Response( + self.get_serializer(overview, many=True).data, + status=status.HTTP_200_OK, + ) + providers_qs = providers_qs.filter(id__in=allowed_ids) + + overview = ( + providers_qs.values("provider") + .annotate(count=Count("id")) + .order_by("provider") + ) + return Response( + self.get_serializer(overview, many=True).data, + status=status.HTTP_200_OK, + ) + @action(detail=False, methods=["get"], url_name="findings") def findings(self, request): tenant_id = self.request.tenant_id