From 1c6d42e60d120e86d315a72cdf75b61852aa2f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Fern=C3=A1ndez=20Poyatos?= Date: Thu, 21 Nov 2024 17:59:21 +0100 Subject: [PATCH] feat(Overview): PRWLR-5433 Add /overviews/providers endpoint (#88) * feat(Overview): PRWLR-5433 add overviews/providers views and serializers * test(Overview): PRWLR-5433 add unit tests * chore(Schema): update API schema * feat(Overview): PRWLR-5433 order by -findings_failed by default * test(Tenant): PRWLR-5433 fix unit test --- src/backend/api/specs/v1.yaml | 85 +++++++++++++++ .../api/tests/integration/test_tenants.py | 2 +- src/backend/api/tests/test_views.py | 24 +++++ src/backend/api/v1/serializers.py | 47 ++++++++ src/backend/api/v1/urls.py | 2 + src/backend/api/v1/views.py | 100 +++++++++++++++++- 6 files changed, 258 insertions(+), 2 deletions(-) diff --git a/src/backend/api/specs/v1.yaml b/src/backend/api/specs/v1.yaml index a8c926986d..755ab8c8df 100644 --- a/src/backend/api/specs/v1.yaml +++ b/src/backend/api/specs/v1.yaml @@ -780,6 +780,38 @@ paths: schema: $ref: '#/components/schemas/OpenApiResponseResponse' description: '' + /api/v1/overviews/providers: + get: + operationId: overviews_providers_retrieve + description: Fetch aggregated summaries of the latest findings and resources + for each provider. This includes counts of passed, failed, and manual findings, + as well as the total number of resources managed by each provider. + summary: List aggregated overview data for providers + parameters: + - in: query + name: fields[provider-overviews] + schema: + type: array + items: + type: string + enum: + - id + - findings + - resources + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + tags: + - Overview + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewProviderResponse' + description: '' /api/v1/provider_groups: get: operationId: provider_groups_list @@ -4368,6 +4400,56 @@ components: $ref: '#/components/schemas/Membership' required: - data + OverviewProvider: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/OverviewProviderTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + id: + type: string + findings: + type: object + properties: + pass: + type: integer + fail: + type: integer + manual: + type: integer + total: + type: integer + readOnly: true + resources: + type: object + properties: + total: + type: integer + readOnly: true + required: + - id + OverviewProviderResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewProvider' + required: + - data + OverviewProviderTypeEnum: + type: string + enum: + - provider-overviews PaginatedComplianceOverviewList: type: object required: @@ -6858,6 +6940,9 @@ tags: - name: Finding description: Endpoints for managing findings, allowing retrieval and filtering of findings that result from scans. +- name: Overview + description: Endpoints for retrieving aggregated summaries of resources from the + system. - name: Compliance Overview description: Endpoints for checking the compliance overview, allowing filtering by scan, provider or compliance framework ID. diff --git a/src/backend/api/tests/integration/test_tenants.py b/src/backend/api/tests/integration/test_tenants.py index 9bd4545425..ee06f49ba8 100644 --- a/src/backend/api/tests/integration/test_tenants.py +++ b/src/backend/api/tests/integration/test_tenants.py @@ -6,7 +6,7 @@ from django.urls import reverse from conftest import TEST_USER, TEST_PASSWORD, get_api_tokens, get_authorization_header -@patch("tasks.beat.schedule_provider_scan") +@patch("api.v1.views.schedule_provider_scan") @pytest.mark.django_db def test_check_resources_between_different_tenants( schedule_mock, diff --git a/src/backend/api/tests/test_views.py b/src/backend/api/tests/test_views.py index 896486a5e1..79589968bd 100644 --- a/src/backend/api/tests/test_views.py +++ b/src/backend/api/tests/test_views.py @@ -3238,3 +3238,27 @@ class TestComplianceOverviewViewSet: # No filters, now compliance_overview1 has more fails assert len(response.json()["data"]) == 1 assert response.json()["data"][0]["id"] == str(compliance_overview1.id) + + +@pytest.mark.django_db +class TestOverviewViewSet: + def test_overview_list_invalid_method(self, authenticated_client): + response = authenticated_client.put(reverse("overview-list")) + assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + + def test_overview_providers_list( + self, authenticated_client, findings_fixture, resources_fixture + ): + response = authenticated_client.get(reverse("overview-providers")) + assert response.status_code == status.HTTP_200_OK + # Only findings from one provider + assert len(response.json()["data"]) == 1 + assert response.json()["data"][0]["attributes"]["findings"]["total"] == len( + findings_fixture + ) + assert response.json()["data"][0]["attributes"]["findings"]["pass"] == 0 + assert response.json()["data"][0]["attributes"]["findings"]["fail"] == 2 + assert response.json()["data"][0]["attributes"]["findings"]["manual"] == 0 + assert response.json()["data"][0]["attributes"]["resources"]["total"] == len( + resources_fixture + ) diff --git a/src/backend/api/v1/serializers.py b/src/backend/api/v1/serializers.py index 40cacd4108..9b0895adcb 100644 --- a/src/backend/api/v1/serializers.py +++ b/src/backend/api/v1/serializers.py @@ -1223,3 +1223,50 @@ class ComplianceOverviewFullSerializer(ComplianceOverviewSerializer): Returns the detailed structure of requirements. """ return obj.requirements + + +# Overviews + + +class OverviewProviderSerializer(serializers.Serializer): + id = serializers.CharField(source="provider") + findings = serializers.SerializerMethodField(read_only=True) + resources = serializers.SerializerMethodField(read_only=True) + + class JSONAPIMeta: + resource_name = "provider-overviews" + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + @extend_schema_field( + { + "type": "object", + "properties": { + "pass": {"type": "integer"}, + "fail": {"type": "integer"}, + "manual": {"type": "integer"}, + "total": {"type": "integer"}, + }, + } + ) + def get_findings(self, obj): + return { + "pass": obj["findings_passed"], + "fail": obj["findings_failed"], + "manual": obj["findings_manual"], + "total": obj["total_findings"], + } + + @extend_schema_field( + { + "type": "object", + "properties": { + "total": {"type": "integer"}, + }, + } + ) + def get_resources(self, obj): + return { + "total": obj["total_resources"], + } diff --git a/src/backend/api/v1/urls.py b/src/backend/api/v1/urls.py index 6978edda29..c212c95e06 100644 --- a/src/backend/api/v1/urls.py +++ b/src/backend/api/v1/urls.py @@ -19,6 +19,7 @@ from api.v1.views import ( ProviderSecretViewSet, InvitationViewSet, InvitationAcceptViewSet, + OverviewViewSet, ComplianceOverviewViewSet, ) @@ -35,6 +36,7 @@ router.register(r"findings", FindingViewSet, basename="finding") router.register( r"compliance-overviews", ComplianceOverviewViewSet, basename="complianceoverview" ) +router.register(r"overviews", OverviewViewSet, basename="overview") tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant") tenants_router.register( diff --git a/src/backend/api/v1/views.py b/src/backend/api/v1/views.py index 418b242ba8..70af1069ab 100644 --- a/src/backend/api/v1/views.py +++ b/src/backend/api/v1/views.py @@ -2,7 +2,7 @@ from celery.result import AsyncResult from django.conf import settings as django_settings from django.contrib.postgres.search import SearchQuery from django.db import transaction -from django.db.models import F, Q, Prefetch, OuterRef, Subquery +from django.db.models import Prefetch, Subquery, OuterRef, Count, Q, F from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control @@ -45,6 +45,7 @@ from api.filters import ( ComplianceOverviewFilter, ) from api.models import ( + StatusChoices, User, Membership, Provider, @@ -91,6 +92,7 @@ from api.v1.serializers import ( InvitationAcceptSerializer, ComplianceOverviewSerializer, ComplianceOverviewFullSerializer, + OverviewProviderSerializer, ) from tasks.beat import schedule_provider_scan from tasks.tasks import ( @@ -202,6 +204,10 @@ class SchemaView(SpectacularAPIView): "description": "Endpoints for managing findings, allowing retrieval and filtering of " "findings that result from scans.", }, + { + "name": "Overview", + "description": "Endpoints for retrieving aggregated summaries of resources from the system.", + }, { "name": "Compliance Overview", "description": "Endpoints for checking the compliance overview, allowing filtering by scan, provider or" @@ -1284,3 +1290,95 @@ class ComplianceOverviewViewSet(BaseRLSViewSet): ] ) return super().list(request, *args, **kwargs) + + +@extend_schema(tags=["Overview"]) +@extend_schema_view( + providers=extend_schema( + summary="List aggregated overview data for providers", + description="Fetch aggregated summaries of the latest findings and resources for each provider. " + "This includes counts of passed, failed, and manual findings, as well as the total number " + "of resources managed by each provider.", + ), +) +@method_decorator(CACHE_DECORATOR, name="list") +class OverviewViewSet(BaseRLSViewSet): + queryset = ComplianceOverview.objects.all() + http_method_names = ["get"] + ordering = ["compliance_id"] + + def get_queryset(self): + return Finding.objects.all() + + def get_serializer_class(self): + if self.action == "providers": + return OverviewProviderSerializer + return super().get_serializer_class() + + @extend_schema(exclude=True) + def list(self, request, *args, **kwargs): + raise MethodNotAllowed(method="GET") + + @extend_schema(exclude=True) + def retrieve(self, request, *args, **kwargs): + raise MethodNotAllowed(method="GET") + + @action(detail=False, methods=["get"], url_name="providers") + def providers(self, request): + # Subquery to get the most recent finding for each uid + latest_finding_ids = ( + Finding.objects.filter( + uid=OuterRef("uid"), scan__provider=OuterRef("scan__provider") + ) + .order_by("-id") # Most recent by id + .values("id")[:1] + ) + + # Filter findings to only include the most recent for each uid + recent_findings = Finding.objects.filter(id__in=Subquery(latest_finding_ids)) + + # Aggregate findings by provider + findings_aggregated = ( + recent_findings.values("scan__provider__provider") + .annotate( + findings_passed=Count("id", filter=Q(status=StatusChoices.PASS.value)), + findings_failed=Count("id", filter=Q(status=StatusChoices.FAIL.value)), + findings_manual=Count( + "id", filter=Q(status=StatusChoices.MANUAL.value) + ), + total_findings=Count("id"), + ) + .order_by("-findings_failed") + ) + + # Aggregate total resources by provider + resources_aggregated = Resource.objects.values("provider__provider").annotate( + total_resources=Count("id") + ) + + # Combine findings and resources data + overview = [] + for findings in findings_aggregated: + provider = findings["scan__provider__provider"] + total_resources = next( + ( + res["total_resources"] + for res in resources_aggregated + if res["provider__provider"] == provider + ), + 0, # Default to 0 if no resources are found + ) + overview.append( + { + "provider": provider, + "total_resources": total_resources, + "total_findings": findings["total_findings"], + "findings_passed": findings["findings_passed"], + "findings_failed": findings["findings_failed"], + "findings_manual": findings["findings_manual"], + } + ) + + serializer = OverviewProviderSerializer(overview, many=True) + + return Response(serializer.data, status=status.HTTP_200_OK)