feat(compliance): enhance compliance overview filters and documentation (#9244)

api/CHANGELOG.md

@@ -17,6 +17,7 @@ All notable changes to the **Prowler API** are documented in this file.
 - Tenant-wide ThreatScore overview aggregation and snapshot persistence with backfill support [(#9148)](https://github.com/prowler-cloud/prowler/pull/9148)
 - Added `metadata`, `details`, and `partition` attributes to `/resources` endpoint & `details`, and `partition` to `/findings` endpoint [(#9098)](https://github.com/prowler-cloud/prowler/pull/9098)
 - Support for MongoDB Atlas provider [(#9167)](https://github.com/prowler-cloud/prowler/pull/9167)
+- Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244)
 
 ### Changed
 - Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190)

api/src/backend/api/filters.py

@@ -761,6 +761,14 @@ class RoleFilter(FilterSet):
 class ComplianceOverviewFilter(FilterSet):
     inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
     scan_id = UUIDFilter(field_name="scan_id")
+    provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
+    provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
+    provider_type = ChoiceFilter(
+        field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
+    )
+    provider_type__in = ChoiceInFilter(
+        field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
+    )
     region = CharFilter(field_name="region")
 
     class Meta:

api/src/backend/api/specs/v1.yaml

@@ -283,8 +283,11 @@ paths:
   /api/v1/compliance-overviews:
     get:
       operationId: compliance_overviews_list
-      description: Retrieve an overview of all the compliance in a given scan.
-      summary: List compliance overviews for a scan
+      description: Retrieve an overview of all compliance frameworks. If scan_id is
+        provided, returns compliance data for that specific scan. If scan_id is omitted,
+        returns compliance data aggregated from the latest completed scan of each
+        provider.
+      summary: List compliance overviews
       parameters:
       - in: query
         name: fields[compliance-overviews]
@@ -343,6 +346,32 @@ paths:
         schema:
           type: string
           format: date-time
+      - in: query
+        name: filter[provider_id]
+        schema:
+          type: string
+          format: uuid
+        description: Filter by specific provider ID.
+      - in: query
+        name: filter[provider_id__in]
+        schema:
+          type: array
+          items:
+            type: string
+            format: uuid
+        description: Filter by multiple provider IDs (comma-separated).
+      - in: query
+        name: filter[provider_type]
+        schema:
+          type: string
+        description: Filter by provider type (e.g., aws, azure, gcp).
+      - in: query
+        name: filter[provider_type__in]
+        schema:
+          type: array
+          items:
+            type: string
+        description: Filter by multiple provider types (comma-separated).
       - in: query
         name: filter[region]
         schema:
@@ -365,8 +394,8 @@ paths:
         schema:
           type: string
           format: uuid
-        description: Related scan ID.
-        required: true
+        description: Optional scan ID. If provided, returns compliance for that scan.
+          If omitted, returns compliance for the latest completed scan per provider.
       - name: filter[search]
         required: false
         in: query
@@ -606,6 +635,77 @@ paths:
         schema:
           type: string
           format: date-time
+      - in: query
+        name: filter[provider_id]
+        schema:
+          type: string
+          format: uuid
+      - in: query
+        name: filter[provider_id__in]
+        schema:
+          type: array
+          items:
+            type: string
+            format: uuid
+        description: Multiple values may be separated by commas.
+        explode: false
+        style: form
+      - in: query
+        name: filter[provider_type]
+        schema:
+          type: string
+          x-spec-enum-id: eca8c51e6bd28935
+          enum:
+          - aws
+          - azure
+          - gcp
+          - github
+          - iac
+          - kubernetes
+          - m365
+          - mongodbatlas
+          - oraclecloud
+        description: |-
+          * `aws` - AWS
+          * `azure` - Azure
+          * `gcp` - GCP
+          * `kubernetes` - Kubernetes
+          * `m365` - M365
+          * `github` - GitHub
+          * `mongodbatlas` - MongoDB Atlas
+          * `iac` - IaC
+          * `oraclecloud` - Oracle Cloud Infrastructure
+      - in: query
+        name: filter[provider_type__in]
+        schema:
+          type: array
+          items:
+            type: string
+            x-spec-enum-id: eca8c51e6bd28935
+            enum:
+            - aws
+            - azure
+            - gcp
+            - github
+            - iac
+            - kubernetes
+            - m365
+            - mongodbatlas
+            - oraclecloud
+        description: |-
+          Multiple values may be separated by commas.
+
+          * `aws` - AWS
+          * `azure` - Azure
+          * `gcp` - GCP
+          * `kubernetes` - Kubernetes
+          * `m365` - M365
+          * `github` - GitHub
+          * `mongodbatlas` - MongoDB Atlas
+          * `iac` - IaC
+          * `oraclecloud` - Oracle Cloud Infrastructure
+        explode: false
+        style: form
       - in: query
         name: filter[region]
         schema:

api/src/backend/api/tests/test_views.py

@@ -5972,6 +5972,28 @@ class TestComplianceOverviewViewSet:
         assert len(response.json()["data"]) >= 1
         mock_backfill_task.assert_not_called()
 
+    def test_compliance_overview_list_without_scan_id(
+        self, authenticated_client, compliance_requirements_overviews_fixture
+    ):
+        # Ensure the endpoint works without passing a scan filter
+        response = authenticated_client.get(reverse("complianceoverview-list"))
+
+        assert response.status_code == status.HTTP_200_OK
+        data = response.json()["data"]
+        assert len(data) == 3
+
+        # Validate payload structure
+        first_item = data[0]
+        assert "id" in first_item
+        assert "attributes" in first_item
+        attributes = first_item["attributes"]
+        assert "framework" in attributes
+        assert "version" in attributes
+        assert "requirements_passed" in attributes
+        assert "requirements_failed" in attributes
+        assert "requirements_manual" in attributes
+        assert "total_requirements" in attributes
+
     def test_compliance_overview_metadata(
         self, authenticated_client, compliance_requirements_overviews_fixture
     ):

api/src/backend/api/v1/views.py

@@ -3234,15 +3234,50 @@ class RoleProviderGroupRelationshipView(RelationshipView, BaseRLSViewSet):
 @extend_schema_view(
     list=extend_schema(
         tags=["Compliance Overview"],
-        summary="List compliance overviews for a scan",
-        description="Retrieve an overview of all the compliance in a given scan.",
+        summary="List compliance overviews",
+        description=(
+            "Retrieve an overview of all compliance frameworks. "
+            "If scan_id is provided, returns compliance data for that specific scan. "
+            "If scan_id is omitted, returns compliance data aggregated from the latest completed scan of each provider."
+        ),
         parameters=[
             OpenApiParameter(
                 name="filter[scan_id]",
-                required=True,
+                required=False,
                 type=OpenApiTypes.UUID,
                 location=OpenApiParameter.QUERY,
-                description="Related scan ID.",
+                description=(
+                    "Optional scan ID. If provided, returns compliance for that scan. "
+                    "If omitted, returns compliance for the latest completed scan per provider."
+                ),
+            ),
+            OpenApiParameter(
+                name="filter[provider_id]",
+                required=False,
+                type=OpenApiTypes.UUID,
+                location=OpenApiParameter.QUERY,
+                description="Filter by specific provider ID.",
+            ),
+            OpenApiParameter(
+                name="filter[provider_id__in]",
+                required=False,
+                type={"type": "array", "items": {"type": "string", "format": "uuid"}},
+                location=OpenApiParameter.QUERY,
+                description="Filter by multiple provider IDs (comma-separated).",
+            ),
+            OpenApiParameter(
+                name="filter[provider_type]",
+                required=False,
+                type=OpenApiTypes.STR,
+                location=OpenApiParameter.QUERY,
+                description="Filter by provider type (e.g., aws, azure, gcp).",
+            ),
+            OpenApiParameter(
+                name="filter[provider_type__in]",
+                required=False,
+                type={"type": "array", "items": {"type": "string"}},
+                location=OpenApiParameter.QUERY,
+                description="Filter by multiple provider types (comma-separated).",
             ),
         ],
         responses={
@@ -3563,57 +3598,93 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
 
     def list(self, request, *args, **kwargs):
         scan_id = request.query_params.get("filter[scan_id]")
-        if not scan_id:
-            raise ValidationError(
-                [
+        tenant_id = self.request.tenant_id
+
+        if scan_id:
+            # Specific scan requested - use optimized summaries with region support
+            region_filter = request.query_params.get(
+                "filter[region]"
+            ) or request.query_params.get("filter[region__in]")
+
+            if region_filter:
+                # Fall back to detailed query with region filtering
+                return self._list_with_region_filter(scan_id, region_filter)
+
+            summaries = list(self._compliance_summaries_queryset(scan_id))
+            if not summaries:
+                # Trigger async backfill for next time
+                backfill_compliance_summaries_task.delay(
+                    tenant_id=self.request.tenant_id, scan_id=scan_id
+                )
+                # Use fallback aggregation for this request
+                return self._list_without_region_aggregation(scan_id)
+
+            # Get compliance template for provider to enrich with framework/version
+            compliance_template = self._get_compliance_template(scan_id=scan_id)
+
+            # Convert to response format with framework/version enrichment
+            response_data = []
+            for summary in summaries:
+                compliance_metadata = compliance_template.get(summary.compliance_id, {})
+                response_data.append(
                     {
-                        "detail": "This query parameter is required.",
-                        "status": 400,
-                        "source": {"pointer": "filter[scan_id]"},
-                        "code": "required",
+                        "id": summary.compliance_id,
+                        "compliance_id": summary.compliance_id,
+                        "framework": compliance_metadata.get("framework", ""),
+                        "version": compliance_metadata.get("version", ""),
+                        "requirements_passed": summary.requirements_passed,
+                        "requirements_failed": summary.requirements_failed,
+                        "requirements_manual": summary.requirements_manual,
+                        "total_requirements": summary.total_requirements,
                     }
-                ]
+                )
+
+            serializer = self.get_serializer(response_data, many=True)
+            return Response(serializer.data)
+        else:
+            # No scan_id provided - use latest scans per provider
+            # First, check if provider filters are present
+            provider_id = request.query_params.get("filter[provider_id]")
+            provider_id__in = request.query_params.get("filter[provider_id__in]")
+            provider_type = request.query_params.get("filter[provider_type]")
+            provider_type__in = request.query_params.get("filter[provider_type__in]")
+
+            scan_filters = {"tenant_id": tenant_id, "state": StateChoices.COMPLETED}
+
+            # Apply provider ID filters
+            if provider_id:
+                scan_filters["provider_id"] = provider_id
+            elif provider_id__in:
+                # Convert comma-separated string to list
+                provider_ids = [pid.strip() for pid in provider_id__in.split(",")]
+                scan_filters["provider_id__in"] = provider_ids
+
+            # Apply provider type filters
+            if provider_type:
+                scan_filters["provider__provider"] = provider_type
+            elif provider_type__in:
+                # Convert comma-separated string to list
+                provider_types = [pt.strip() for pt in provider_type__in.split(",")]
+                scan_filters["provider__provider__in"] = provider_types
+
+            latest_scan_ids = (
+                Scan.all_objects.filter(**scan_filters)
+                .order_by("provider_id", "-inserted_at")
+                .distinct("provider_id")
+                .values_list("id", flat=True)
             )
 
-        region_filter = request.query_params.get(
-            "filter[region]"
-        ) or request.query_params.get("filter[region__in]")
-
-        if region_filter:
-            # Fall back to detailed query with region filtering
-            return self._list_with_region_filter(scan_id, region_filter)
-
-        summaries = list(self._compliance_summaries_queryset(scan_id))
-        if not summaries:
-            # Trigger async backfill for next time
-            backfill_compliance_summaries_task.delay(
-                tenant_id=self.request.tenant_id, scan_id=scan_id
-            )
-            # Use fallback aggregation for this request
-            return self._list_without_region_aggregation(scan_id)
-
-        # Get compliance template for provider to enrich with framework/version
-        compliance_template = self._get_compliance_template(scan_id=scan_id)
-
-        # Convert to response format with framework/version enrichment
-        response_data = []
-        for summary in summaries:
-            compliance_metadata = compliance_template.get(summary.compliance_id, {})
-            response_data.append(
-                {
-                    "id": summary.compliance_id,
-                    "compliance_id": summary.compliance_id,
-                    "framework": compliance_metadata.get("framework", ""),
-                    "version": compliance_metadata.get("version", ""),
-                    "requirements_passed": summary.requirements_passed,
-                    "requirements_failed": summary.requirements_failed,
-                    "requirements_manual": summary.requirements_manual,
-                    "total_requirements": summary.total_requirements,
-                }
+            base_queryset = self.get_queryset()
+            queryset = self.filter_queryset(
+                base_queryset.filter(scan_id__in=latest_scan_ids)
             )
 
-        serializer = self.get_serializer(response_data, many=True)
-        return Response(serializer.data)
+            # Aggregate compliance data across latest scans
+            compliance_template = self._get_compliance_template()
+            data = self._aggregate_compliance_overview(
+                queryset, template_metadata=compliance_template
+            )
+            return Response(data)
 
     @action(detail=False, methods=["get"], url_name="metadata")
     def metadata(self, request):

docs/user-guide/tutorials/prowler-app-lighthouse.mdx

@@ -142,7 +142,7 @@ The following API endpoints are accessible to Prowler Lighthouse AI. Data from t
 
 **Compliance Management:**
 
-- List compliance overviews for a scan - `/api/v1/compliance-overviews`
+- List compliance overviews (optionally filter by scan) - `/api/v1/compliance-overviews`
 - Retrieve data from a specific compliance overview - `/api/v1/compliance-overviews/{id}`
 
 #### Excluded API Endpoints