diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 44aa4c6194..af352b2ba1 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -7715,7 +7715,7 @@ paths: - Provider requestBody: content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -7767,7 +7767,7 @@ paths: responses: '201': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -7775,6 +7775,12 @@ paths: type: array items: $ref: '#/components/schemas/Provider' + meta: + type: object + properties: + version: + type: string + example: "v1" examples: success: summary: All providers created successfully @@ -7786,20 +7792,26 @@ paths: provider: aws uid: "123456789012" alias: "AWS Production" + connection: + connected: null + last_checked_at: null inserted_at: "2025-01-20T10:30:00Z" updated_at: "2025-01-20T10:30:00Z" - - type: providers - id: "550e8400-e29b-41d4-a716-446655440002" - attributes: - provider: gcp - uid: "my-gcp-project" - alias: "GCP Staging" - inserted_at: "2025-01-20T10:30:00Z" - updated_at: "2025-01-20T10:30:00Z" + relationships: + secret: + data: null + provider_groups: + meta: + count: 0 + data: [] + links: + self: "https://api.prowler.com/api/v1/providers/550e8400-e29b-41d4-a716-446655440001" + meta: + version: "v1" description: All providers created successfully '400': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -7838,7 +7850,7 @@ paths: - Provider requestBody: content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -7876,7 +7888,7 @@ paths: responses: '200': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -7884,6 +7896,12 @@ paths: type: array items: $ref: '#/components/schemas/Provider' + meta: + type: object + properties: + version: + type: string + example: "v1" examples: success: summary: All providers updated successfully @@ -7895,12 +7913,26 @@ paths: provider: aws uid: "123456789012" alias: "AWS Production (Updated)" + connection: + connected: null + last_checked_at: null inserted_at: "2025-01-20T10:30:00Z" updated_at: "2025-01-20T11:00:00Z" + relationships: + secret: + data: null + provider_groups: + meta: + count: 0 + data: [] + links: + self: "https://api.prowler.com/api/v1/providers/550e8400-e29b-41d4-a716-446655440001" + meta: + version: "v1" description: All providers updated successfully '400': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -8232,7 +8264,7 @@ paths: - Provider requestBody: content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -8305,7 +8337,7 @@ paths: responses: '201': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -8315,8 +8347,11 @@ paths: $ref: '#/components/schemas/ProviderSecret' meta: type: object - description: Contains skipped providers info if any were skipped + description: Contains version and optionally skipped providers info properties: + version: + type: string + example: "v1" skipped: type: array items: @@ -8347,6 +8382,10 @@ paths: data: type: providers id: "550e8400-e29b-41d4-a716-446655440001" + links: + self: "https://api.prowler.com/api/v1/providers/secrets/660e8400-e29b-41d4-a716-446655440001" + meta: + version: "v1" some_skipped: summary: Some providers skipped (already have secrets) value: @@ -8363,7 +8402,10 @@ paths: data: type: providers id: "550e8400-e29b-41d4-a716-446655440002" + links: + self: "https://api.prowler.com/api/v1/providers/secrets/660e8400-e29b-41d4-a716-446655440001" meta: + version: "v1" skipped: - provider_id: "550e8400-e29b-41d4-a716-446655440001" source_index: 0 @@ -8371,7 +8413,7 @@ paths: description: Provider secrets created successfully. May include meta.skipped if some providers were skipped. '400': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -8411,7 +8453,7 @@ paths: - Provider requestBody: content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -8459,7 +8501,7 @@ paths: responses: '200': content: - application/json: + application/vnd.api+json: schema: type: object properties: @@ -8467,6 +8509,12 @@ paths: type: array items: $ref: '#/components/schemas/ProviderSecret' + meta: + type: object + properties: + version: + type: string + example: "v1" examples: all_updated: summary: All secrets updated successfully @@ -8484,10 +8532,14 @@ paths: data: type: providers id: "550e8400-e29b-41d4-a716-446655440001" + links: + self: "https://api.prowler.com/api/v1/providers/secrets/660e8400-e29b-41d4-a716-446655440001" + meta: + version: "v1" description: All provider secrets updated successfully '400': content: - application/json: + application/vnd.api+json: schema: type: object properties: diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 399a6ecf48..12359ecd59 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -998,7 +998,7 @@ class ProviderBatchCreateSerializer(BaseSerializerV1): """Serializer for batch creation of providers with all-or-nothing semantics (JSON:API compliant).""" class Meta: - resource_name = "providers-batch" + resource_name = "providers" def validate(self, attrs): data = self.initial_data.get("data", []) @@ -1139,7 +1139,7 @@ class ProviderBatchUpdateSerializer(BaseSerializerV1): """Serializer for batch update of providers with all-or-nothing semantics (JSON:API compliant).""" class Meta: - resource_name = "providers-batch" + resource_name = "providers" def validate(self, attrs): data = self.initial_data.get("data", []) @@ -1991,6 +1991,13 @@ class ProviderSecretSerializer(RLSSerializer): "url", ] + def get_root_meta(self, _resource, _many): + meta = super().get_root_meta(_resource, _many) + skipped = self.context.get("_skipped_providers") + if skipped: + meta["skipped"] = skipped + return meta + class ProviderSecretCreateSerializer(RLSSerializer, BaseWriteProviderSecretSerializer): secret = ProviderSecretField(write_only=True) @@ -2086,7 +2093,7 @@ class ProviderSecretBatchCreateSerializer(BaseSerializerV1): """ class Meta: - resource_name = "provider-secrets-batch" + resource_name = "provider-secrets" def _extract_providers_data(self, relationships, idx): """ @@ -2369,7 +2376,7 @@ class ProviderSecretBatchUpdateSerializer(BaseSerializerV1): """ class Meta: - resource_name = "provider-secrets-batch" + resource_name = "provider-secrets" def validate(self, attrs): data = self.initial_data.get("data", []) diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 40ce930be0..003827dd74 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -44,7 +44,7 @@ from django.db.models import ( Window, ) from django.db.models.functions import Coalesce, RowNumber -from django.http import HttpResponse, JsonResponse, QueryDict +from django.http import HttpResponse, QueryDict from django.shortcuts import redirect from django.urls import reverse from django.utils.dateparse import parse_date @@ -281,6 +281,18 @@ from api.v1.serializers import ( UserUpdateSerializer, ) + +class JSONAPIRawParser(JSONParser): + """Parser that accepts application/vnd.api+json but passes through raw JSON. + + Unlike rest_framework_json_api's JSONParser which transforms + JSON:API payloads, this parser just parses JSON and leaves the + data structure intact for batch serializers to handle. + """ + + media_type = "application/vnd.api+json" + + logger = logging.getLogger(BackendLogger.API) CACHE_DECORATOR = cache_control( @@ -1560,7 +1572,7 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): "batch_create" in action_map.values() or "batch_update" in action_map.values() ): - return [JSONParser()] + return [JSONAPIRawParser(), JSONParser()] return super().get_parsers() def partial_update(self, request, *args, **kwargs): @@ -1632,7 +1644,7 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): summary="Batch create providers", description="Create multiple providers in a single atomic operation. JSON:API compliant with all-or-nothing semantics. Secrets must be added separately via the provider secrets endpoint.", request={ - "application/json": { + "application/vnd.api+json": { "type": "object", "required": ["data"], "properties": { @@ -1695,7 +1707,7 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): methods=["post"], url_path="batch", url_name="batch", - parser_classes=[JSONParser], + parser_classes=[JSONAPIRawParser, JSONParser], ) def batch_create(self, request): serializer = ProviderBatchCreateSerializer( @@ -1706,12 +1718,15 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): validated_items = serializer.validated_data.get("_validated_items", []) - # Create all providers + # Create all providers atomically created_provider_ids = [] - for item in validated_items: - item_data = item["data"] - provider = Provider.objects.create(tenant_id=request.tenant_id, **item_data) - created_provider_ids.append(provider.id) + with transaction.atomic(): + for item in validated_items: + item_data = item["data"] + provider = Provider.objects.create( + tenant_id=request.tenant_id, **item_data + ) + created_provider_ids.append(provider.id) created_providers = ( Provider.objects.filter(id__in=created_provider_ids) @@ -1730,7 +1745,7 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): summary="Batch update providers", description="Update multiple providers in a single atomic operation. JSON:API compliant with all-or-nothing semantics. Only alias can be updated.", request={ - "application/json": { + "application/vnd.api+json": { "type": "object", "required": ["data"], "properties": { @@ -1781,7 +1796,7 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): methods=["patch"], url_path="batch", url_name="batch-update", - parser_classes=[JSONParser], + parser_classes=[JSONAPIRawParser, JSONParser], ) def batch_update(self, request): serializer = ProviderBatchUpdateSerializer( @@ -1792,15 +1807,16 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): validated_items = serializer.validated_data.get("_validated_items", []) - # Update all providers + # Update all providers atomically updated_provider_ids = [] - for item_data in validated_items: - item_data.pop("index") - provider = item_data.pop("provider") - for key, value in item_data.items(): - setattr(provider, key, value) - provider.save() - updated_provider_ids.append(provider.id) + with transaction.atomic(): + for item_data in validated_items: + item_data.pop("index") + provider = item_data.pop("provider") + for key, value in item_data.items(): + setattr(provider, key, value) + provider.save() + updated_provider_ids.append(provider.id) updated_providers = ( Provider.objects.filter(id__in=updated_provider_ids) @@ -3538,7 +3554,7 @@ class ProviderSecretViewSet(BaseRLSViewSet): "batch_create" in action_map.values() or "batch_update" in action_map.values() ): - return [JSONParser()] + return [JSONAPIRawParser(), JSONParser()] return super().get_parsers() @extend_schema( @@ -3552,7 +3568,7 @@ class ProviderSecretViewSet(BaseRLSViewSet): "and reported in meta.skipped." ), request={ - "application/json": { + "application/vnd.api+json": { "type": "object", "required": ["data"], "properties": { @@ -3638,7 +3654,7 @@ class ProviderSecretViewSet(BaseRLSViewSet): methods=["post"], url_path="batch", url_name="batch", - parser_classes=[JSONParser], + parser_classes=[JSONAPIRawParser, JSONParser], ) def batch_create(self, request): serializer = ProviderSecretBatchCreateSerializer( @@ -3650,36 +3666,31 @@ class ProviderSecretViewSet(BaseRLSViewSet): validated_items = serializer.validated_data.get("_validated_items", []) skipped_providers = serializer.validated_data.get("_skipped_providers", []) - # Create all validated secrets + # Create all validated secrets atomically created_secret_ids = [] - for item_data in validated_items: - item_data.pop("source_index", None) - provider = item_data.pop("provider") - secret = ProviderSecret.objects.create( - tenant_id=request.tenant_id, provider=provider, **item_data - ) - created_secret_ids.append(secret.id) + with transaction.atomic(): + for item_data in validated_items: + item_data.pop("source_index", None) + provider = item_data.pop("provider") + secret = ProviderSecret.objects.create( + tenant_id=request.tenant_id, provider=provider, **item_data + ) + created_secret_ids.append(secret.id) created_secrets = ProviderSecret.objects.filter( id__in=created_secret_ids ).select_related("provider") + # Pass skipped providers through context so the serializer's + # get_root_meta() can include them in the response meta. + context = self.get_serializer_context() + if skipped_providers: + context["_skipped_providers"] = skipped_providers + response_serializer = ProviderSecretSerializer( - created_secrets, many=True, context=self.get_serializer_context() + created_secrets, many=True, context=context ) - # Include skipped providers in meta if any - if skipped_providers: - # Use JsonResponse for custom meta - builds JSON:API structure manually - return JsonResponse( - { - "data": response_serializer.data, - "meta": {"skipped": skipped_providers}, - }, - status=201, - ) - - # Standard response without meta return Response(data=response_serializer.data, status=status.HTTP_201_CREATED) @extend_schema( @@ -3690,7 +3701,7 @@ class ProviderSecretViewSet(BaseRLSViewSet): "JSON:API compliant with all-or-nothing semantics." ), request={ - "application/json": { + "application/vnd.api+json": { "type": "object", "required": ["data"], "properties": { @@ -3749,7 +3760,7 @@ class ProviderSecretViewSet(BaseRLSViewSet): methods=["patch"], url_path="batch", url_name="batch-update", - parser_classes=[JSONParser], + parser_classes=[JSONAPIRawParser, JSONParser], ) def batch_update(self, request): serializer = ProviderSecretBatchUpdateSerializer( @@ -3760,15 +3771,16 @@ class ProviderSecretViewSet(BaseRLSViewSet): validated_items = serializer.validated_data.get("_validated_items", []) - # Update all validated secrets + # Update all validated secrets atomically updated_secret_ids = [] - for item_data in validated_items: - item_data.pop("source_index", None) - provider_secret = item_data.pop("provider_secret") - for key, value in item_data.items(): - setattr(provider_secret, key, value) - provider_secret.save() - updated_secret_ids.append(provider_secret.id) + with transaction.atomic(): + for item_data in validated_items: + item_data.pop("source_index", None) + provider_secret = item_data.pop("provider_secret") + for key, value in item_data.items(): + setattr(provider_secret, key, value) + provider_secret.save() + updated_secret_ids.append(provider_secret.id) updated_secrets = ProviderSecret.objects.filter( id__in=updated_secret_ids