From e4640a04970f7b9bfb8ba81c93e69392068cbf6c Mon Sep 17 00:00:00 2001 From: pedrooot Date: Wed, 21 Jan 2026 23:40:45 +0100 Subject: [PATCH] feat(api): support batch add provider & secrets --- api/src/backend/api/specs/v1.yaml | 525 +++++++ .../test_provider_secrets_batch.py | 1389 +++++++++++++++++ .../tests/integration/test_providers_batch.py | 921 +++++++++++ api/src/backend/api/v1/serializers.py | 717 +++++++++ api/src/backend/api/v1/urls.py | 12 + api/src/backend/api/v1/views.py | 510 +++++- api/src/backend/config/django/base.py | 2 + 7 files changed, 4048 insertions(+), 28 deletions(-) create mode 100644 api/src/backend/api/tests/integration/test_provider_secrets_batch.py create mode 100644 api/src/backend/api/tests/integration/test_providers_batch.py diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 0e68d6b364..44aa4c6194 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -7703,6 +7703,233 @@ paths: schema: $ref: '#/components/schemas/ProviderCreateResponse' description: '' + /api/v1/providers/batch: + post: + operationId: providers_batch_create + description: | + Create multiple providers in a single atomic operation. + JSON:API compliant with all-or-nothing semantics. + Secrets must be added separately via the provider secrets endpoint. + summary: Batch create providers + tags: + - Provider + requestBody: + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + type: object + properties: + type: + type: string + example: providers + description: Resource type identifier (should be "providers") + attributes: + type: object + properties: + provider: + type: string + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: Type of cloud provider + uid: + type: string + description: Unique identifier for the provider (e.g., AWS account ID, Azure subscription ID) + alias: + type: string + description: Human-readable name for the provider + required: + - provider + - uid + required: + - type + - attributes + maxItems: 100 + minItems: 1 + required: + - data + required: true + security: + - JWT or API Key: [] + responses: + '201': + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/Provider' + examples: + success: + summary: All providers created successfully + value: + data: + - type: providers + id: "550e8400-e29b-41d4-a716-446655440001" + attributes: + provider: aws + uid: "123456789012" + alias: "AWS Production" + inserted_at: "2025-01-20T10:30:00Z" + updated_at: "2025-01-20T10:30:00Z" + - type: providers + id: "550e8400-e29b-41d4-a716-446655440002" + attributes: + provider: gcp + uid: "my-gcp-project" + alias: "GCP Staging" + inserted_at: "2025-01-20T10:30:00Z" + updated_at: "2025-01-20T10:30:00Z" + description: All providers created successfully + '400': + content: + application/json: + schema: + type: object + properties: + errors: + type: array + items: + type: object + properties: + detail: + type: string + source: + type: object + properties: + pointer: + type: string + examples: + validation_errors: + summary: Validation errors - entire batch rejected (all-or-nothing) + value: + errors: + - detail: "Invalid AWS account ID format. Must be 12 digits." + source: + pointer: "/data/0/attributes/uid" + - detail: "Provider with uid '111111111111' already exists" + source: + pointer: "/data/1/attributes/uid" + description: Validation errors - entire batch rejected (all-or-nothing) + patch: + operationId: providers_batch_update + description: | + Update multiple providers in a single atomic operation. + JSON:API compliant with all-or-nothing semantics. + Only alias can be updated. + summary: Batch update providers + tags: + - Provider + requestBody: + content: + application/json: + schema: + type: object + properties: + data: + type: array + description: Array of provider objects to update (max 100) + items: + type: object + properties: + type: + type: string + example: providers + description: Resource type identifier (should be "providers") + id: + type: string + format: uuid + description: Provider UUID to update + attributes: + type: object + properties: + alias: + type: string + description: New display name for the provider + required: + - type + - id + - attributes + maxItems: 100 + minItems: 1 + required: + - data + required: true + security: + - JWT or API Key: [] + responses: + '200': + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/Provider' + examples: + success: + summary: All providers updated successfully + value: + data: + - type: providers + id: "550e8400-e29b-41d4-a716-446655440001" + attributes: + provider: aws + uid: "123456789012" + alias: "AWS Production (Updated)" + inserted_at: "2025-01-20T10:30:00Z" + updated_at: "2025-01-20T11:00:00Z" + description: All providers updated successfully + '400': + content: + application/json: + schema: + type: object + properties: + errors: + type: array + items: + type: object + properties: + detail: + type: string + description: Error message + source: + type: object + properties: + pointer: + type: string + description: JSON pointer to the error location (e.g., /data/0/id) + examples: + validation_errors: + summary: All items failed validation + value: + errors: + - detail: "Provider '550e8400-e29b-41d4-a716-446655440099' not found" + source: + pointer: "/data/0/id" + - detail: "This field is required." + source: + pointer: "/data/1/id" + description: Validation errors - entire batch rejected (all-or-nothing) /api/v1/providers/{id}: get: operationId: providers_retrieve @@ -7992,6 +8219,304 @@ paths: schema: $ref: '#/components/schemas/ProviderSecretCreateResponse' description: '' + /api/v1/providers/secrets/batch: + post: + operationId: providers_secrets_batch_create + description: | + Create multiple provider secrets in a single atomic operation. + Supports to-many relationships where one secret definition can be associated with multiple providers. + JSON:API compliant with all-or-nothing semantics for hard errors. + Providers that already have secrets are skipped and reported in meta.skipped. + summary: Batch create provider secrets + tags: + - Provider + requestBody: + content: + application/json: + schema: + type: object + properties: + data: + type: array + description: Array of provider-secret objects (max 100) + items: + type: object + properties: + type: + type: string + example: provider-secrets + description: Resource type identifier (should be "provider-secrets") + attributes: + type: object + properties: + name: + type: string + description: Human-readable name for the secret + secret_type: + type: string + enum: + - static + - role + - service_account + description: Type of secret/authentication method + secret: + type: object + description: Credentials object (varies by provider/secret_type) + required: + - secret_type + - secret + relationships: + type: object + properties: + providers: + type: object + description: To-many relationship - one secret definition can create secrets for multiple providers + properties: + data: + type: array + items: + type: object + properties: + type: + type: string + enum: + - providers + id: + type: string + format: uuid + required: + - type + - id + required: + - data + required: + - providers + required: + - type + - attributes + - relationships + maxItems: 100 + minItems: 1 + required: + - data + required: true + security: + - JWT or API Key: [] + responses: + '201': + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/ProviderSecret' + meta: + type: object + description: Contains skipped providers info if any were skipped + properties: + skipped: + type: array + items: + type: object + properties: + provider_id: + type: string + format: uuid + source_index: + type: integer + description: Index of the item in the request that contained this provider + reason: + type: string + examples: + all_created: + summary: All secrets created successfully + value: + data: + - type: provider-secrets + id: "660e8400-e29b-41d4-a716-446655440001" + attributes: + name: "AWS Production Credentials" + secret_type: role + inserted_at: "2025-01-20T10:30:00Z" + updated_at: "2025-01-20T10:30:00Z" + relationships: + provider: + data: + type: providers + id: "550e8400-e29b-41d4-a716-446655440001" + some_skipped: + summary: Some providers skipped (already have secrets) + value: + data: + - type: provider-secrets + id: "660e8400-e29b-41d4-a716-446655440001" + attributes: + name: "Shared AWS Secret" + secret_type: static + inserted_at: "2025-01-20T10:30:00Z" + updated_at: "2025-01-20T10:30:00Z" + relationships: + provider: + data: + type: providers + id: "550e8400-e29b-41d4-a716-446655440002" + meta: + skipped: + - provider_id: "550e8400-e29b-41d4-a716-446655440001" + source_index: 0 + reason: "Provider already has a secret." + description: Provider secrets created successfully. May include meta.skipped if some providers were skipped. + '400': + content: + application/json: + schema: + type: object + properties: + errors: + type: array + items: + type: object + properties: + detail: + type: string + description: Error message + source: + type: object + properties: + pointer: + type: string + description: JSON pointer to the error location (e.g., /data/0/attributes/secret_type) + examples: + validation_errors: + summary: Validation errors - entire batch rejected (all-or-nothing) + value: + errors: + - detail: "Role secret type is not supported for GCP providers" + source: + pointer: "/data/0/attributes/secret_type" + - detail: "Provider '550e8400-e29b-41d4-a716-446655440099' not found." + source: + pointer: "/data/1/relationships/providers/data/0" + description: Validation errors - entire batch rejected (all-or-nothing) + patch: + operationId: providers_secrets_batch_update + description: | + Update multiple provider secrets in a single atomic operation. + JSON:API compliant with all-or-nothing semantics. + summary: Batch update provider secrets + tags: + - Provider + requestBody: + content: + application/json: + schema: + type: object + properties: + data: + type: array + description: Array of provider-secret objects to update (max 100) + items: + type: object + properties: + type: + type: string + example: provider-secrets + description: Resource type identifier (should be "provider-secrets") + id: + type: string + format: uuid + description: Provider secret UUID to update + attributes: + type: object + properties: + name: + type: string + description: Human-readable name for the secret + secret_type: + type: string + enum: + - static + - role + - service_account + description: Type of secret/authentication method + secret: + type: object + description: Credentials object (varies by provider/secret_type) + required: + - type + - id + - attributes + maxItems: 100 + minItems: 1 + required: + - data + required: true + security: + - JWT or API Key: [] + responses: + '200': + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/ProviderSecret' + examples: + all_updated: + summary: All secrets updated successfully + value: + data: + - type: provider-secrets + id: "660e8400-e29b-41d4-a716-446655440001" + attributes: + name: "AWS Production Credentials (Updated)" + secret_type: role + inserted_at: "2025-01-20T10:30:00Z" + updated_at: "2025-01-20T11:00:00Z" + relationships: + provider: + data: + type: providers + id: "550e8400-e29b-41d4-a716-446655440001" + description: All provider secrets updated successfully + '400': + content: + application/json: + schema: + type: object + properties: + errors: + type: array + items: + type: object + properties: + detail: + type: string + description: Error message + source: + type: object + properties: + pointer: + type: string + description: JSON pointer to the error location (e.g., /data/0/id) + examples: + validation_errors: + summary: Validation errors - entire batch rejected (all-or-nothing) + value: + errors: + - detail: "Provider secret '660e8400-e29b-41d4-a716-446655440099' not found." + source: + pointer: "/data/0/id" + - detail: "This field is required." + source: + pointer: "/data/1/id" + description: Validation errors - entire batch rejected (all-or-nothing) /api/v1/providers/secrets/{id}: get: operationId: providers_secrets_retrieve diff --git a/api/src/backend/api/tests/integration/test_provider_secrets_batch.py b/api/src/backend/api/tests/integration/test_provider_secrets_batch.py new file mode 100644 index 0000000000..cfa74718c0 --- /dev/null +++ b/api/src/backend/api/tests/integration/test_provider_secrets_batch.py @@ -0,0 +1,1389 @@ +import pytest +from django.conf import settings +from django.test import override_settings +from django.urls import reverse +from rest_framework import status + +from api.models import Provider, ProviderSecret + + +@pytest.mark.django_db +class TestProviderSecretBatchCreate: + """Tests for the batch provider secret creation endpoint.""" + + content_type = "application/json" + + def test_batch_create_single_secret_success( + self, authenticated_client, providers_fixture + ): + """Test creating a single provider secret via batch endpoint.""" + provider = providers_fixture[0] # AWS provider + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "AWS Production Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + assert len(response_data["data"]) == 1 + assert response_data["data"][0]["attributes"]["name"] == "AWS Production Secret" + assert response_data["data"][0]["attributes"]["secret_type"] == "static" + + def test_batch_create_backwards_compatible_provider_singular( + self, authenticated_client, providers_fixture + ): + """Test creating with backwards-compatible 'provider' (singular) format.""" + provider = providers_fixture[0] # AWS provider + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "AWS Production Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "provider": { + "data": {"type": "providers", "id": str(provider.id)} + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + assert len(response_data["data"]) == 1 + + def test_batch_create_multiple_secrets_success( + self, authenticated_client, tenants_fixture + ): + """Test creating multiple provider secrets in one batch.""" + tenant = tenants_fixture[0] + + # Create new providers without secrets for this test + provider1 = Provider.objects.create( + provider="aws", + uid="111111111111", + alias="aws_batch_1", + tenant_id=tenant.id, + ) + provider2 = Provider.objects.create( + provider="gcp", + uid="gcp-batch-project-1", + alias="gcp_batch_1", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "AWS Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider1.id)}] + } + }, + }, + { + "type": "provider-secrets", + "attributes": { + "name": "GCP Secret", + "secret_type": "static", + "secret": { + "client_id": "123456789.apps.googleusercontent.com", + "client_secret": "GOCSPX-abcdefghijklmnop", + "refresh_token": "1//0abc-refresh-token-xyz", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider2.id)}] + } + }, + }, + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + assert len(response_data["data"]) == 2 + + def test_batch_create_one_secret_for_multiple_providers( + self, authenticated_client, tenants_fixture + ): + """Test creating one secret definition that applies to multiple providers.""" + tenant = tenants_fixture[0] + + # Create multiple AWS providers + provider1 = Provider.objects.create( + provider="aws", + uid="111111111111", + alias="aws_multi_1", + tenant_id=tenant.id, + ) + provider2 = Provider.objects.create( + provider="aws", + uid="222222222222", + alias="aws_multi_2", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Shared AWS Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [ + {"type": "providers", "id": str(provider1.id)}, + {"type": "providers", "id": str(provider2.id)}, + ] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + # Should create 2 secrets (one per provider) + assert len(response_data["data"]) == 2 + + def test_batch_create_aws_role_secret(self, authenticated_client, tenants_fixture): + """Test creating an AWS role assumption secret.""" + tenant = tenants_fixture[0] + + provider = Provider.objects.create( + provider="aws", + uid="222222222222", + alias="aws_role_test", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "AWS Role Secret", + "secret_type": "role", + "secret": { + "role_arn": "arn:aws:iam::123456789012:role/ProwlerRole", + "external_id": "my-external-id-123", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + assert len(response_data["data"]) == 1 + assert response_data["data"][0]["attributes"]["secret_type"] == "role" + + def test_batch_create_gcp_service_account_secret( + self, authenticated_client, tenants_fixture + ): + """Test creating a GCP service account secret.""" + tenant = tenants_fixture[0] + + provider = Provider.objects.create( + provider="gcp", + uid="gcp-sa-test-project", + alias="gcp_sa_test", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "GCP Service Account Secret", + "secret_type": "service_account", + "secret": { + "service_account_key": { + "type": "service_account", + "project_id": "my-project", + "private_key_id": "key123", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIE...\n-----END PRIVATE KEY-----\n", + "client_email": "test@my-project.iam.gserviceaccount.com", + "client_id": "123456789", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + } + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + assert len(response_data["data"]) == 1 + assert ( + response_data["data"][0]["attributes"]["secret_type"] == "service_account" + ) + + def test_batch_create_duplicate_provider_in_batch_error( + self, authenticated_client, tenants_fixture + ): + """Test that duplicate providers in batch cause all-or-nothing failure.""" + tenant = tenants_fixture[0] + initial_count = ProviderSecret.objects.count() + + provider = Provider.objects.create( + provider="aws", + uid="333333333333", + alias="aws_dup_test", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "First Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + }, + { + "type": "provider-secrets", + "attributes": { + "name": "Second Secret (duplicate provider)", + "secret_type": "static", + "secret": { + "aws_access_key_id": "TESTEXAMPLEKEY00002", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKE2", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + }, + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + # All-or-nothing: should fail with 400 + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Duplicate provider" in str(e.get("detail", "")) for e in errors) + + # Verify NO secrets were created + assert ProviderSecret.objects.count() == initial_count + + def test_batch_create_provider_already_has_secret_skipped( + self, authenticated_client, providers_fixture, provider_secret_fixture + ): + """Test that providers already having a secret are skipped and reported in meta.""" + # provider_secret_fixture creates secrets for all providers + provider = providers_fixture[0] + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Another Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + # Should succeed with 201 but skip the provider + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + # No secrets created (all were skipped) + assert len(response_data["data"]) == 0 + # Should have meta.skipped + assert "meta" in response_data + assert "skipped" in response_data["meta"] + assert len(response_data["meta"]["skipped"]) == 1 + assert "already has a secret" in response_data["meta"]["skipped"][0]["reason"] + + def test_batch_create_mixed_skip_and_create( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + provider_secret_fixture, + ): + """Test batch with some providers skipped (already have secrets) and some created.""" + tenant = tenants_fixture[0] + existing_provider = providers_fixture[0] # Has secret from fixture + + # Create a new provider without a secret + new_provider = Provider.objects.create( + provider="aws", + uid="999999999999", + alias="aws_new_for_mixed", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Shared Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [ + { + "type": "providers", + "id": str(existing_provider.id), + }, # Will be skipped + { + "type": "providers", + "id": str(new_provider.id), + }, # Will be created + ] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + + # One secret created for the new provider + assert len(response_data["data"]) == 1 + + # One skipped (existing provider already has secret) + assert "meta" in response_data + assert len(response_data["meta"]["skipped"]) == 1 + assert ( + str(existing_provider.id) + == response_data["meta"]["skipped"][0]["provider_id"] + ) + + def test_batch_create_provider_not_found_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-existent providers cause all-or-nothing failure.""" + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Secret for non-existent provider", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [ + { + "type": "providers", + "id": "00000000-0000-0000-0000-000000000000", + } + ] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("not found" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_invalid_secret_type_for_provider_error( + self, authenticated_client, tenants_fixture + ): + """Test that invalid secret_type for a provider causes all-or-nothing failure.""" + tenant = tenants_fixture[0] + + # GCP doesn't support role assumption + provider = Provider.objects.create( + provider="gcp", + uid="gcp-invalid-type-test", + alias="gcp_invalid_type", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Invalid Secret Type", + "secret_type": "role", + "secret": { + "role_arn": "arn:aws:iam::123456789012:role/ProwlerRole", + "external_id": "my-external-id", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("not supported" in str(e.get("detail", "")).lower() for e in errors) + + def test_batch_create_permission_denied( + self, authenticated_client_no_permissions_rbac, tenants_fixture + ): + """Test that users without MANAGE_PROVIDERS permission cannot batch create.""" + tenant = tenants_fixture[0] + + provider = Provider.objects.create( + provider="aws", + uid="444444444444", + alias="aws_perm_test", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client_no_permissions_rbac.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_batch_create_exceeds_limit_error( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests exceeding the limit are rejected.""" + limit = settings.API_BATCH_MAX_SIZE + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{i:012d}", + alias=f"aws_limit_{i}", + tenant_id=tenant.id, + ) + for i in range(limit + 1) + ] + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": f"Secret {i}", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(providers[i].id)}] + } + }, + } + for i in range(limit + 1) + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any(f"Maximum {limit}" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_empty_array_error( + self, authenticated_client, tenants_fixture + ): + """Test that empty batch requests are rejected.""" + payload = {"data": []} + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("At least one secret" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_invalid_data_format_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-array data is rejected.""" + payload = { + "data": { + "type": "provider-secrets", + "attributes": { + "name": "Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + } + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Must be an array" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_missing_providers_relationship_error( + self, authenticated_client, tenants_fixture + ): + """Test that missing providers relationship is rejected.""" + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Secret without provider", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": {}, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any( + "Providers relationship is required" in str(e.get("detail", "")) + for e in errors + ) + + def test_batch_create_sets_correct_tenant( + self, authenticated_client, tenants_fixture + ): + """Test that batch-created secrets have correct tenant assignment.""" + tenant = tenants_fixture[0] + + provider = Provider.objects.create( + provider="aws", + uid="666666666666", + alias="aws_tenant_test", + tenant_id=tenant.id, + ) + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Tenant Test Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(provider.id)}] + } + }, + } + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + assert response.status_code == status.HTTP_201_CREATED + secret_id = response.json()["data"][0]["id"] + + secret = ProviderSecret.objects.get(id=secret_id) + assert secret.tenant_id == tenant.id + + def test_batch_create_multiple_errors_all_reported( + self, authenticated_client, tenants_fixture + ): + """Test that all validation errors are reported (all-or-nothing).""" + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Secret 1", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [ + { + "type": "providers", + "id": "00000000-0000-0000-0000-000000000001", + } + ] + } + }, + }, + { + "type": "provider-secrets", + "attributes": { + "name": "Secret 2", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + "relationships": { + "providers": { + "data": [ + { + "type": "providers", + "id": "00000000-0000-0000-0000-000000000002", + } + ] + } + }, + }, + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + # Should have errors for both items + error_pointers = [e.get("source", {}).get("pointer", "") for e in errors] + assert any("/data/0" in p for p in error_pointers) + assert any("/data/1" in p for p in error_pointers) + + def test_batch_create_at_exact_limit_success( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests at exactly the limit are accepted.""" + limit = settings.API_BATCH_MAX_SIZE + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{700000000000 + i}", + alias=f"aws_exact_limit_{i}", + tenant_id=tenant.id, + ) + for i in range(limit) + ] + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": f"Secret {i}", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLE", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(providers[i].id)}] + } + }, + } + for i in range(limit) + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + response_data = response.json() + assert len(response_data["data"]) == limit + + @override_settings(API_BATCH_MAX_SIZE=5) + def test_batch_create_respects_custom_limit_setting( + self, authenticated_client, tenants_fixture + ): + """Test that batch endpoint respects custom API_BATCH_MAX_SIZE setting.""" + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{900000000000 + i}", + alias=f"aws_custom_limit_{i}", + tenant_id=tenant.id, + ) + for i in range(6) + ] + + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": f"Secret {i}", + "secret_type": "static", + "secret": { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLE", + }, + }, + "relationships": { + "providers": { + "data": [{"type": "providers", "id": str(providers[i].id)}] + } + }, + } + for i in range(6) + ] + } + + response = authenticated_client.post( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Maximum 5" in str(e.get("detail", "")) for e in errors) + + +@pytest.mark.django_db +class TestProviderSecretBatchUpdate: + """Tests for the batch provider secret update endpoint.""" + + content_type = "application/json" + + def test_batch_update_single_secret_success( + self, authenticated_client, providers_fixture, provider_secret_fixture + ): + """Test updating a single provider secret via batch endpoint.""" + secret = provider_secret_fixture[0] + new_name = "Updated AWS Secret Name" + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secret.id), + "attributes": { + "name": new_name, + }, + } + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["name"] == new_name + + secret.refresh_from_db() + assert secret.name == new_name + + def test_batch_update_multiple_secrets_success( + self, authenticated_client, providers_fixture, provider_secret_fixture + ): + """Test updating multiple provider secrets in one batch.""" + secret1 = provider_secret_fixture[0] + secret2 = provider_secret_fixture[1] + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secret1.id), + "attributes": { + "name": "Updated Secret 1", + }, + }, + { + "type": "provider-secrets", + "id": str(secret2.id), + "attributes": { + "name": "Updated Secret 2", + }, + }, + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + + secret1.refresh_from_db() + secret2.refresh_from_db() + assert secret1.name == "Updated Secret 1" + assert secret2.name == "Updated Secret 2" + + def test_batch_update_secret_credentials( + self, authenticated_client, providers_fixture, provider_secret_fixture + ): + """Test updating secret credentials via batch endpoint.""" + secret = provider_secret_fixture[0] + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secret.id), + "attributes": { + "secret": { + "aws_access_key_id": "NEWAKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "NEWwJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + } + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + + def test_batch_update_secret_not_found_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-existent secrets cause all-or-nothing failure.""" + payload = { + "data": [ + { + "type": "provider-secrets", + "id": "00000000-0000-0000-0000-000000000000", + "attributes": { + "name": "Updated Name", + }, + } + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("not found" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_duplicate_id_in_batch_error( + self, authenticated_client, providers_fixture, provider_secret_fixture + ): + """Test that duplicate IDs cause all-or-nothing failure.""" + secret = provider_secret_fixture[0] + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secret.id), + "attributes": { + "name": "First Update", + }, + }, + { + "type": "provider-secrets", + "id": str(secret.id), + "attributes": { + "name": "Second Update (duplicate)", + }, + }, + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + # All-or-nothing: should fail with 400 + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Duplicate secret ID" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_missing_id_error(self, authenticated_client, tenants_fixture): + """Test that missing ID is rejected.""" + payload = { + "data": [ + { + "type": "provider-secrets", + "attributes": { + "name": "Updated Name", + }, + } + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("required" in str(e.get("detail", "")).lower() for e in errors) + + def test_batch_update_permission_denied( + self, + authenticated_client_no_permissions_rbac, + providers_fixture, + provider_secret_fixture, + ): + """Test that users without MANAGE_PROVIDERS permission cannot batch update.""" + secret = provider_secret_fixture[0] + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secret.id), + "attributes": { + "name": "Updated Name", + }, + } + ] + } + + response = authenticated_client_no_permissions_rbac.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_batch_update_exceeds_limit_error( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests exceeding the limit are rejected.""" + limit = settings.API_BATCH_MAX_SIZE + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{i:012d}", + alias=f"aws_update_limit_{i}", + tenant_id=tenant.id, + ) + for i in range(limit + 1) + ] + + secrets = [ + ProviderSecret.objects.create( + provider=providers[i], + tenant_id=tenant.id, + name=f"Secret {i}", + secret_type="static", + secret={ + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + ) + for i in range(limit + 1) + ] + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secrets[i].id), + "attributes": { + "name": f"Updated Secret {i}", + }, + } + for i in range(limit + 1) + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any(f"Maximum {limit}" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_empty_array_error( + self, authenticated_client, tenants_fixture + ): + """Test that empty batch requests are rejected.""" + payload = {"data": []} + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("At least one secret" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_invalid_data_format_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-array data is rejected.""" + payload = { + "data": { + "type": "provider-secrets", + "id": "00000000-0000-0000-0000-000000000000", + "attributes": { + "name": "Updated Name", + }, + } + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Must be an array" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_preserves_other_fields( + self, authenticated_client, providers_fixture, provider_secret_fixture + ): + """Test that updating one field doesn't affect other fields.""" + secret = provider_secret_fixture[0] + original_secret_type = secret.secret_type + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secret.id), + "attributes": { + "name": "Only Name Updated", + }, + } + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + + secret.refresh_from_db() + assert secret.name == "Only Name Updated" + assert secret.secret_type == original_secret_type + + def test_batch_update_multiple_errors_all_reported( + self, authenticated_client, tenants_fixture + ): + """Test that all validation errors are reported (all-or-nothing).""" + payload = { + "data": [ + { + "type": "provider-secrets", + "id": "00000000-0000-0000-0000-000000000001", + "attributes": { + "name": "Secret 1", + }, + }, + { + "type": "provider-secrets", + "id": "00000000-0000-0000-0000-000000000002", + "attributes": { + "name": "Secret 2", + }, + }, + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + error_pointers = [e.get("source", {}).get("pointer", "") for e in errors] + assert any("/data/0" in p for p in error_pointers) + assert any("/data/1" in p for p in error_pointers) + + def test_batch_update_at_exact_limit_success( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests at exactly the limit are accepted.""" + limit = settings.API_BATCH_MAX_SIZE + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{400000000000 + i}", + alias=f"aws_update_exact_{i}", + tenant_id=tenant.id, + ) + for i in range(limit) + ] + + secrets = [ + ProviderSecret.objects.create( + provider=providers[i], + tenant_id=tenant.id, + name=f"Secret {i}", + secret_type="static", + secret={ + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLE", + }, + ) + for i in range(limit) + ] + + payload = { + "data": [ + { + "type": "provider-secrets", + "id": str(secrets[i].id), + "attributes": { + "name": f"Updated Secret {i}", + }, + } + for i in range(limit) + ] + } + + response = authenticated_client.patch( + reverse("providersecret-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == limit diff --git a/api/src/backend/api/tests/integration/test_providers_batch.py b/api/src/backend/api/tests/integration/test_providers_batch.py new file mode 100644 index 0000000000..72762df2d0 --- /dev/null +++ b/api/src/backend/api/tests/integration/test_providers_batch.py @@ -0,0 +1,921 @@ +import uuid + +import pytest +from django.conf import settings +from django.test import override_settings +from django.urls import reverse +from rest_framework import status + +from api.models import Provider + + +@pytest.mark.django_db +class TestProviderBatchCreate: + """Tests for the batch provider creation endpoint.""" + + content_type = "application/json" + + def test_batch_create_single_provider_success( + self, authenticated_client, tenants_fixture + ): + """Test creating a single provider via batch endpoint.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "111111111111", + "alias": "Test AWS Account", + }, + } + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["provider"] == "aws" + assert data[0]["attributes"]["uid"] == "111111111111" + assert data[0]["attributes"]["alias"] == "Test AWS Account" + + def test_batch_create_multiple_providers_mixed_types( + self, authenticated_client, tenants_fixture + ): + """Test creating multiple providers of different types in one batch.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "222222222222", + "alias": "AWS Account 1", + }, + }, + { + "type": "providers", + "attributes": { + "provider": "azure", + "uid": "a1b2c3d4-e5f6-4890-abcd-ef1234567890", + "alias": "Azure Subscription", + }, + }, + { + "type": "providers", + "attributes": { + "provider": "gcp", + "uid": "my-gcp-project-id", + "alias": "GCP Project", + }, + }, + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json()["data"] + assert len(data) == 3 + + providers_by_type = {p["attributes"]["provider"]: p for p in data} + assert "aws" in providers_by_type + assert "azure" in providers_by_type + assert "gcp" in providers_by_type + + def test_batch_create_duplicate_uid_in_batch_error( + self, authenticated_client, tenants_fixture + ): + """Test that duplicate UIDs within same batch fails entire batch (all-or-nothing).""" + initial_count = Provider.objects.count() + + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "444444444444", + "alias": "AWS Account 1", + }, + }, + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "444444444444", + "alias": "AWS Account 2 (duplicate)", + }, + }, + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + # All-or-nothing: entire batch fails + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + + # Should have duplicate error + assert any("Duplicate UID" in str(e.get("detail", "")) for e in errors) + + # Verify no providers were created + assert Provider.objects.count() == initial_count + + def test_batch_create_existing_uid_error( + self, authenticated_client, providers_fixture + ): + """Test that UIDs already existing in tenant are rejected.""" + existing_provider = providers_fixture[0] + + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": existing_provider.provider, + "uid": existing_provider.uid, + "alias": "Duplicate of existing", + }, + } + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("already exists" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_invalid_uid_format_error( + self, authenticated_client, tenants_fixture + ): + """Test that invalid UID formats are rejected with proper error messages.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "invalid-aws-uid", + "alias": "Invalid AWS", + }, + } + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("/data/0/attributes" in str(e.get("source", {})) for e in errors) + + def test_batch_create_permission_denied( + self, authenticated_client_no_permissions_rbac, tenants_fixture + ): + """Test that users without MANAGE_PROVIDERS permission cannot batch create.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "555555555555", + "alias": "Test", + }, + } + ] + } + + response = authenticated_client_no_permissions_rbac.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_batch_create_exceeds_limit_error( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests exceeding the limit are rejected.""" + limit = settings.API_BATCH_MAX_SIZE + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": f"{i:012d}", + "alias": f"Provider {i}", + }, + } + for i in range(limit + 1) + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any(f"Maximum {limit}" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_empty_array_error( + self, authenticated_client, tenants_fixture + ): + """Test that empty batch requests are rejected.""" + payload = {"data": []} + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("At least one provider" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_invalid_data_format_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-array data is rejected.""" + payload = { + "data": { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "666666666666", + }, + } + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Must be an array" in str(e.get("detail", "")) for e in errors) + + def test_batch_create_sets_correct_tenant( + self, authenticated_client, tenants_fixture + ): + """Test that batch-created providers have correct tenant assignment.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "777777777777", + "alias": "Tenant 1 Provider", + }, + } + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + assert response.status_code == status.HTTP_201_CREATED + provider_id = response.json()["data"][0]["id"] + + provider = Provider.objects.get(id=provider_id) + assert provider.tenant_id == tenants_fixture[0].id + + def test_batch_create_mixed_valid_invalid_error( + self, authenticated_client, tenants_fixture + ): + """Test that mixed valid/invalid items fails entire batch (all-or-nothing).""" + initial_count = Provider.objects.count() + + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "888888888888", + "alias": "Valid AWS", + }, + }, + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "invalid-uid", + "alias": "Invalid AWS", + }, + }, + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + # All-or-nothing: entire batch fails + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + + # Should have error for invalid item + assert len(errors) >= 1 + assert any( + "/data/1" in str(e.get("source", {}).get("pointer", "")) for e in errors + ) + + # No providers should have been created + assert Provider.objects.count() == initial_count + + def test_batch_create_multiple_errors_reported( + self, authenticated_client, tenants_fixture + ): + """Test that all validation errors are reported, not just the first one.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": "invalid1", + "alias": "Invalid 1", + }, + }, + { + "type": "providers", + "attributes": { + "provider": "azure", + "uid": "not-a-uuid", + "alias": "Invalid 2", + }, + }, + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + # Should have errors for both items + error_pointers = [e.get("source", {}).get("pointer", "") for e in errors] + assert any("/data/0" in p for p in error_pointers) + assert any("/data/1" in p for p in error_pointers) + + def test_batch_create_at_exact_limit_success( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests at exactly the limit are accepted.""" + limit = settings.API_BATCH_MAX_SIZE + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": f"{i:012d}", + "alias": f"Provider {i}", + }, + } + for i in range(limit) + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json()["data"] + assert len(data) == limit + + @override_settings(API_BATCH_MAX_SIZE=5) + def test_batch_create_respects_custom_limit_setting( + self, authenticated_client, tenants_fixture + ): + """Test that batch endpoint respects custom API_BATCH_MAX_SIZE setting.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": f"{900000000000 + i}", + "alias": f"Provider {i}", + }, + } + for i in range(6) + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Maximum 5" in str(e.get("detail", "")) for e in errors) + + @override_settings(API_BATCH_MAX_SIZE=3) + def test_batch_create_at_custom_limit_success( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests at exactly the custom limit are accepted.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": { + "provider": "aws", + "uid": f"{800000000000 + i}", + "alias": f"Provider {i}", + }, + } + for i in range(3) + ] + } + + response = authenticated_client.post( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json()["data"] + assert len(data) == 3 + + +@pytest.mark.django_db +class TestProviderBatchUpdate: + """Tests for the batch provider update endpoint.""" + + content_type = "application/json" + + def test_batch_update_single_provider_success( + self, authenticated_client, providers_fixture + ): + """Test updating a single provider via batch endpoint.""" + provider = providers_fixture[0] + new_alias = "Updated AWS Account" + + payload = { + "data": [ + { + "type": "providers", + "id": str(provider.id), + "attributes": { + "alias": new_alias, + }, + } + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["alias"] == new_alias + + # Verify in database + provider.refresh_from_db() + assert provider.alias == new_alias + + def test_batch_update_multiple_providers_success( + self, authenticated_client, providers_fixture + ): + """Test updating multiple providers in one batch.""" + provider1 = providers_fixture[0] + provider2 = providers_fixture[1] + + payload = { + "data": [ + { + "type": "providers", + "id": str(provider1.id), + "attributes": {"alias": "Updated Provider 1"}, + }, + { + "type": "providers", + "id": str(provider2.id), + "attributes": {"alias": "Updated Provider 2"}, + }, + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + + # Verify in database + provider1.refresh_from_db() + provider2.refresh_from_db() + assert provider1.alias == "Updated Provider 1" + assert provider2.alias == "Updated Provider 2" + + def test_batch_update_provider_not_found_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-existent providers are rejected.""" + fake_id = str(uuid.uuid4()) + + payload = { + "data": [ + { + "type": "providers", + "id": fake_id, + "attributes": {"alias": "New Alias"}, + } + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("not found" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_duplicate_id_in_batch_error( + self, authenticated_client, providers_fixture + ): + """Test that duplicate IDs within same batch fails entire batch (all-or-nothing).""" + provider = providers_fixture[0] + original_alias = provider.alias + + payload = { + "data": [ + { + "type": "providers", + "id": str(provider.id), + "attributes": {"alias": "First Update"}, + }, + { + "type": "providers", + "id": str(provider.id), + "attributes": {"alias": "Second Update (duplicate)"}, + }, + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + # All-or-nothing: entire batch fails + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + + # Should have duplicate error + assert any("Duplicate provider ID" in str(e.get("detail", "")) for e in errors) + + # Verify provider was not updated + provider.refresh_from_db() + assert provider.alias == original_alias + + def test_batch_update_permission_denied( + self, authenticated_client_no_permissions_rbac, providers_fixture + ): + """Test that users without MANAGE_PROVIDERS permission cannot batch update.""" + provider = providers_fixture[0] + + payload = { + "data": [ + { + "type": "providers", + "id": str(provider.id), + "attributes": {"alias": "New Alias"}, + } + ] + } + + response = authenticated_client_no_permissions_rbac.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_batch_update_exceeds_limit_error( + self, authenticated_client, providers_fixture + ): + """Test that batch requests exceeding the limit are rejected.""" + limit = settings.API_BATCH_MAX_SIZE + provider = providers_fixture[0] + + payload = { + "data": [ + { + "type": "providers", + "id": str(provider.id), + "attributes": {"alias": f"Provider {i}"}, + } + for i in range(limit + 1) + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any(f"Maximum {limit}" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_empty_array_error( + self, authenticated_client, tenants_fixture + ): + """Test that empty batch requests are rejected.""" + payload = {"data": []} + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("At least one provider" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_invalid_data_format_error( + self, authenticated_client, tenants_fixture + ): + """Test that non-array data is rejected.""" + payload = { + "data": { + "type": "providers", + "id": str(uuid.uuid4()), + "attributes": {"alias": "Test"}, + } + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Must be an array" in str(e.get("detail", "")) for e in errors) + + def test_batch_update_missing_id_error(self, authenticated_client, tenants_fixture): + """Test that missing ID is rejected.""" + payload = { + "data": [ + { + "type": "providers", + "attributes": {"alias": "New Alias"}, + } + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("required" in str(e.get("detail", "")).lower() for e in errors) + + def test_batch_update_preserves_other_fields( + self, authenticated_client, providers_fixture + ): + """Test that updating alias doesn't change other fields.""" + provider = providers_fixture[0] + original_uid = provider.uid + original_provider_type = provider.provider + + payload = { + "data": [ + { + "type": "providers", + "id": str(provider.id), + "attributes": {"alias": "Updated Alias Only"}, + } + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + + provider.refresh_from_db() + assert provider.alias == "Updated Alias Only" + assert provider.uid == original_uid + assert provider.provider == original_provider_type + + def test_batch_update_multiple_errors_reported( + self, authenticated_client, tenants_fixture + ): + """Test that all validation errors are reported, not just the first one.""" + fake_id1 = str(uuid.uuid4()) + fake_id2 = str(uuid.uuid4()) + + payload = { + "data": [ + { + "type": "providers", + "id": fake_id1, + "attributes": {"alias": "Provider 1"}, + }, + { + "type": "providers", + "id": fake_id2, + "attributes": {"alias": "Provider 2"}, + }, + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + # Should have errors for both items + error_pointers = [e.get("source", {}).get("pointer", "") for e in errors] + assert any("/data/0" in p for p in error_pointers) + assert any("/data/1" in p for p in error_pointers) + + def test_batch_update_at_exact_limit_success( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests at exactly the limit are accepted.""" + limit = settings.API_BATCH_MAX_SIZE + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{700000000000 + i}", + alias=f"Provider {i}", + tenant_id=tenant.id, + ) + for i in range(limit) + ] + + payload = { + "data": [ + { + "type": "providers", + "id": str(providers[i].id), + "attributes": {"alias": f"Updated Provider {i}"}, + } + for i in range(limit) + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == limit + + @override_settings(API_BATCH_MAX_SIZE=5) + def test_batch_update_respects_custom_limit_setting( + self, authenticated_client, tenants_fixture + ): + """Test that batch update endpoint respects custom API_BATCH_MAX_SIZE setting.""" + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{600000000000 + i}", + alias=f"Provider {i}", + tenant_id=tenant.id, + ) + for i in range(6) + ] + + payload = { + "data": [ + { + "type": "providers", + "id": str(providers[i].id), + "attributes": {"alias": f"Updated Provider {i}"}, + } + for i in range(6) + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert any("Maximum 5" in str(e.get("detail", "")) for e in errors) + + @override_settings(API_BATCH_MAX_SIZE=3) + def test_batch_update_at_custom_limit_success( + self, authenticated_client, tenants_fixture + ): + """Test that batch requests at exactly the custom limit are accepted.""" + tenant = tenants_fixture[0] + + providers = [ + Provider.objects.create( + provider="aws", + uid=f"{500000000000 + i}", + alias=f"Provider {i}", + tenant_id=tenant.id, + ) + for i in range(3) + ] + + payload = { + "data": [ + { + "type": "providers", + "id": str(providers[i].id), + "attributes": {"alias": f"Updated Provider {i}"}, + } + for i in range(3) + ] + } + + response = authenticated_client.patch( + reverse("provider-batch"), + data=payload, + content_type=self.content_type, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 3 diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index a1f65404d1..399a6ecf48 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -979,6 +979,283 @@ class ProviderUpdateSerializer(BaseWriteSerializer): } +class ProviderBatchItemSerializer(RLSSerializer, BaseWriteSerializer): + class Meta: + model = Provider + fields = ["alias", "provider", "uid"] + + def validate(self, attrs): + provider_type = attrs.get("provider") + uid = attrs.get("uid") + if provider_type and uid: + validator_method = getattr(Provider, f"validate_{provider_type}_uid", None) + if validator_method: + validator_method(uid) + return attrs + + +class ProviderBatchCreateSerializer(BaseSerializerV1): + """Serializer for batch creation of providers with all-or-nothing semantics (JSON:API compliant).""" + + class Meta: + resource_name = "providers-batch" + + def validate(self, attrs): + data = self.initial_data.get("data", []) + + if not isinstance(data, list): + raise ValidationError({"data": "Must be an array of provider objects"}) + + if len(data) > settings.API_BATCH_MAX_SIZE: + raise ValidationError( + {"data": f"Maximum {settings.API_BATCH_MAX_SIZE} providers per batch"} + ) + + if len(data) == 0: + raise ValidationError({"data": "At least one provider required"}) + + seen_uids = {} + all_errors = [] + validated_items = [] + tenant_id = self.context.get("tenant_id") + + for idx, item in enumerate(data): + current_errors = [] + item_type = item.get("type") + + if not item_type: + current_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + elif item_type != "providers": + current_errors.append( + { + "detail": f"Invalid type '{item_type}'. Expected 'providers'.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + + item_attrs = item.get("attributes", {}) + if not item_attrs: + current_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/attributes"}, + } + ) + all_errors.extend(current_errors) + continue + + provider_type = item_attrs.get("provider") + uid = item_attrs.get("uid") + key = (provider_type, uid) + + # Validate provider type before any DB queries + valid_provider_types = [choice.value for choice in Provider.ProviderChoices] + provider_type_valid = provider_type in valid_provider_types + + if not provider_type: + current_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/attributes/provider"}, + } + ) + elif not provider_type_valid: + current_errors.append( + { + "detail": f"Invalid provider type '{provider_type}'. Must be one of: {', '.join(valid_provider_types)}.", + "source": {"pointer": f"/data/{idx}/attributes/provider"}, + } + ) + + if key in seen_uids: + current_errors.append( + { + "detail": f"Duplicate UID '{uid}' at index {idx} (first at {seen_uids[key]})", + "source": {"pointer": f"/data/{idx}/attributes/uid"}, + } + ) + else: + seen_uids[key] = idx + + # Only check DB if provider type is valid (to avoid enum errors) + if ( + provider_type_valid + and uid + and Provider.objects.filter( + tenant_id=tenant_id, + provider=provider_type, + uid=uid, + is_deleted=False, + ).exists() + ): + current_errors.append( + { + "detail": f"Provider with uid '{uid}' already exists", + "source": {"pointer": f"/data/{idx}/attributes/uid"}, + } + ) + + item_serializer = ProviderBatchItemSerializer( + data=item_attrs, context=self.context + ) + if not item_serializer.is_valid(): + for field, field_errors in item_serializer.errors.items(): + current_errors.append( + { + "detail": str(field_errors[0]), + "source": {"pointer": f"/data/{idx}/attributes/{field}"}, + } + ) + + if current_errors: + all_errors.extend(current_errors) + else: + validated_items.append( + {"index": idx, "data": item_serializer.validated_data} + ) + + # All-or-nothing: if any errors, fail the entire batch + if all_errors: + raise ValidationError(all_errors) + + attrs["_validated_items"] = validated_items + return attrs + + +class ProviderBatchUpdateItemSerializer(BaseWriteSerializer): + """Serializer for validating individual provider update items in batch.""" + + class Meta: + model = Provider + fields = ["alias"] + + +class ProviderBatchUpdateSerializer(BaseSerializerV1): + """Serializer for batch update of providers with all-or-nothing semantics (JSON:API compliant).""" + + class Meta: + resource_name = "providers-batch" + + def validate(self, attrs): + data = self.initial_data.get("data", []) + + if not isinstance(data, list): + raise ValidationError({"data": "Must be an array of provider objects"}) + + if len(data) > settings.API_BATCH_MAX_SIZE: + raise ValidationError( + {"data": f"Maximum {settings.API_BATCH_MAX_SIZE} providers per batch"} + ) + + if len(data) == 0: + raise ValidationError({"data": "At least one provider required"}) + + all_errors = [] + validated_items = [] + tenant_id = self.context.get("tenant_id") + seen_ids = {} + + for idx, item in enumerate(data): + current_errors = [] + item_type = item.get("type") + item_id = item.get("id") + + if not item_type: + current_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + elif item_type != "providers": + current_errors.append( + { + "detail": f"Invalid type '{item_type}'. Expected 'providers'.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + + if not item_id: + current_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/id"}, + } + ) + all_errors.extend(current_errors) + continue + + if item_id in seen_ids: + current_errors.append( + { + "detail": f"Duplicate provider ID '{item_id}' at index {idx} (first at {seen_ids[item_id]})", + "source": {"pointer": f"/data/{idx}/id"}, + } + ) + else: + seen_ids[item_id] = idx + + try: + provider = Provider.objects.get( + id=item_id, tenant_id=tenant_id, is_deleted=False + ) + except Provider.DoesNotExist: + current_errors.append( + { + "detail": f"Provider '{item_id}' not found.", + "source": {"pointer": f"/data/{idx}/id"}, + } + ) + all_errors.extend(current_errors) + continue + + item_attrs = item.get("attributes", {}) + if not item_attrs: + current_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/attributes"}, + } + ) + all_errors.extend(current_errors) + continue + + item_serializer = ProviderBatchUpdateItemSerializer( + data=item_attrs, context=self.context + ) + if not item_serializer.is_valid(): + for field, field_errors in item_serializer.errors.items(): + current_errors.append( + { + "detail": str(field_errors[0]), + "source": {"pointer": f"/data/{idx}/attributes/{field}"}, + } + ) + + if current_errors: + all_errors.extend(current_errors) + else: + validated_items.append( + { + "index": idx, + "provider": provider, + **item_serializer.validated_data, + } + ) + + # All-or-nothing: if any errors, fail the entire batch + if all_errors: + raise ValidationError(all_errors) + + attrs["_validated_items"] = validated_items + return attrs + + # Scans @@ -1775,6 +2052,446 @@ class ProviderSecretUpdateSerializer(BaseWriteProviderSecretSerializer): return validated_attrs +class ProviderSecretBatchItemSerializer(BaseWriteProviderSecretSerializer): + """Serializer for an individual item in the batch of secrets.""" + + secret = ProviderSecretField(write_only=True) + + class Meta: + model = ProviderSecret + fields = ["name", "secret_type", "secret"] + + def validate(self, attrs): + # Provider is passed via context since it's validated separately + provider = self.context.get("provider") + secret_type = attrs.get("secret_type") + secret = attrs.get("secret") + + if provider and secret_type and secret: + self.validate_secret_based_on_provider( + provider.provider, secret_type, secret + ) + return attrs + + +class ProviderSecretBatchCreateSerializer(BaseSerializerV1): + """ + Serializer for batch creation of provider secrets. + + Supports to-many relationship format where one secret definition can be + associated with multiple providers. Each provider creates a separate secret. + + JSON:API compliant: all-or-nothing for hard errors, soft skips for providers + that already have secrets (reported in meta.skipped). + """ + + class Meta: + resource_name = "provider-secrets-batch" + + def _extract_providers_data(self, relationships, idx): + """ + Extract providers data from relationships, supporting both formats: + - to-one: relationships.provider.data (single object) - backwards compatible + - to-many: relationships.providers.data (array) - new format + + Returns (providers_list, errors) where providers_list is normalized to array. + """ + errors = [] + + # Try to-many format first (providers plural) + providers_rel = relationships.get("providers", {}) + providers_data = providers_rel.get("data") + + if providers_data is not None: + # to-many format + if isinstance(providers_data, dict): + # Single object in to-many field - normalize to array + providers_data = [providers_data] + elif not isinstance(providers_data, list): + errors.append( + { + "detail": "Must be an array of provider resource identifiers.", + "source": { + "pointer": f"/data/{idx}/relationships/providers/data" + }, + } + ) + return None, errors + return providers_data, errors + + # Fall back to to-one format (provider singular) for backwards compatibility + provider_rel = relationships.get("provider", {}) + provider_data = provider_rel.get("data") + + if provider_data is not None: + if isinstance(provider_data, dict): + return [provider_data], errors + else: + errors.append( + { + "detail": "Must be a provider resource identifier object.", + "source": { + "pointer": f"/data/{idx}/relationships/provider/data" + }, + } + ) + return None, errors + + # No providers relationship found + errors.append( + { + "detail": "Providers relationship is required.", + "source": {"pointer": f"/data/{idx}/relationships/providers"}, + } + ) + return None, errors + + def validate(self, attrs): + data = self.initial_data.get("data", []) + + if not isinstance(data, list): + raise ValidationError( + {"data": "Must be an array of provider-secret objects"} + ) + + if len(data) > settings.API_BATCH_MAX_SIZE: + raise ValidationError( + {"data": f"Maximum {settings.API_BATCH_MAX_SIZE} secrets per batch"} + ) + + if len(data) == 0: + raise ValidationError({"data": "At least one secret required"}) + + hard_errors = [] # Will cause full batch failure + skipped_providers = [] # Already have secrets, reported in meta + validated_items = [] + tenant_id = self.context.get("tenant_id") + seen_providers = {} # Track duplicates across the entire batch + + for idx, item in enumerate(data): + item_type = item.get("type") + + # Validate type + if not item_type: + hard_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + continue + elif item_type != "provider-secrets": + hard_errors.append( + { + "detail": f"Invalid type '{item_type}'. Expected 'provider-secrets'.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + continue + + # Validate attributes + item_attrs = item.get("attributes", {}) + if not item_attrs: + hard_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/attributes"}, + } + ) + continue + + # Extract providers (supports both to-one and to-many formats) + relationships = item.get("relationships", {}) + providers_data, extract_errors = self._extract_providers_data( + relationships, idx + ) + if extract_errors: + hard_errors.extend(extract_errors) + continue + + if not providers_data: + hard_errors.append( + { + "detail": "At least one provider is required.", + "source": { + "pointer": f"/data/{idx}/relationships/providers/data" + }, + } + ) + continue + + # Process each provider in the relationship + for prov_idx, prov_data in enumerate(providers_data): + provider_id = ( + prov_data.get("id") if isinstance(prov_data, dict) else None + ) + provider_type = ( + prov_data.get("type") if isinstance(prov_data, dict) else None + ) + + # Validate provider resource identifier + if not provider_id: + hard_errors.append( + { + "detail": "Provider id is required.", + "source": { + "pointer": f"/data/{idx}/relationships/providers/data/{prov_idx}/id" + }, + } + ) + continue + + if provider_type and provider_type != "providers": + hard_errors.append( + { + "detail": f"Invalid type '{provider_type}'. Expected 'providers'.", + "source": { + "pointer": f"/data/{idx}/relationships/providers/data/{prov_idx}/type" + }, + } + ) + continue + + # Check for duplicate provider in entire batch + if provider_id in seen_providers: + prev_idx, prev_prov_idx = seen_providers[provider_id] + hard_errors.append( + { + "detail": f"Duplicate provider '{provider_id}' (first at data/{prev_idx}/providers/{prev_prov_idx}).", + "source": { + "pointer": f"/data/{idx}/relationships/providers/data/{prov_idx}" + }, + } + ) + continue + + seen_providers[provider_id] = (idx, prov_idx) + + # Validate provider exists and belongs to tenant + try: + provider = Provider.objects.get( + id=provider_id, tenant_id=tenant_id, is_deleted=False + ) + except Provider.DoesNotExist: + hard_errors.append( + { + "detail": f"Provider '{provider_id}' not found.", + "source": { + "pointer": f"/data/{idx}/relationships/providers/data/{prov_idx}" + }, + } + ) + continue + + # Soft skip: provider already has a secret + if ProviderSecret.objects.filter( + provider_id=provider_id, tenant_id=tenant_id + ).exists(): + skipped_providers.append( + { + "provider_id": str(provider_id), + "source_index": idx, + "reason": "Provider already has a secret.", + } + ) + continue + + # Validate secret attributes for this specific provider + item_context = {**self.context, "provider": provider} + item_serializer = ProviderSecretBatchItemSerializer( + data=item_attrs, context=item_context + ) + + if not item_serializer.is_valid(): + for field, field_errors in item_serializer.errors.items(): + pointer = f"/data/{idx}/attributes/{field}" + hard_errors.append( + { + "detail": str(field_errors[0]), + "source": {"pointer": pointer}, + } + ) + continue + + validated_items.append( + { + "source_index": idx, + **item_serializer.validated_data, + "provider": provider, + } + ) + + # All-or-nothing: if any hard errors, fail the entire batch + if hard_errors: + raise ValidationError(hard_errors) + + attrs["_validated_items"] = validated_items + attrs["_skipped_providers"] = skipped_providers + return attrs + + +class ProviderSecretBatchUpdateItemSerializer(BaseWriteProviderSecretSerializer): + """Serializer for validating individual provider secret update items in batch.""" + + secret = ProviderSecretField(write_only=True, required=False) + + class Meta: + model = ProviderSecret + fields = ["name", "secret_type", "secret"] + extra_kwargs = { + "name": {"required": False}, + "secret_type": {"required": False}, + } + + def validate(self, attrs): + provider = self.context.get("provider") + secret_type = attrs.get("secret_type") + secret = attrs.get("secret") + + if provider and secret_type and secret: + self.validate_secret_based_on_provider( + provider.provider, secret_type, secret + ) + elif provider and secret and not secret_type: + existing_secret = ProviderSecret.objects.filter(provider=provider).first() + if existing_secret: + self.validate_secret_based_on_provider( + provider.provider, existing_secret.secret_type, secret + ) + return attrs + + +class ProviderSecretBatchUpdateSerializer(BaseSerializerV1): + """ + Serializer for batch update of provider secrets. + + JSON:API compliant with all-or-nothing semantics for validation errors. + """ + + class Meta: + resource_name = "provider-secrets-batch" + + def validate(self, attrs): + data = self.initial_data.get("data", []) + + if not isinstance(data, list): + raise ValidationError( + {"data": "Must be an array of provider-secret objects"} + ) + + if len(data) > settings.API_BATCH_MAX_SIZE: + raise ValidationError( + {"data": f"Maximum {settings.API_BATCH_MAX_SIZE} secrets per batch"} + ) + + if len(data) == 0: + raise ValidationError({"data": "At least one secret required"}) + + hard_errors = [] # Will cause full batch failure + validated_items = [] + tenant_id = self.context.get("tenant_id") + seen_ids = {} + + for idx, item in enumerate(data): + item_type = item.get("type") + item_id = item.get("id") + + # Validate type + if not item_type: + hard_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + continue + elif item_type != "provider-secrets": + hard_errors.append( + { + "detail": f"Invalid type '{item_type}'. Expected 'provider-secrets'.", + "source": {"pointer": f"/data/{idx}/type"}, + } + ) + continue + + # Validate id + if not item_id: + hard_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/id"}, + } + ) + continue + + # Check for duplicate id in batch + if item_id in seen_ids: + hard_errors.append( + { + "detail": f"Duplicate secret ID '{item_id}' (first at data/{seen_ids[item_id]}).", + "source": {"pointer": f"/data/{idx}/id"}, + } + ) + continue + + seen_ids[item_id] = idx + + # Validate secret exists and belongs to tenant + try: + provider_secret = ProviderSecret.objects.select_related("provider").get( + id=item_id, tenant_id=tenant_id + ) + except ProviderSecret.DoesNotExist: + hard_errors.append( + { + "detail": f"Provider secret '{item_id}' not found.", + "source": {"pointer": f"/data/{idx}/id"}, + } + ) + continue + + # Validate attributes + item_attrs = item.get("attributes", {}) + if not item_attrs: + hard_errors.append( + { + "detail": "This field is required.", + "source": {"pointer": f"/data/{idx}/attributes"}, + } + ) + continue + + item_context = {**self.context, "provider": provider_secret.provider} + + item_serializer = ProviderSecretBatchUpdateItemSerializer( + data=item_attrs, context=item_context + ) + if not item_serializer.is_valid(): + for field, field_errors in item_serializer.errors.items(): + hard_errors.append( + { + "detail": str(field_errors[0]), + "source": {"pointer": f"/data/{idx}/attributes/{field}"}, + } + ) + continue + + validated_items.append( + { + "source_index": idx, + "provider_secret": provider_secret, + **item_serializer.validated_data, + } + ) + + # All-or-nothing: if any hard errors, fail the entire batch + if hard_errors: + raise ValidationError(hard_errors) + + attrs["_validated_items"] = validated_items + return attrs + + # Invitations diff --git a/api/src/backend/api/v1/urls.py b/api/src/backend/api/v1/urls.py index 840f027b42..e08f98ca9d 100644 --- a/api/src/backend/api/v1/urls.py +++ b/api/src/backend/api/v1/urls.py @@ -111,6 +111,18 @@ urlpatterns = [ ProviderSecretViewSet.as_view({"get": "list", "post": "create"}), name="providersecret-list", ), + path( + "providers/secrets/batch", + ProviderSecretViewSet.as_view( + {"post": "batch_create", "patch": "batch_update"} + ), + name="providersecret-batch", + ), + path( + "providers/batch", + ProviderViewSet.as_view({"post": "batch_create", "patch": "batch_update"}), + name="provider-batch", + ), path( "providers/secrets/", ProviderSecretViewSet.as_view( diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index c7b38e0ef8..40ce930be0 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -3,7 +3,6 @@ import glob import json import logging import os - from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone @@ -11,7 +10,6 @@ from decimal import ROUND_HALF_UP, Decimal, InvalidOperation from urllib.parse import urljoin import sentry_sdk - from allauth.socialaccount.models import SocialAccount, SocialApp from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter @@ -46,7 +44,7 @@ from django.db.models import ( Window, ) from django.db.models.functions import Coalesce, RowNumber -from django.http import HttpResponse, QueryDict +from django.http import HttpResponse, JsonResponse, QueryDict from django.shortcuts import redirect from django.urls import reverse from django.utils.dateparse import parse_date @@ -72,15 +70,30 @@ from rest_framework.exceptions import ( ValidationError, ) from rest_framework.generics import GenericAPIView, get_object_or_404 +from rest_framework.parsers import JSONParser from rest_framework.permissions import SAFE_METHODS from rest_framework_json_api.views import RelationshipView, Response from rest_framework_simplejwt.exceptions import InvalidToken, TokenError - -from api.attack_paths import ( - get_queries_for_provider, - get_query_by_id, - views_helpers as attack_paths_views_helpers, +from tasks.beat import schedule_provider_scan +from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils +from tasks.jobs.export import get_s3_client +from tasks.tasks import ( + backfill_compliance_summaries_task, + backfill_scan_resource_summaries_task, + check_integration_connection_task, + check_lighthouse_connection_task, + check_lighthouse_provider_connection_task, + check_provider_connection_task, + delete_provider_task, + delete_tenant_task, + jira_integration_task, + mute_historical_findings_task, + perform_scan_task, + refresh_lighthouse_provider_models_task, ) + +from api.attack_paths import get_queries_for_provider, get_query_by_id +from api.attack_paths import views_helpers as attack_paths_views_helpers from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset from api.compliance import ( PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, @@ -90,6 +103,7 @@ from api.db_router import MainRouter from api.db_utils import rls_transaction from api.exceptions import TaskFailedException from api.filters import ( + AttackPathsScanFilter, AttackSurfaceOverviewFilter, CategoryOverviewFilter, ComplianceOverviewFilter, @@ -102,7 +116,6 @@ from api.filters import ( InvitationFilter, LatestFindingFilter, LatestResourceFilter, - AttackPathsScanFilter, LighthouseProviderConfigFilter, LighthouseProviderModelsFilter, MembershipFilter, @@ -124,6 +137,7 @@ from api.filters import ( UserFilter, ) from api.models import ( + AttackPathsScan, AttackSurfaceOverview, ComplianceOverviewSummary, ComplianceRequirementOverview, @@ -131,7 +145,6 @@ from api.models import ( Finding, Integration, Invitation, - AttackPathsScan, LighthouseConfiguration, LighthouseProviderConfiguration, LighthouseProviderModels, @@ -177,9 +190,9 @@ from api.utils import ( from api.uuid_utils import datetime_to_uuid7, uuid7_start from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin from api.v1.serializers import ( + AttackPathsQueryResultSerializer, AttackPathsQueryRunRequestSerializer, AttackPathsQuerySerializer, - AttackPathsQueryResultSerializer, AttackPathsScanSerializer, AttackSurfaceOverviewSerializer, CategoryOverviewSerializer, @@ -223,11 +236,15 @@ from api.v1.serializers import ( ProcessorCreateSerializer, ProcessorSerializer, ProcessorUpdateSerializer, + ProviderBatchCreateSerializer, + ProviderBatchUpdateSerializer, ProviderCreateSerializer, ProviderGroupCreateSerializer, ProviderGroupMembershipSerializer, ProviderGroupSerializer, ProviderGroupUpdateSerializer, + ProviderSecretBatchCreateSerializer, + ProviderSecretBatchUpdateSerializer, ProviderSecretCreateSerializer, ProviderSecretSerializer, ProviderSecretUpdateSerializer, @@ -263,23 +280,6 @@ from api.v1.serializers import ( UserSerializer, UserUpdateSerializer, ) -from tasks.beat import schedule_provider_scan -from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils -from tasks.jobs.export import get_s3_client -from tasks.tasks import ( - backfill_compliance_summaries_task, - backfill_scan_resource_summaries_task, - check_integration_connection_task, - check_lighthouse_connection_task, - check_lighthouse_provider_connection_task, - check_provider_connection_task, - delete_provider_task, - delete_tenant_task, - jira_integration_task, - mute_historical_findings_task, - perform_scan_task, - refresh_lighthouse_provider_models_task, -) logger = logging.getLogger(BackendLogger.API) @@ -1543,12 +1543,26 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): def get_serializer_class(self): if self.action == "create": return ProviderCreateSerializer + elif self.action == "batch_create": + return ProviderBatchCreateSerializer + elif self.action == "batch_update": + return ProviderBatchUpdateSerializer elif self.action == "partial_update": return ProviderUpdateSerializer elif self.action in ["connection", "destroy"]: return TaskSerializer return super().get_serializer_class() + def get_parsers(self): + # Check action_map since self.action is not set yet during parser initialization + action_map = getattr(self, "action_map", {}) + if ( + "batch_create" in action_map.values() + or "batch_update" in action_map.values() + ): + return [JSONParser()] + return super().get_parsers() + def partial_update(self, request, *args, **kwargs): instance = self.get_object() serializer = self.get_serializer( @@ -1613,6 +1627,193 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): }, ) + @extend_schema( + tags=["Provider"], + summary="Batch create providers", + description="Create multiple providers in a single atomic operation. JSON:API compliant with all-or-nothing semantics. Secrets must be added separately via the provider secrets endpoint.", + request={ + "application/json": { + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "array", + "description": "Array of provider objects to create (max 100)", + "items": { + "type": "object", + "required": ["type", "attributes"], + "properties": { + "type": { + "type": "string", + "enum": ["providers"], + "description": "Must be 'providers'", + }, + "attributes": { + "type": "object", + "required": ["provider", "uid"], + "properties": { + "provider": { + "type": "string", + "enum": [ + "aws", + "azure", + "gcp", + "kubernetes", + "github", + "m365", + ], + "description": "Cloud provider type", + }, + "uid": { + "type": "string", + "description": "Provider unique identifier (e.g., AWS 12-digit account ID)", + }, + "alias": { + "type": "string", + "description": "Optional display name", + }, + }, + }, + }, + }, + }, + }, + }, + }, + responses={ + 201: OpenApiResponse( + response=ProviderSerializer(many=True), + description="All providers created successfully", + ), + 400: OpenApiResponse( + description="Validation errors - entire batch rejected (all-or-nothing)" + ), + }, + ) + @action( + detail=False, + methods=["post"], + url_path="batch", + url_name="batch", + parser_classes=[JSONParser], + ) + def batch_create(self, request): + serializer = ProviderBatchCreateSerializer( + data=request.data, context=self.get_serializer_context() + ) + # All-or-nothing: raise ValidationError if any item fails validation + serializer.is_valid(raise_exception=True) + + validated_items = serializer.validated_data.get("_validated_items", []) + + # Create all providers + created_provider_ids = [] + for item in validated_items: + item_data = item["data"] + provider = Provider.objects.create(tenant_id=request.tenant_id, **item_data) + created_provider_ids.append(provider.id) + + created_providers = ( + Provider.objects.filter(id__in=created_provider_ids) + .select_related("secret") + .prefetch_related("provider_groups") + ) + + response_serializer = ProviderSerializer( + created_providers, many=True, context=self.get_serializer_context() + ) + + return Response(data=response_serializer.data, status=status.HTTP_201_CREATED) + + @extend_schema( + tags=["Provider"], + summary="Batch update providers", + description="Update multiple providers in a single atomic operation. JSON:API compliant with all-or-nothing semantics. Only alias can be updated.", + request={ + "application/json": { + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "array", + "description": "Array of provider objects to update (max 100)", + "items": { + "type": "object", + "required": ["type", "id", "attributes"], + "properties": { + "type": { + "type": "string", + "enum": ["providers"], + "description": "Must be 'providers'", + }, + "id": { + "type": "string", + "format": "uuid", + "description": "Provider UUID to update", + }, + "attributes": { + "type": "object", + "properties": { + "alias": { + "type": "string", + "description": "New display name for the provider", + }, + }, + }, + }, + }, + }, + }, + }, + }, + responses={ + 200: OpenApiResponse( + response=ProviderSerializer(many=True), + description="All providers updated successfully", + ), + 400: OpenApiResponse( + description="Validation errors - entire batch rejected (all-or-nothing)" + ), + }, + ) + @action( + detail=False, + methods=["patch"], + url_path="batch", + url_name="batch-update", + parser_classes=[JSONParser], + ) + def batch_update(self, request): + serializer = ProviderBatchUpdateSerializer( + data=request.data, context=self.get_serializer_context() + ) + # All-or-nothing: raise ValidationError if any item fails validation + serializer.is_valid(raise_exception=True) + + validated_items = serializer.validated_data.get("_validated_items", []) + + # Update all providers + updated_provider_ids = [] + for item_data in validated_items: + item_data.pop("index") + provider = item_data.pop("provider") + for key, value in item_data.items(): + setattr(provider, key, value) + provider.save() + updated_provider_ids.append(provider.id) + + updated_providers = ( + Provider.objects.filter(id__in=updated_provider_ids) + .select_related("secret") + .prefetch_related("provider_groups") + ) + + response_serializer = ProviderSerializer( + updated_providers, many=True, context=self.get_serializer_context() + ) + + return Response(data=response_serializer.data, status=status.HTTP_200_OK) + @extend_schema_view( list=extend_schema( @@ -3322,10 +3523,263 @@ class ProviderSecretViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "create": return ProviderSecretCreateSerializer + elif self.action == "batch_create": + return ProviderSecretBatchCreateSerializer + elif self.action == "batch_update": + return ProviderSecretBatchUpdateSerializer elif self.action == "partial_update": return ProviderSecretUpdateSerializer return super().get_serializer_class() + def get_parsers(self): + # Check action_map since self.action is not set yet during parser initialization + action_map = getattr(self, "action_map", {}) + if ( + "batch_create" in action_map.values() + or "batch_update" in action_map.values() + ): + return [JSONParser()] + return super().get_parsers() + + @extend_schema( + tags=["Provider"], + summary="Batch create provider secrets", + description=( + "Create multiple provider secrets in a single atomic operation. " + "Supports to-many relationships where one secret definition can be " + "associated with multiple providers. JSON:API compliant with all-or-nothing " + "semantics for hard errors. Providers that already have secrets are skipped " + "and reported in meta.skipped." + ), + request={ + "application/json": { + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "array", + "description": "Array of provider-secret objects (max 100)", + "items": { + "type": "object", + "required": ["type", "attributes", "relationships"], + "properties": { + "type": { + "type": "string", + "enum": ["provider-secrets"], + }, + "attributes": { + "type": "object", + "required": ["secret_type", "secret"], + "properties": { + "name": {"type": "string"}, + "secret_type": { + "type": "string", + "enum": [ + "static", + "role", + "service_account", + ], + }, + "secret": { + "type": "object", + "description": "Credentials (varies by provider/secret_type)", + }, + }, + }, + "relationships": { + "type": "object", + "required": ["providers"], + "properties": { + "providers": { + "type": "object", + "description": "To-many relationship: one secret can be created for multiple providers", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "required": ["type", "id"], + "properties": { + "type": { + "type": "string", + "enum": ["providers"], + }, + "id": { + "type": "string", + "format": "uuid", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + responses={ + 201: OpenApiResponse( + description=( + "All secrets created successfully. Response includes meta.skipped " + "if any providers were skipped because they already have secrets." + ), + ), + 400: OpenApiResponse( + description="Validation errors - entire batch rejected (all-or-nothing)" + ), + }, + ) + @action( + detail=False, + methods=["post"], + url_path="batch", + url_name="batch", + parser_classes=[JSONParser], + ) + def batch_create(self, request): + serializer = ProviderSecretBatchCreateSerializer( + data=request.data, context=self.get_serializer_context() + ) + # All-or-nothing: raises ValidationError on any hard error + serializer.is_valid(raise_exception=True) + + validated_items = serializer.validated_data.get("_validated_items", []) + skipped_providers = serializer.validated_data.get("_skipped_providers", []) + + # Create all validated secrets + created_secret_ids = [] + for item_data in validated_items: + item_data.pop("source_index", None) + provider = item_data.pop("provider") + secret = ProviderSecret.objects.create( + tenant_id=request.tenant_id, provider=provider, **item_data + ) + created_secret_ids.append(secret.id) + + created_secrets = ProviderSecret.objects.filter( + id__in=created_secret_ids + ).select_related("provider") + + response_serializer = ProviderSecretSerializer( + created_secrets, many=True, context=self.get_serializer_context() + ) + + # Include skipped providers in meta if any + if skipped_providers: + # Use JsonResponse for custom meta - builds JSON:API structure manually + return JsonResponse( + { + "data": response_serializer.data, + "meta": {"skipped": skipped_providers}, + }, + status=201, + ) + + # Standard response without meta + return Response(data=response_serializer.data, status=status.HTTP_201_CREATED) + + @extend_schema( + tags=["Provider"], + summary="Batch update provider secrets", + description=( + "Update multiple provider secrets in a single atomic operation. " + "JSON:API compliant with all-or-nothing semantics." + ), + request={ + "application/json": { + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "array", + "description": "Array of provider-secret objects (max 100)", + "items": { + "type": "object", + "required": ["type", "id", "attributes"], + "properties": { + "type": { + "type": "string", + "enum": ["provider-secrets"], + }, + "id": { + "type": "string", + "format": "uuid", + "description": "Provider secret UUID to update", + }, + "attributes": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "secret_type": { + "type": "string", + "enum": [ + "static", + "role", + "service_account", + ], + }, + "secret": { + "type": "object", + "description": "Credentials (varies by provider/secret_type)", + }, + }, + }, + }, + }, + }, + }, + }, + }, + responses={ + 200: OpenApiResponse( + response=ProviderSecretSerializer(many=True), + description="All secrets updated successfully", + ), + 400: OpenApiResponse( + description="Validation errors - entire batch rejected (all-or-nothing)" + ), + }, + ) + @action( + detail=False, + methods=["patch"], + url_path="batch", + url_name="batch-update", + parser_classes=[JSONParser], + ) + def batch_update(self, request): + serializer = ProviderSecretBatchUpdateSerializer( + data=request.data, context=self.get_serializer_context() + ) + # All-or-nothing: raises ValidationError on any hard error + serializer.is_valid(raise_exception=True) + + validated_items = serializer.validated_data.get("_validated_items", []) + + # Update all validated secrets + updated_secret_ids = [] + for item_data in validated_items: + item_data.pop("source_index", None) + provider_secret = item_data.pop("provider_secret") + for key, value in item_data.items(): + setattr(provider_secret, key, value) + provider_secret.save() + updated_secret_ids.append(provider_secret.id) + + updated_secrets = ProviderSecret.objects.filter( + id__in=updated_secret_ids + ).select_related("provider") + + response_serializer = ProviderSecretSerializer( + updated_secrets, many=True, context=self.get_serializer_context() + ) + + return Response(data=response_serializer.data, status=status.HTTP_200_OK) + @extend_schema_view( list=extend_schema( diff --git a/api/src/backend/config/django/base.py b/api/src/backend/config/django/base.py index 80b96952d7..7d399bef91 100644 --- a/api/src/backend/config/django/base.py +++ b/api/src/backend/config/django/base.py @@ -293,6 +293,8 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" DJANGO_DELETION_BATCH_SIZE = env.int("DJANGO_DELETION_BATCH_SIZE", 5000) +API_BATCH_MAX_SIZE = env.int("DJANGO_API_BATCH_MAX_SIZE", 100) + # SAML requirement CSRF_COOKIE_SECURE = True SESSION_COOKIE_SECURE = True