Merge remote-tracking branch 'origin/PROWLER-1777-dynamic-compliance-framework-discovery' into PROWLER-1778-dynamic-output-report-generation

api/src/backend/api/filters.py

@@ -1,5 +1,4 @@
 from datetime import date, datetime, timedelta, timezone
-from functools import lru_cache
 
 from dateutil.parser import parse
 from django.conf import settings
@@ -67,19 +66,8 @@ from api.uuid_utils import (
     uuid7_range,
     uuid7_start,
 )
+from api.provider_types import get_provider_type_choices
 from api.v1.serializers import TaskBase
-from prowler.providers.common.provider import Provider as SDKProvider
-
-
-@lru_cache(maxsize=1)
-def get_provider_type_choices():
-    """Provider-type filter choices driven by the SDK's available providers
-    instead of a static enum, so filtering covers external providers too.
-
-    Cached because the installed providers are fixed for the process lifetime
-    and provider-type filters live on hot list endpoints.
-    """
-    return [(name, name) for name in SDKProvider.get_available_providers()]
 
 
 class CustomDjangoFilterBackend(DjangoFilterBackend):

api/src/backend/api/migrations/0095_backfill_provider_str.py

@@ -1,31 +1,25 @@
 from django.db import migrations
-from tasks.tasks import backfill_provider_str_task
-
-from api.db_router import MainRouter
-
-
-def trigger_provider_str_backfill(apps, _schema_editor):
-    """Dispatch a per-tenant Celery task to populate the transitional
-    `provider_str` shadow column for rows created before the sync trigger
-    from 0094 existed.
-
-    New writes are already covered by the trigger, so this only fills the gap
-    left by pre-existing rows. The work runs in the background, batched per
-    tenant, so the migration itself finishes in seconds regardless of table
-    size.
-    """
-    Tenant = apps.get_model("api", "Tenant")
-    tenant_ids = Tenant.objects.using(MainRouter.admin_db).values_list("id", flat=True)
-
-    for tenant_id in tenant_ids:
-        backfill_provider_str_task.delay(tenant_id=str(tenant_id))
 
 
 class Migration(migrations.Migration):
+    """Synchronous backfill of the `provider_str` shadow column.
+
+    A single UPDATE fills rows that predate the 0094 trigger. The providers
+    table is small, so this is safe inline and guarantees the column is fully
+    populated before 0096 sets it NOT NULL (no race with an async backfill).
+    Runs on the migration connection, which is exempt from RLS.
+    """
+
     dependencies = [
         ("api", "0094_provider_str_shadow_column"),
     ]
 
     operations = [
-        migrations.RunPython(trigger_provider_str_backfill, migrations.RunPython.noop),
+        migrations.RunSQL(
+            sql=(
+                "UPDATE providers SET provider_str = provider::text "
+                "WHERE provider_str IS NULL;"
+            ),
+            reverse_sql=migrations.RunSQL.noop,
+        ),
     ]

api/src/backend/api/migrations/0096_provider_enum_to_varchar_contract.py

@@ -2,20 +2,14 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-    """Contract step of the zero-downtime migration of Provider.provider from a
-    native PostgreSQL enum to varchar.
+    """Contract step: promote `provider_str` into `provider`.
 
-    The shadow column added in 0094 has been kept in sync by the trigger and
-    backfilled in 0095, so it now holds the value for every row. This migration
-    promotes it into place: drop the trigger and the enum column, rename the
-    shadow column to `provider`, and drop the orphaned enum type. The column
-    name is preserved throughout, and varchar accepts the same string values
-    the enum held, so app instances running the previous release keep working
-    against the swapped column.
-
-    The drop/rename runs in this migration's transaction so `provider` never
-    disappears for readers. The partial unique index is dropped here and
-    rebuilt concurrently in the next migration to avoid a long write lock.
+    Drops the trigger and enum column, renames the shadow column, sets it NOT
+    NULL, and drops the enum type. The unique index is dropped and recreated in
+    the same transaction, so there is no window for duplicate active providers;
+    recreated non-concurrently since the table is small, with a short
+    lock_timeout so the migration fails fast instead of queueing behind a
+    long-running transaction.
     """
 
     dependencies = [
@@ -38,6 +32,7 @@ class Migration(migrations.Migration):
             database_operations=[
                 migrations.RunSQL(
                     sql=(
+                        "SET LOCAL lock_timeout = '10s';\n"
                         "DROP TRIGGER IF EXISTS providers_sync_provider_str ON providers;\n"
                         "DROP FUNCTION IF EXISTS sync_provider_str();\n"
                         "DROP INDEX IF EXISTS unique_provider_uids;\n"
@@ -45,7 +40,9 @@ class Migration(migrations.Migration):
                         "ALTER TABLE providers RENAME COLUMN provider_str TO provider;\n"
                         "ALTER TABLE providers ALTER COLUMN provider SET DEFAULT 'aws';\n"
                         "ALTER TABLE providers ALTER COLUMN provider SET NOT NULL;\n"
-                        "DROP TYPE provider;"
+                        "DROP TYPE provider;\n"
+                        "CREATE UNIQUE INDEX unique_provider_uids ON providers "
+                        "(tenant_id, provider, uid) WHERE NOT is_deleted;"
                     ),
                     reverse_sql=migrations.RunSQL.noop,
                 ),

api/src/backend/api/migrations/0097_provider_unique_index_concurrent.py

@@ -1,28 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    """Rebuild the partial unique index on `providers` after the enum-to-varchar
-    contract in 0096 dropped it along with the old enum column.
-
-    Built with CREATE INDEX CONCURRENTLY (hence `atomic = False`) so the rebuild
-    holds no long write lock on a large table. The index keeps the name and
-    predicate Django expects for the existing `unique_provider_uids` constraint,
-    which stays in the model state untouched, so no state operation is needed.
-    """
-
-    atomic = False
-
-    dependencies = [
-        ("api", "0096_provider_enum_to_varchar_contract"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            sql=(
-                "CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS unique_provider_uids "
-                "ON providers (tenant_id, provider, uid) WHERE NOT is_deleted;"
-            ),
-            reverse_sql="DROP INDEX CONCURRENTLY IF EXISTS unique_provider_uids;",
-        ),
-    ]

api/src/backend/api/provider_types.py

@@ -0,0 +1,15 @@
+from functools import lru_cache
+
+from prowler.providers.common.provider import Provider as SDKProvider
+
+
+@lru_cache(maxsize=1)
+def get_provider_type_choices():
+    """Provider-type choices from the SDK's available providers, so they cover
+    external providers and not just a static enum.
+
+    Cached for the process lifetime; hot-installing a provider needs
+    coordinated cache invalidation (tracked separately) to show up here without
+    a restart. Shared by the filters and the provider serializer.
+    """
+    return [(name, name) for name in SDKProvider.get_available_providers()]

api/src/backend/api/specs/v1.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: Prowler API
-  version: 1.30.0
+  version: 1.31.0
   description: |-
     Prowler API specification.
 

api/src/backend/api/tests/test_serializers.py

@@ -13,45 +13,72 @@ from api.v1.serializers import (
 
 
 class TestExternalProviderSecretValidation:
-    """A non-built-in provider's secret is validated against the credential
-    schema it declares through the SDK contract, or accepted as-is when it
-    declares none (then validated by the provider's test_connection)."""
+    """A non-built-in provider's secret is validated against the schema it
+    declares for the chosen secret type through the SDK contract, or accepted as
+    an object when it declares none (then validated by test_connection)."""
 
-    class _Credentials(BaseModel):
+    class _StaticCredentials(BaseModel):
         api_url: str
         api_key: str
 
-    def test_secret_validated_against_declared_schema(self):
+    class _RoleCredentials(BaseModel):
+        role_arn: str
+
+    def _patch(self, schemas):
         provider_class = MagicMock()
-        provider_class.get_credentials_schema.return_value = [self._Credentials]
-        with patch(
+        provider_class.get_credentials_schema.return_value = schemas
+        return patch(
             "api.v1.serializers.SDKProvider.get_class", return_value=provider_class
-        ):
+        )
+
+    def test_secret_validated_against_its_type_schema(self):
+        with self._patch({"static": self._StaticCredentials}):
             BaseWriteProviderSecretSerializer._validate_external_provider_secret(
-                "external-template", {"api_url": "u", "api_key": "k"}
+                "external-template", "static", {"api_url": "u", "api_key": "k"}
             )
 
     def test_secret_rejected_when_schema_violated(self):
-        provider_class = MagicMock()
-        provider_class.get_credentials_schema.return_value = [self._Credentials]
-        with patch(
-            "api.v1.serializers.SDKProvider.get_class", return_value=provider_class
-        ):
+        with self._patch({"static": self._StaticCredentials}):
             with pytest.raises(ValidationError):
                 BaseWriteProviderSecretSerializer._validate_external_provider_secret(
-                    "external-template", {"api_url": "u"}
+                    "external-template", "static", {"api_url": "u"}
+                )
+
+    def test_secret_must_match_its_type_not_another(self):
+        """A secret is validated against the schema for its declared secret_type,
+        not "any declared schema": a role-shaped secret under secret_type=static
+        is rejected. See PR #11402 review (josema-xyz / Alan-TheGentleman)."""
+        schemas = {"static": self._StaticCredentials, "role": self._RoleCredentials}
+        with self._patch(schemas):
+            with pytest.raises(ValidationError):
+                BaseWriteProviderSecretSerializer._validate_external_provider_secret(
+                    "external-template", "static", {"role_arn": "arn:aws:iam::x"}
+                )
+
+    def test_rejects_secret_type_not_declared_by_provider(self):
+        with self._patch({"static": self._StaticCredentials}):
+            with pytest.raises(ValidationError):
+                BaseWriteProviderSecretSerializer._validate_external_provider_secret(
+                    "external-template", "role", {"role_arn": "arn"}
                 )
 
     def test_secret_accepted_when_no_schema_declared(self):
-        provider_class = MagicMock()
-        provider_class.get_credentials_schema.return_value = []
-        with patch(
-            "api.v1.serializers.SDKProvider.get_class", return_value=provider_class
-        ):
+        with self._patch({}):
             BaseWriteProviderSecretSerializer._validate_external_provider_secret(
-                "external-template", {"anything": "goes"}
+                "external-template", "static", {"anything": "goes"}
             )
 
+    @pytest.mark.parametrize("bad_secret", [["a", "b"], "a-string", None, 42])
+    def test_secret_rejected_when_not_a_json_object(self, bad_secret):
+        """Even with no declared schema, a non-object secret must be rejected so
+        a list/string/null cannot be persisted and blow up later at
+        ``{**secret}``. See PR #11402 review (Alan-TheGentleman)."""
+        with self._patch({}):
+            with pytest.raises(ValidationError):
+                BaseWriteProviderSecretSerializer._validate_external_provider_secret(
+                    "external-template", "static", bad_secret
+                )
+
 
 class TestProviderEnumSerializerField:
     """The provider field accepts whatever the SDK exposes (built-in or

api/src/backend/api/v1/serializers.py

@@ -72,6 +72,7 @@ from api.v1.serializer_utils.lighthouse import (
     OpenAICredentialsSerializer,
 )
 from api.v1.serializer_utils.processors import ProcessorConfigField
+from api.provider_types import get_provider_type_choices
 from api.v1.serializer_utils.providers import ProviderSecretField
 from prowler.lib.mutelist.mutelist import Mutelist
 from prowler.providers.common.provider import Provider as SDKProvider
@@ -856,12 +857,9 @@ class ProviderGroupMembershipSerializer(RLSSerializer, BaseWriteSerializer):
 # Providers
 class ProviderEnumSerializerField(serializers.ChoiceField):
     def __init__(self, **kwargs):
-        # The SDK is the source of truth for which providers exist, so the
-        # accepted values track the installed providers (built-in or external)
-        # instead of a static enum.
-        kwargs["choices"] = [
-            (name, name) for name in SDKProvider.get_available_providers()
-        ]
+        # Accepted values track the SDK's installed providers (built-in or
+        # external), shared with the filters via one cached source.
+        kwargs["choices"] = get_provider_type_choices()
         super().__init__(**kwargs)
 
 
@@ -1546,34 +1544,45 @@ class FindingMetadataSerializer(BaseSerializerV1):
 # Provider secrets
 class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
     @staticmethod
-    def _validate_external_provider_secret(provider_type: str, secret: dict):
-        """Validate a non-built-in provider's secret against the credential
-        schemas it declares through the SDK contract (one model per secret type;
-        the secret must match one).
+    def _validate_external_provider_secret(
+        provider_type: str, secret_type: str, secret: dict
+    ):
+        """Validate a non-built-in provider's secret against the schema it
+        declares for the given secret type through the SDK contract.
 
-        Providers that declare no schema have their secret accepted as-is; the
-        credentials are then validated by the provider's ``test_connection``.
+        The provider maps each secret type to one model, so the chosen
+        secret_type stays bound to the shape it claims. Providers that declare
+        no schema have their secret accepted as an object and validated by the
+        provider's ``test_connection``.
         """
+        if not isinstance(secret, dict):
+            raise serializers.ValidationError({"secret": ["Must be a JSON object."]})
         schemas = SDKProvider.get_class(provider_type).get_credentials_schema()
         if not schemas:
             return
-        collected_errors = []
-        for schema in schemas:
-            try:
-                schema.model_validate(secret)
-                return
-            except PydanticValidationError as error:
-                collected_errors.append(error)
-        raise serializers.ValidationError(
-            {
-                "secret": [
-                    f"{'/'.join(str(loc) for loc in item['loc']) or 'secret'}: "
-                    f"{item['msg']}"
-                    for error in collected_errors
-                    for item in error.errors()
-                ]
-            }
-        )
+        schema = schemas.get(secret_type)
+        if schema is None:
+            raise serializers.ValidationError(
+                {
+                    "secret_type": [
+                        f"'{secret_type}' is not supported by provider "
+                        f"'{provider_type}'. Supported types: "
+                        f"{', '.join(sorted(schemas))}."
+                    ]
+                }
+            )
+        try:
+            schema.model_validate(secret)
+        except PydanticValidationError as error:
+            raise serializers.ValidationError(
+                {
+                    "secret": [
+                        f"{'/'.join(str(loc) for loc in item['loc']) or 'secret'}: "
+                        f"{item['msg']}"
+                        for item in error.errors()
+                    ]
+                }
+            )
 
     @staticmethod
     def validate_secret_based_on_provider(
@@ -1583,7 +1592,7 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
         # SDK contract; built-in providers keep their explicit serializers below.
         if not SDKProvider.is_builtin(provider_type):
             BaseWriteProviderSecretSerializer._validate_external_provider_secret(
-                provider_type, secret
+                provider_type, secret_type, secret
             )
             return
 

api/src/backend/tasks/jobs/backfill.py

@@ -40,35 +40,6 @@ from api.models import (
 logger = get_task_logger(__name__)
 
 
-def backfill_provider_str(tenant_id: str, batch_size: int = 1000):
-    """Populate the transitional `provider_str` shadow column for rows that
-    predate the sync trigger, copying `provider::text` in bounded batches.
-
-    Each batch runs in its own RLS transaction so the lock is held only for the
-    rows in that batch, keeping the operation safe on a large table. Idempotent:
-    only rows where `provider_str IS NULL` are touched, so an interrupted run
-    resumes cleanly and a completed column is a no-op on retry.
-    """
-    total_updated = 0
-    while True:
-        with rls_transaction(tenant_id) as cursor:
-            cursor.execute(
-                "UPDATE providers SET provider_str = provider::text "
-                "WHERE id IN ("
-                "    SELECT id FROM providers WHERE provider_str IS NULL LIMIT %s"
-                ")",
-                [batch_size],
-            )
-            updated = cursor.rowcount
-        total_updated += updated
-        if updated < batch_size:
-            break
-    logger.info(
-        "Backfilled provider_str for tenant %s: %d rows", tenant_id, total_updated
-    )
-    return {"tenant_id": tenant_id, "updated": total_updated}
-
-
 def backfill_resource_scan_summaries(tenant_id: str, scan_id: str):
     with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
         if ResourceScanSummary.objects.filter(

api/src/backend/tasks/tasks.py

@@ -19,7 +19,6 @@ from tasks.jobs.backfill import (
     backfill_daily_severity_summaries,
     backfill_finding_group_summaries,
     backfill_provider_compliance_scores,
-    backfill_provider_str,
     backfill_resource_scan_summaries,
     aggregate_scan_category_summaries,
     aggregate_scan_resource_group_summaries,
@@ -723,12 +722,6 @@ def backfill_finding_group_summaries_task(tenant_id: str, days: int = None):
     return backfill_finding_group_summaries(tenant_id=tenant_id, days=days)
 
 
-@shared_task(name="backfill-provider-str", queue="backfill")
-def backfill_provider_str_task(tenant_id: str):
-    """Backfill the transitional provider_str shadow column for a tenant."""
-    return backfill_provider_str(tenant_id=tenant_id)
-
-
 @shared_task(name="scan-category-summaries", queue="overview")
 @handle_provider_deletion
 def aggregate_scan_category_summaries_task(tenant_id: str, scan_id: str):