feat(sdk): add sdk_only provider property to hide providers from the app (#11578)

prowler/CHANGELOG.md

@@ -13,6 +13,12 @@ All notable changes to the **Prowler SDK** are documented in this file.
 - CIS Controls v8.1 universal compliance framework mapping existing checks across 18 providers (AWS, Azure, GCP, Kubernetes, M365, GitHub, AlibabaCloud, OracleCloud, GoogleWorkspace, Okta, Cloudflare, Vercel, MongoDB Atlas, OpenStack, Linode, StackIT, NHN, and Scaleway) to the 18 CIS Critical Security Controls and their Safeguards [(#11700)](https://github.com/prowler-cloud/prowler/pull/11700)
 - CIS Microsoft 365 Foundations Benchmark v7.0.0 compliance framework for the M365 provider [(#11699)](https://github.com/prowler-cloud/prowler/pull/11699)
 - `waf_regional_webacl_logging_enabled` check for AWS provider, verifying that each AWS WAF Classic Regional Web ACL has logging enabled to a Kinesis Data Firehose stream [(#11539)](https://github.com/prowler-cloud/prowler/pull/11539)
+- `sdk_only` provider property (default `true`) and `Provider.get_app_providers()`, so a provider (built-in or external) stays CLI/SDK-only and hidden from the app unless it declares `sdk_only = False` [(#11427)](https://github.com/prowler-cloud/prowler/pull/11427)
+- `Provider.get_scan_arguments()`, `Provider.get_connection_arguments()` and `Provider.get_credentials_schema()` contract methods, so a provider persisted as a stored uid plus a secret dict can be constructed and validated programmatically (to be consumed by the API in a later change) [(#11578)](https://github.com/prowler-cloud/prowler/pull/11578)
+
+### 🐞 Fixed
+
+- Compliance frameworks contributed by several external packages under the same provider are now merged instead of overwritten, so every entry-point directory a provider contributes is discovered [(#11578)](https://github.com/prowler-cloud/prowler/pull/11578)
 
 ### 🐞 Fixed
 

prowler/config/config.py

@@ -88,15 +88,22 @@ actual_directory = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
 
 
 def _get_ep_compliance_dirs() -> dict:
-    """Discover compliance directories from entry points. Returns {provider: path}."""
+    """Discover compliance directories from entry points. Returns {provider: [paths]}.
+
+    A provider may be contributed by several packages, so accumulate every
+    directory instead of overwriting.
+    """
     dirs = {}
     for ep in importlib.metadata.entry_points(group="prowler.compliance"):
         try:
             module = ep.load()
             if hasattr(module, "__path__"):
-                dirs[ep.name] = module.__path__[0]
+                path = module.__path__[0]
             elif hasattr(module, "__file__"):
-                dirs[ep.name] = os.path.dirname(module.__file__)
+                path = os.path.dirname(module.__file__)
+            else:
+                continue
+            dirs.setdefault(ep.name, []).append(path)
         except Exception as error:
             logger.warning(
                 f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
@@ -145,12 +152,15 @@ def get_available_compliance_frameworks(provider=None):
                             continue
                     if name not in available_compliance_frameworks:
                         available_compliance_frameworks.append(name)
-    # External per-provider compliance via entry points.
+    # External compliance via entry points; a provider may be served by
+    # several packages, so iterate every directory it contributes.
     ep_dirs = _get_ep_compliance_dirs()
-    for prov, path in ep_dirs.items():
+    for prov, paths in ep_dirs.items():
         if provider and prov != provider:
             continue
-        if os.path.isdir(path):
+        for path in paths:
+            if not os.path.isdir(path):
+                continue
             for file in os.scandir(path):
                 if file.is_file() and file.name.endswith(".json"):
                     name = file.name.removesuffix(".json")

prowler/providers/alibabacloud/alibabacloud_provider.py

@@ -53,6 +53,7 @@ class AlibabacloudProvider(Provider):
     """
 
     _type: str = "alibabacloud"
+    sdk_only: bool = False
     _identity: AlibabaCloudIdentityInfo
     _session: AlibabaCloudSession
     _audit_resources: list = []

prowler/providers/aws/aws_provider.py

@@ -90,6 +90,7 @@ class AwsProvider(Provider):
     """
 
     _type: str = "aws"
+    sdk_only: bool = False
     _identity: AWSIdentityInfo
     _session: AWSSession
     _organizations_metadata: AWSOrganizationsInfo

prowler/providers/azure/azure_provider.py

@@ -97,6 +97,7 @@ class AzureProvider(Provider):
     """
 
     _type: str = "azure"
+    sdk_only: bool = False
     _session: DefaultAzureCredential
     _identity: AzureIdentityInfo
     _audit_config: dict

prowler/providers/cloudflare/cloudflare_provider.py

@@ -46,6 +46,7 @@ class CloudflareProvider(Provider):
     """Cloudflare provider."""
 
     _type: str = "cloudflare"
+    sdk_only: bool = False
     _session: CloudflareSession
     _identity: CloudflareIdentityInfo
     _audit_config: dict

prowler/providers/common/provider.py

@@ -142,6 +142,10 @@ class Provider(ABC):
 
     _cli_help_text: str = ""
 
+    # CLI/SDK-only provider, hidden from the app (API/UI). Defaults True; a
+    # provider opts into the app with ``sdk_only = False``. See get_app_providers().
+    sdk_only: bool = True
+
     @classmethod
     def from_cli_args(cls, arguments: Namespace, fixer_config: dict) -> "Provider":
         """Instantiate the provider from CLI arguments and return the instance.
@@ -209,6 +213,65 @@ class Provider(ABC):
             f"{self.__class__.__name__} has not implemented get_mutelist_finding_args()"
         )
 
+    @classmethod
+    def get_scan_arguments(
+        cls,
+        provider_uid: str,
+        secret: dict,
+        mutelist_content: Optional[dict] = None,
+    ) -> dict:
+        """Build the provider constructor kwargs from a stored uid and secret.
+
+        This is the programmatic construction interface intended for callers
+        that will persist a provider as a single ``uid`` plus a ``secret`` dict
+        (e.g. the API), as opposed to the CLI which passes explicit per-provider
+        flags.
+
+        The base implementation is a default: it passes the secret through, adds
+        the mutelist, and intentionally drops ``provider_uid``. The API consumes
+        this contract for external providers, so an external provider whose uid
+        is part of the scan scope (e.g. a subscription or project id) or that
+        renames/filters secret keys overrides this to inject the uid into the
+        right kwarg; until it does, the base default is not the final shape for
+        that provider. Built-in providers whose scope derives from the uid are
+        mapped on the API side and do not go through this method.
+        """
+        kwargs = {**secret}
+        if mutelist_content is not None:
+            kwargs["mutelist_content"] = mutelist_content
+        return kwargs
+
+    @classmethod
+    def get_connection_arguments(cls, provider_uid: str, secret: dict) -> dict:
+        """Build the ``test_connection`` kwargs from a stored uid and secret.
+
+        Companion to :meth:`get_scan_arguments` for the connection check, which
+        often needs a different shape than the constructor. The base passes the
+        secret through and intentionally drops ``provider_uid``. An external
+        provider whose uid is part of the scope overrides this to add its
+        identity kwarg (and ``provider_id`` where its ``test_connection``
+        expects it); built-in providers are mapped on the API side and do not go
+        through this method.
+        """
+        return {**secret}
+
+    @classmethod
+    def get_credentials_schema(cls) -> dict:
+        """Return the provider's credential schemas keyed by secret type.
+
+        Maps each secret type the provider accepts (``"static"``, ``"role"`` or
+        ``"service_account"``) to the pydantic model that validates a secret of
+        that type. The provider declares which type each schema belongs to, so
+        the API validates a secret against the model for the secret type it is
+        created with and the chosen type stays bound to the shape it claims.
+
+        Each model documents each field via ``Field(description=...)`` and
+        whether it is required (no default) or optional. An empty dict means no
+        schema is declared: the secret is accepted as an object and validated by
+        :meth:`test_connection`.
+        """
+        return {}
+
     def display_compliance_table(
         self,
         _findings: list,
@@ -637,6 +700,28 @@ class Provider(ABC):
             providers.add(ep.name)
         return sorted(providers)
 
+    @staticmethod
+    def get_app_providers() -> list[str]:
+        """Return the providers the app (API/UI) may expose: those with
+        ``sdk_only = False``.
+
+        Counterpart of :meth:`get_available_providers`, which lists every
+        provider for the CLI. A provider whose class cannot be imported is
+        treated as ``sdk_only`` (excluded) so a broken plug-in never leaks in.
+        """
+        app_providers = []
+        for name in Provider.get_available_providers():
+            try:
+                provider_class = Provider.get_class(name)
+            except Exception as error:
+                logger.warning(
+                    f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+                )
+                continue
+            if not getattr(provider_class, "sdk_only", True):
+                app_providers.append(name)
+        return app_providers
+
     @staticmethod
     def is_tool_wrapper_provider(provider: str) -> bool:
         """Return True if the provider delegates scanning to an external tool.

prowler/providers/gcp/gcp_provider.py

@@ -61,6 +61,7 @@ class GcpProvider(Provider):
     """
 
     _type: str = "gcp"
+    sdk_only: bool = False
     _session: Credentials
     _project_ids: list
     _excluded_project_ids: list

prowler/providers/github/github_provider.py

@@ -91,6 +91,7 @@ class GithubProvider(Provider):
     """
 
     _type: str = "github"
+    sdk_only: bool = False
     _auth_method: str = None
     MAX_REPO_LIST_LINES: int = 10_000
     MAX_REPO_NAME_LENGTH: int = 500

prowler/providers/googleworkspace/googleworkspace_provider.py

@@ -54,6 +54,7 @@ class GoogleworkspaceProvider(Provider):
     """
 
     _type: str = "googleworkspace"
+    sdk_only: bool = False
     _session: GoogleWorkspaceSession
     _identity: GoogleWorkspaceIdentityInfo
     _domain_resource: GoogleWorkspaceResource

prowler/providers/iac/iac_provider.py

@@ -28,6 +28,7 @@ from prowler.providers.common.provider import Provider
 
 class IacProvider(Provider):
     _type: str = "iac"
+    sdk_only: bool = False
     audit_metadata: Audit_Metadata
 
     def __init__(

prowler/providers/image/image_provider.py

@@ -59,6 +59,7 @@ class ImageProvider(Provider):
     """
 
     _type: str = "image"
+    sdk_only: bool = False
     FINDING_BATCH_SIZE: int = 100
     MAX_IMAGE_LIST_LINES: int = 10_000
     MAX_IMAGE_NAME_LENGTH: int = 500

prowler/providers/kubernetes/kubernetes_provider.py

@@ -58,6 +58,7 @@ class KubernetesProvider(Provider):
     """
 
     _type: str = "kubernetes"
+    sdk_only: bool = False
     _session: KubernetesSession
     _namespaces: list
     _audit_config: dict

prowler/providers/m365/m365_provider.py

@@ -99,6 +99,7 @@ class M365Provider(Provider):
     """
 
     _type: str = "m365"
+    sdk_only: bool = False
     _session: DefaultAzureCredential  # Must be used besides being named for Azure
     _identity: M365IdentityInfo
     _audit_config: dict

prowler/providers/mongodbatlas/mongodbatlas_provider.py

@@ -36,6 +36,7 @@ class MongodbatlasProvider(Provider):
     """
 
     _type: str = "mongodbatlas"
+    sdk_only: bool = False
     _session: MongoDBAtlasSession
     _identity: MongoDBAtlasIdentityInfo
     _audit_config: dict

prowler/providers/okta/okta_provider.py

@@ -78,6 +78,7 @@ class OktaProvider(Provider):
     """
 
     _type: str = "okta"
+    sdk_only: bool = False
     _auth_method: str = None
     _session: OktaSession
     _identity: OktaIdentityInfo

prowler/providers/openstack/openstack_provider.py

@@ -36,6 +36,7 @@ class OpenstackProvider(Provider):
     """OpenStack provider responsible for bootstrapping the SDK session."""
 
     _type: str = "openstack"
+    sdk_only: bool = False
     _session: OpenStackSession
     _identity: OpenStackIdentityInfo
     _audit_config: dict

prowler/providers/oraclecloud/oraclecloud_provider.py

@@ -59,6 +59,7 @@ class OraclecloudProvider(Provider):
     """
 
     _type: str = "oraclecloud"
+    sdk_only: bool = False
     _identity: OCIIdentityInfo
     _session: OCISession
     _audit_config: dict

prowler/providers/vercel/vercel_provider.py

@@ -33,6 +33,7 @@ class VercelProvider(Provider):
     """Vercel provider."""
 
     _type: str = "vercel"
+    sdk_only: bool = False
     _session: VercelSession
     _identity: VercelIdentityInfo
     _audit_config: dict

tests/config/config_test.py

@@ -488,7 +488,7 @@ class Test_Config:
             with open(json_path, "w") as f:
                 json.dump({"Framework": "CIS", "Provider": "aws"}, f)
 
-            mock_dirs.return_value = {"aws": tmpdir}
+            mock_dirs.return_value = {"aws": [tmpdir]}
 
             frameworks = get_available_compliance_frameworks("aws")
 
@@ -497,6 +497,32 @@ class Test_Config:
                 f"{frameworks.count('cis_2.0_aws')} occurrences in: {frameworks}"
             )
 
+    @mock.patch("prowler.config.config._get_ep_compliance_dirs")
+    def test_get_available_compliance_frameworks_merges_multiple_ep_dirs_same_provider(
+        self, mock_dirs
+    ):
+        """Frameworks from every package contributing the same provider must
+        surface, not just the last directory discovered."""
+        import json
+        import tempfile
+
+        with (
+            tempfile.TemporaryDirectory() as pkg_a,
+            tempfile.TemporaryDirectory() as pkg_b,
+        ):
+            with open(os.path.join(pkg_a, "cis_1.0_template.json"), "w") as f:
+                json.dump({"Framework": "CIS", "Provider": "template"}, f)
+            with open(os.path.join(pkg_b, "nis2_1.0_template.json"), "w") as f:
+                json.dump({"Framework": "NIS2", "Provider": "template"}, f)
+
+            # Two packages register `prowler.compliance` with the same name.
+            mock_dirs.return_value = {"template": [pkg_a, pkg_b]}
+
+            frameworks = get_available_compliance_frameworks("template")
+
+            assert "cis_1.0_template" in frameworks
+            assert "nis2_1.0_template" in frameworks
+
     def test_load_and_validate_config_file_aws(self):
         path = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
         config_test_file = f"{path}/fixtures/config.yaml"

tests/providers/external/test_dynamic_provider_loading.py

@@ -344,6 +344,84 @@ class TestProviderDiscovery:
         assert help_text["nohelptext"] == ""
 
 
+class TestSdkOnly:
+    """The ``sdk_only`` flag (default True) and Provider.get_app_providers."""
+
+    def test_base_contract_defaults_to_sdk_only(self):
+        # The default must be True so nothing leaks into the app implicitly.
+        assert Provider.sdk_only is True
+
+    def test_external_provider_without_flag_is_sdk_only(self):
+        # FakeExternalProvider does not override the flag -> inherits True.
+        assert FakeExternalProvider.sdk_only is True
+
+    @patch("prowler.providers.common.provider.Provider.get_class")
+    @patch("prowler.providers.common.provider.Provider.get_available_providers")
+    def test_get_app_providers_filters_out_sdk_only(
+        self, mock_available, mock_get_class
+    ):
+        app_cls = type("AppProvider", (Provider,), {"sdk_only": False})
+        sdk_cls = type("SdkProvider", (Provider,), {"sdk_only": True})
+        mock_available.return_value = ["appone", "sdkone", "apptwo"]
+        mock_get_class.side_effect = lambda name: {
+            "appone": app_cls,
+            "sdkone": sdk_cls,
+            "apptwo": app_cls,
+        }[name]
+
+        app_providers = Provider.get_app_providers()
+
+        assert app_providers == ["appone", "apptwo"]
+
+    @patch("prowler.providers.common.provider.Provider.get_class")
+    @patch("prowler.providers.common.provider.Provider.get_available_providers")
+    def test_get_app_providers_excludes_provider_that_fails_to_load(
+        self, mock_available, mock_get_class
+    ):
+        # A provider whose class cannot be imported is treated as sdk_only
+        # (excluded) so a broken plug-in never leaks into the app.
+        app_cls = type("AppProvider", (Provider,), {"sdk_only": False})
+        mock_available.return_value = ["appone", "broken"]
+
+        def _get_class(name):
+            if name == "broken":
+                raise ImportError("missing transitive dep")
+            return app_cls
+
+        mock_get_class.side_effect = _get_class
+
+        assert Provider.get_app_providers() == ["appone"]
+
+    def test_app_exposed_builtins_declare_sdk_only_false(self):
+        # The providers implemented end-to-end in the API/UI must opt in.
+        app_providers = set(Provider.get_app_providers())
+        for name in (
+            "aws",
+            "azure",
+            "gcp",
+            "kubernetes",
+            "m365",
+            "github",
+            "mongodbatlas",
+            "iac",
+            "oraclecloud",
+            "alibabacloud",
+            "cloudflare",
+            "openstack",
+            "image",
+            "googleworkspace",
+            "vercel",
+            "okta",
+        ):
+            assert name in app_providers, f"{name} should be exposed to the app"
+
+    def test_sdk_only_builtins_are_hidden_from_app(self):
+        # Built-ins not implemented in the API stay SDK-only via the default.
+        app_providers = set(Provider.get_app_providers())
+        for name in ("llm", "nhn", "scaleway", "stackit"):
+            assert name not in app_providers, f"{name} must be hidden from the app"
+
+
 class TestIsToolWrapperProvider:
     """Tests for Provider.is_tool_wrapper_provider — the helper that combines the
     built-in EXTERNAL_TOOL_PROVIDERS frozenset with the is_external_tool_provider
@@ -1456,7 +1534,7 @@ class TestCompliance:
 
         dirs = _get_ep_compliance_dirs()
 
-        assert dirs["fakeexternal"] == "/path/to/compliance"
+        assert dirs["fakeexternal"] == ["/path/to/compliance"]
 
     @patch("prowler.config.config.importlib.metadata.entry_points")
     def test_get_ep_compliance_dirs_file_fallback(self, mock_ep):
@@ -1472,7 +1550,7 @@ class TestCompliance:
 
         dirs = _get_ep_compliance_dirs()
 
-        assert dirs["ext"] == "/path/to/compliance"
+        assert dirs["ext"] == ["/path/to/compliance"]
 
     @patch("prowler.config.config.importlib.metadata.entry_points")
     def test_get_ep_compliance_dirs_handles_load_exception(self, mock_ep):
@@ -1502,7 +1580,7 @@ class TestCompliance:
             with open(json_path, "w") as f:
                 json.dump({"Framework": "Custom", "Provider": "ext"}, f)
 
-            mock_dirs.return_value = {"ext": tmpdir}
+            mock_dirs.return_value = {"ext": [tmpdir]}
 
             frameworks = get_available_compliance_frameworks("ext")
 
@@ -2168,6 +2246,37 @@ class TestDispatchFallbacks:
 class TestBaseContractDefaults:
     """Tests for Provider base class default implementations."""
 
+    def test_get_scan_arguments_passes_secret_through(self):
+        """Base get_scan_arguments returns the secret unchanged when no mutelist."""
+        kwargs = FakeProviderNoHelpText.get_scan_arguments("uid", {"token": "x"})
+
+        assert kwargs == {"token": "x"}
+
+    def test_get_scan_arguments_adds_mutelist_content(self):
+        """Base get_scan_arguments adds mutelist_content when provided."""
+        kwargs = FakeProviderNoHelpText.get_scan_arguments(
+            "uid", {"token": "x"}, {"Mutelist": {}}
+        )
+
+        assert kwargs == {"token": "x", "mutelist_content": {"Mutelist": {}}}
+
+    def test_get_scan_arguments_preserves_empty_mutelist_content(self):
+        """Base get_scan_arguments passes an explicit empty mutelist through so it
+        is not mistaken for an absent mutelist that triggers provider defaults."""
+        kwargs = FakeProviderNoHelpText.get_scan_arguments("uid", {"token": "x"}, {})
+
+        assert kwargs == {"token": "x", "mutelist_content": {}}
+
+    def test_get_connection_arguments_passes_secret_through(self):
+        """Base get_connection_arguments returns the secret unchanged."""
+        kwargs = FakeProviderNoHelpText.get_connection_arguments("uid", {"token": "x"})
+
+        assert kwargs == {"token": "x"}
+
+    def test_get_credentials_schema_defaults_to_empty(self):
+        """Base get_credentials_schema declares no schema by default."""
+        assert FakeProviderNoHelpText.get_credentials_schema() == {}
+
     def test_from_cli_args_raises_not_implemented(self):
         """Base Provider.from_cli_args raises NotImplementedError."""
         with pytest.raises(NotImplementedError):