feat(scan): add mutelist and config file to scan (#5310)

Co-authored-by: Pepe Fagoaga <pepe@prowler.com>

prowler/__main__.py

@@ -236,9 +236,6 @@ def prowler():
     # Sort final check list
     checks_to_execute = sorted(checks_to_execute)
 
-    # Setup Mutelist
-    global_provider.mutelist = args.mutelist_file
-
     # Setup Output Options
     if provider == "aws":
         output_options = AWSOutputOptions(

prowler/providers/aws/aws_provider.py

@@ -13,7 +13,12 @@ from colorama import Fore, Style
 from pytz import utc
 from tzlocal import get_localzone
 
-from prowler.config.config import aws_services_json_file, get_default_mute_file_path
+from prowler.config.config import (
+    aws_services_json_file,
+    default_config_file_path,
+    get_default_mute_file_path,
+    load_and_validate_config_file,
+)
 from prowler.lib.check.utils import list_modules, recover_checks_from_service
 from prowler.lib.logger import logger
 from prowler.lib.utils.utils import open_file, parse_json_file, print_boxes
@@ -71,6 +76,7 @@ class AwsProvider(Provider):
     _audit_config: dict
     _scan_unused_services: bool = False
     _enabled_regions: set = set()
+    _mutelist: AWSMutelist
     # TODO: this is not optional, enforce for all providers
     audit_metadata: Audit_Metadata
 
@@ -88,8 +94,11 @@ class AwsProvider(Provider):
         scan_unused_services: bool = False,
         resource_tags: list[str] = [],
         resource_arn: list[str] = [],
-        audit_config: dict = {},
+        config_path: str = None,
+        config_content: dict = None,
         fixer_config: dict = {},
+        mutelist_path: str = None,
+        mutelist_content: dict = None,
         aws_access_key_id: str = None,
         aws_secret_access_key: str = None,
         aws_session_token: Optional[str] = None,
@@ -110,8 +119,11 @@ class AwsProvider(Provider):
             - scan_unused_services: A boolean indicating whether to scan unused services. False by default.
             - resource_tags: A list of tags to filter the resources to audit.
             - resource_arn: A list of ARNs of the resources to audit.
-            - audit_config: The audit configuration.
+            - config_path: The path to the configuration file.
+            - config_content: The content of the configuration file.
             - fixer_config: The fixer configuration.
+            - mutelist_path: The path to the mutelist file.
+            - mutelist_content: The content of the mutelist file.
             - aws_access_key_id: The AWS access key ID.
             - aws_secret_access_key: The AWS secret access key.
             - aws_session_token: The AWS session token, optional.
@@ -281,10 +293,32 @@ class AwsProvider(Provider):
         self._scan_unused_services = scan_unused_services
 
         # Audit Config
-        self._audit_config = audit_config
+        if config_content:
+            self._audit_config = config_content
+        else:
+            if not config_path:
+                config_path = default_config_file_path
+            self._audit_config = load_and_validate_config_file(self._type, config_path)
+
         # Fixer Config
         self._fixer_config = fixer_config
 
+        # Mutelist
+        if mutelist_content:
+            self._mutelist = AWSMutelist(
+                mutelist_content=mutelist_content,
+                session=self._session.current_session,
+                aws_account_id=self._identity.account,
+            )
+        else:
+            if not mutelist_path:
+                mutelist_path = get_default_mute_file_path(self.type)
+            self._mutelist = AWSMutelist(
+                mutelist_path=mutelist_path,
+                session=self._session.current_session,
+                aws_account_id=self._identity.account,
+            )
+
         Provider.set_global_provider(self)
 
     @property
@@ -326,23 +360,6 @@ class AwsProvider(Provider):
         """
         return self._mutelist
 
-    # TODO: this is going to be called from another place soon, since the provider
-    # shouldn't hold the mutelist
-    @mutelist.setter
-    def mutelist(self, mutelist_path):
-        """
-        mutelist.setter sets the provider's mutelist.
-        """
-        # Set default mutelist path if none is set
-        if not mutelist_path:
-            mutelist_path = get_default_mute_file_path(self.type)
-
-        self._mutelist = AWSMutelist(
-            mutelist_path=mutelist_path,
-            session=self._session.current_session,
-            aws_account_id=self._identity.account,
-        )
-
     @property
     def get_output_mapping(self):
         return {

prowler/providers/azure/azure_provider.py

@@ -17,7 +17,11 @@ from azure.mgmt.subscription import SubscriptionClient
 from colorama import Fore, Style
 from msgraph import GraphServiceClient
 
-from prowler.config.config import get_default_mute_file_path
+from prowler.config.config import (
+    default_config_file_path,
+    get_default_mute_file_path,
+    load_and_validate_config_file,
+)
 from prowler.lib.logger import logger
 from prowler.lib.utils.utils import print_boxes
 from prowler.providers.azure.exceptions.exceptions import (
@@ -109,8 +113,11 @@ class AzureProvider(Provider):
         tenant_id: str = None,
         region: str = "AzureCloud",
         subscription_ids: list = [],
-        audit_config: dict = {},
+        config_path: str = None,
+        config_content: dict = None,
         fixer_config: dict = {},
+        mutelist_path: str = None,
+        mutelist_content: dict = None,
         client_id: str = None,
         client_secret: str = None,
     ):
@@ -125,8 +132,11 @@ class AzureProvider(Provider):
             tenant_id (str): The Azure Active Directory tenant ID.
             region (str): The Azure region.
             subscription_ids (list): List of subscription IDs.
-            audit_config (dict): The audit configuration for the Azure provider.
+            config_path (str): The path to the configuration file.
+            config_content (dict): The configuration content.
             fixer_config (dict): The fixer configuration.
+            mutelist_path (str): The path to the mutelist file.
+            mutelist_content (dict): The mutelist content.
             client_id (str): The Azure client ID.
             client_secret (str): The Azure client secret.
 
@@ -192,10 +202,28 @@ class AzureProvider(Provider):
         self._locations = self.get_locations(self.session)
 
         # Audit Config
-        self._audit_config = audit_config
+        if config_content:
+            self._audit_config = config_content
+        else:
+            if not config_path:
+                config_path = default_config_file_path
+            self._audit_config = load_and_validate_config_file(self._type, config_path)
+
         # Fixer Config
         self._fixer_config = fixer_config
 
+        # Mutelist
+        if mutelist_content:
+            self._mutelist = AzureMutelist(
+                mutelist_content=mutelist_content,
+            )
+        else:
+            if not mutelist_path:
+                mutelist_path = get_default_mute_file_path(self.type)
+            self._mutelist = AzureMutelist(
+                mutelist_path=mutelist_path,
+            )
+
         Provider.set_global_provider(self)
 
     @property
@@ -238,17 +266,6 @@ class AzureProvider(Provider):
         """Mutelist object associated with this Azure provider."""
         return self._mutelist
 
-    @mutelist.setter
-    def mutelist(self, mutelist_path):
-        """
-        mutelist.setter sets the provider's mutelist.
-        """
-        # Set default mutelist path if none is set
-        if not mutelist_path:
-            mutelist_path = get_default_mute_file_path(self.type)
-
-        self._mutelist = AzureMutelist(mutelist_path)
-
     @property
     def get_output_mapping(self):
         """Dictionary that maps output keys to their corresponding values."""

prowler/providers/common/provider.py

@@ -163,9 +163,7 @@ class Provider(ABC):
             provider_class = getattr(
                 import_module(provider_class_path), provider_class_name
             )
-            audit_config = load_and_validate_config_file(
-                arguments.provider, arguments.config_file
-            )
+
             fixer_config = load_and_validate_config_file(
                 arguments.provider, arguments.fixer_config
             )
@@ -185,8 +183,9 @@ class Provider(ABC):
                         arguments.scan_unused_services,
                         arguments.resource_tag,
                         arguments.resource_arn,
-                        audit_config,
-                        fixer_config,
+                        arguments.config_file,
+                        arguments.mutelist_file,
+                        fixer_config=fixer_config,
                     )
                 elif "azure" in provider_class_name.lower():
                     provider_class(
@@ -197,8 +196,9 @@ class Provider(ABC):
                         arguments.tenant_id,
                         arguments.azure_region,
                         arguments.subscription_id,
-                        audit_config,
-                        fixer_config,
+                        arguments.config_file,
+                        arguments.mutelist_file,
+                        fixer_config=fixer_config,
                     )
                 elif "gcp" in provider_class_name.lower():
                     provider_class(
@@ -207,16 +207,18 @@ class Provider(ABC):
                         arguments.credentials_file,
                         arguments.impersonate_service_account,
                         arguments.list_project_id,
-                        audit_config,
-                        fixer_config,
+                        arguments.config_file,
+                        arguments.mutelist_file,
+                        fixer_config=fixer_config,
                     )
                 elif "kubernetes" in provider_class_name.lower():
                     provider_class(
                         arguments.kubeconfig_file,
                         arguments.context,
                         arguments.namespace,
-                        audit_config,
-                        fixer_config,
+                        arguments.config_file,
+                        arguments.mutelist_file,
+                        fixer_config=fixer_config,
                     )
 
         except TypeError as error:

prowler/providers/gcp/gcp_provider.py

@@ -10,7 +10,11 @@ from google.oauth2.credentials import Credentials
 from googleapiclient import discovery
 from googleapiclient.errors import HttpError
 
-from prowler.config.config import get_default_mute_file_path
+from prowler.config.config import (
+    default_config_file_path,
+    get_default_mute_file_path,
+    load_and_validate_config_file,
+)
 from prowler.lib.logger import logger
 from prowler.lib.utils.utils import print_boxes
 from prowler.providers.common.models import Audit_Metadata, Connection
@@ -48,8 +52,11 @@ class GcpProvider(Provider):
         credentials_file: str = None,
         impersonate_service_account: str = None,
         list_project_ids: bool = False,
-        audit_config: dict = {},
+        config_path: str = None,
+        config_content: dict = None,
         fixer_config: dict = {},
+        mutelist_path: str = None,
+        mutelist_content: dict = None,
         client_id: str = None,
         client_secret: str = None,
         refresh_token: str = None,
@@ -63,8 +70,11 @@ class GcpProvider(Provider):
             credentials_file: str
             impersonate_service_account: str
             list_project_ids: bool
-            audit_config: dict
+            config_path: str
+            config_content: dict
             fixer_config: dict
+            mutelist_path: str
+            mutelist_content: dict
             client_id: str
             client_secret: str
             refresh_token: str
@@ -143,9 +153,28 @@ class GcpProvider(Provider):
 
         # TODO: move this to the providers, pending for AWS, GCP, AZURE and K8s
         # Audit Config
-        self._audit_config = audit_config
+        if config_content:
+            self._audit_config = config_content
+        else:
+            if not config_path:
+                config_path = default_config_file_path
+            self._audit_config = load_and_validate_config_file(self._type, config_path)
+
+        # Fixer Config
         self._fixer_config = fixer_config
 
+        # Mutelist
+        if mutelist_content:
+            self._mutelist = GCPMutelist(
+                mutelist_content=mutelist_content,
+            )
+        else:
+            if not mutelist_path:
+                mutelist_path = get_default_mute_file_path(self.type)
+            self._mutelist = GCPMutelist(
+                mutelist_path=mutelist_path,
+            )
+
         Provider.set_global_provider(self)
 
     @property
@@ -195,17 +224,6 @@ class GcpProvider(Provider):
         """
         return self._mutelist
 
-    @mutelist.setter
-    def mutelist(self, mutelist_path):
-        """
-        mutelist.setter sets the provider's mutelist.
-        """
-        # Set default mutelist path if none is set
-        if not mutelist_path:
-            mutelist_path = get_default_mute_file_path(self.type)
-
-        self._mutelist = GCPMutelist(mutelist_path)
-
     @property
     def get_output_mapping(self):
         return {

prowler/providers/kubernetes/kubernetes_provider.py

@@ -7,7 +7,11 @@ from requests.exceptions import Timeout
 from yaml import parser, safe_load
 
 from kubernetes import client, config
-from prowler.config.config import get_default_mute_file_path
+from prowler.config.config import (
+    default_config_file_path,
+    get_default_mute_file_path,
+    load_and_validate_config_file,
+)
 from prowler.lib.logger import logger
 from prowler.lib.utils.utils import print_boxes
 from prowler.providers.common.models import Audit_Metadata, Connection
@@ -43,8 +47,11 @@ class KubernetesProvider(Provider):
         kubeconfig_file: str = None,
         context: str = None,
         namespace: list = None,
-        audit_config: dict = {},
+        config_path: str = None,
+        config_content: dict = {},
         fixer_config: dict = {},
+        mutelist_path: str = None,
+        mutelist_content: dict = {},
         kubeconfig_content: dict = None,
     ):
         """
@@ -54,8 +61,11 @@ class KubernetesProvider(Provider):
             kubeconfig_content (dict): Content of the kubeconfig file.
             context (str): Context name.
             namespace (list): List of namespaces.
-            audit_config (dict): Audit configuration.
+            config_content (dict): Audit configuration.
+            config_path (str): Path to the configuration file.
             fixer_config (dict): Fixer configuration.
+            mutelist_path (str): Path to the mutelist file.
+            mutelist_content (dict): Mutelist content.
         """
 
         logger.info("Instantiating Kubernetes Provider ...")
@@ -79,10 +89,28 @@ class KubernetesProvider(Provider):
         )
 
         # Audit Config
-        self._audit_config = audit_config
+        if config_content:
+            self._audit_config = config_content
+        else:
+            if not config_path:
+                config_path = default_config_file_path
+            self._audit_config = load_and_validate_config_file(self._type, config_path)
+
         # Fixer Config
         self._fixer_config = fixer_config
 
+        # Mutelist
+        if mutelist_content:
+            self._mutelist = KubernetesMutelist(
+                mutelist_content=mutelist_content,
+            )
+        else:
+            if not mutelist_path:
+                mutelist_path = get_default_mute_file_path(self.type)
+            self._mutelist = KubernetesMutelist(
+                mutelist_path=mutelist_path,
+            )
+
         Provider.set_global_provider(self)
 
     @property
@@ -116,17 +144,6 @@ class KubernetesProvider(Provider):
         """
         return self._mutelist
 
-    @mutelist.setter
-    def mutelist(self, mutelist_path):
-        """
-        mutelist.setter sets the provider's mutelist.
-        """
-        # Set default mutelist path if none is set
-        if not mutelist_path:
-            mutelist_path = get_default_mute_file_path(self.type)
-
-        self._mutelist = KubernetesMutelist(mutelist_path)
-
     @property
     def get_output_mapping(self):
         return {

tests/providers/aws/aws_provider_test.py

@@ -15,7 +15,6 @@ from moto import mock_aws
 from pytest import raises
 from tzlocal import get_localzone
 
-from prowler.config.config import load_and_validate_config_file
 from prowler.providers.aws.aws_provider import (
     AwsProvider,
     get_aws_available_regions,
@@ -258,7 +257,7 @@ class TestAWSProvider:
 
         assert aws_provider.type == "aws"
         assert aws_provider.scan_unused_services is True
-        assert aws_provider.audit_config == {}
+        assert aws_provider.audit_config
         assert aws_provider.session.current_session.region_name == AWS_REGION_US_EAST_1
 
     @mock_aws
@@ -452,7 +451,7 @@ class TestAWSProvider:
 
             assert aws_provider.type == "aws"
             assert aws_provider.scan_unused_services is False
-            assert aws_provider.audit_config == {}
+            assert aws_provider.audit_config != {}
             assert (
                 aws_provider.session.current_session.region_name == AWS_REGION_US_EAST_1
             )
@@ -651,9 +650,8 @@ aws:
         config_file_input.write(bytes(config, encoding="raw_unicode_escape"))
         config_file_input.close()
         config_file_input = config_file_input.name
-        audit_config = load_and_validate_config_file("aws", config_file_input)
         aws_provider = AwsProvider(
-            audit_config=audit_config,
+            config_path=config_file_input,
         )
 
         os.remove(config_file_input)
@@ -688,9 +686,7 @@ aws:
         with open(mutelist_file.name, "w") as mutelist_file:
             mutelist_file.write(json.dumps(mutelist, indent=4))
 
-        aws_provider = AwsProvider()
-
-        aws_provider.mutelist = mutelist_file.name
+        aws_provider = AwsProvider(mutelist_path=mutelist_file.name)
 
         os.remove(mutelist_file.name)
 
@@ -700,13 +696,12 @@ aws:
 
     @mock_aws
     def test_aws_provider_mutelist_none(self):
-        aws_provider = AwsProvider()
 
         with patch(
             "prowler.providers.aws.aws_provider.get_default_mute_file_path",
             return_value=None,
         ):
-            aws_provider.mutelist = None
+            aws_provider = AwsProvider(mutelist_path=None)
 
         assert isinstance(aws_provider.mutelist, AWSMutelist)
         assert aws_provider.mutelist.mutelist == {}
@@ -754,9 +749,8 @@ aws:
             )
         )
 
-        aws_provider = AwsProvider()
+        aws_provider = AwsProvider(mutelist_path=mutelist_bucket_object_uri)
 
-        aws_provider.mutelist = mutelist_bucket_object_uri
         os.remove(mutelist_file.name)
 
         assert isinstance(aws_provider.mutelist, AWSMutelist)
@@ -794,7 +788,7 @@ aws:
             "prowler.providers.aws.lib.mutelist.mutelist.AWSMutelist.get_mutelist_file_from_lambda",
             return_value=mutelist["Mutelist"],
         ):
-            aws_provider.mutelist = lambda_mutelist_path
+            aws_provider = AwsProvider(mutelist_path=lambda_mutelist_path)
 
         assert isinstance(aws_provider.mutelist, AWSMutelist)
         assert aws_provider.mutelist.mutelist == mutelist["Mutelist"]
@@ -831,7 +825,7 @@ aws:
             "prowler.providers.aws.lib.mutelist.mutelist.AWSMutelist.get_mutelist_file_from_dynamodb",
             return_value=mutelist["Mutelist"],
         ):
-            aws_provider.mutelist = dynamodb_mutelist_path
+            aws_provider = AwsProvider(mutelist_path=dynamodb_mutelist_path)
 
         assert isinstance(aws_provider.mutelist, AWSMutelist)
         assert aws_provider.mutelist.mutelist == mutelist["Mutelist"]

tests/providers/aws/utils.py

@@ -8,6 +8,7 @@ from moto import mock_aws
 from prowler.config.config import (
     default_config_file_path,
     default_fixer_config_file_path,
+    get_default_mute_file_path,
 )
 from prowler.providers.aws.aws_provider import AwsProvider
 from prowler.providers.common.models import Audit_Metadata
@@ -99,6 +100,7 @@ def set_mocked_aws_provider(
     profile_region: str = None,
     audit_config: dict = {},
     fixer_config: dict = {},
+    mutelist: dict = None,
     scan_unused_services: bool = True,
     audit_session: session.Session = session.Session(
         profile_name=None,
@@ -144,6 +146,7 @@ def set_mocked_aws_provider(
     provider._audit_resources = []
     provider._audit_config = audit_config
     provider._fixer_config = fixer_config
+    provider._mutelist = mutelist
     provider.audit_metadata = Audit_Metadata(
         services_scanned=0,
         expected_checks=expected_checks,
@@ -168,6 +171,7 @@ def set_default_provider_arguments(
     arguments.send_sh_only_fails = False
     arguments.config_file = default_config_file_path
     arguments.fixer_config = default_fixer_config_file_path
+    arguments.mutelist_file = get_default_mute_file_path("aws")
 
     return arguments
 

tests/providers/azure/azure_provider_test.py

@@ -35,7 +35,6 @@ class TestAzureProvider:
         client_id = None
         client_secret = None
 
-        audit_config = load_and_validate_config_file("azure", default_config_file_path)
         fixer_config = load_and_validate_config_file(
             "azure", default_fixer_config_file_path
         )
@@ -56,7 +55,7 @@ class TestAzureProvider:
                 tenant_id,
                 azure_region,
                 subscription_id,
-                audit_config=audit_config,
+                config_path=default_config_file_path,
                 fixer_config=fixer_config,
                 client_id=client_id,
                 client_secret=client_secret,

tests/providers/gcp/gcp_provider_test.py

@@ -27,7 +27,6 @@ class TestGCPProvider:
         list_project_id = False
         credentials_file = ""
         impersonate_service_account = ""
-        audit_config = load_and_validate_config_file("gcp", default_config_file_path)
         fixer_config = load_and_validate_config_file(
             "gcp", default_fixer_config_file_path
         )
@@ -70,7 +69,7 @@ class TestGCPProvider:
                 credentials_file,
                 impersonate_service_account,
                 list_project_id,
-                audit_config=audit_config,
+                config_path=default_config_file_path,
                 fixer_config=fixer_config,
                 client_id=client_id,
                 client_secret=client_secret,

tests/providers/kubernetes/kubernetes_provider_test.py

@@ -52,9 +52,6 @@ class TestKubernetesProvider:
             arguments.context = None
             arguments.only_logs = False
             arguments.namespace = None
-            audit_config = load_and_validate_config_file(
-                "kubernetes", default_config_file_path
-            )
             fixer_config = load_and_validate_config_file(
                 "kubernetes", default_fixer_config_file_path
             )
@@ -64,7 +61,7 @@ class TestKubernetesProvider:
                 arguments.kubeconfig_file,
                 arguments.context,
                 arguments.namespace,
-                audit_config=audit_config,
+                config_path=default_config_file_path,
                 fixer_config=fixer_config,
             )
             assert isinstance(kubernetes_provider.session, KubernetesSession)