diff --git a/prowler/config/config.yaml b/prowler/config/config.yaml index cceb549948..e319f4a029 100644 --- a/prowler/config/config.yaml +++ b/prowler/config/config.yaml @@ -378,6 +378,37 @@ aws: # Minimum retention period in hours for Kinesis streams min_kinesis_stream_retention_hours: 168 # 7 days + # Detect Secrets plugin configuration + detect_secrets_plugins: [ + {"name": "ArtifactoryDetector"}, + {"name": "AWSKeyDetector"}, + {"name": "AzureStorageKeyDetector"}, + {"name": "BasicAuthDetector"}, + {"name": "CloudantDetector"}, + {"name": "DiscordBotTokenDetector"}, + {"name": "GitHubTokenDetector"}, + {"name": "GitLabTokenDetector"}, + {"name": "Base64HighEntropyString", "limit": 6.0}, + {"name": "HexHighEntropyString", "limit": 3.0}, + {"name": "IbmCloudIamDetector"}, + {"name": "IbmCosHmacDetector"}, + # {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885 + {"name": "JwtTokenDetector"}, + {"name": "KeywordDetector"}, + {"name": "MailchimpDetector"}, + {"name": "NpmDetector"}, + {"name": "OpenAIDetector"}, + {"name": "PrivateKeyDetector"}, + {"name": "PypiTokenDetector"}, + {"name": "SendGridDetector"}, + {"name": "SlackDetector"}, + {"name": "SoftlayerDetector"}, + {"name": "SquareOAuthDetector"}, + {"name": "StripeDetector"}, + # {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878 + {"name": "TwilioKeyDetector"}, + ] + # Azure Configuration azure: diff --git a/prowler/lib/utils/utils.py b/prowler/lib/utils/utils.py index dbc735f293..0a5787f7a7 100644 --- a/prowler/lib/utils/utils.py +++ b/prowler/lib/utils/utils.py @@ -26,6 +26,36 @@ from detect_secrets.settings import transient_settings from prowler.config.config import encoding_format_utf_8 from prowler.lib.logger import logger +default_detect_secrets_plugins = [ + {"name": "ArtifactoryDetector"}, + {"name": "AWSKeyDetector"}, + {"name": "AzureStorageKeyDetector"}, + {"name": "BasicAuthDetector"}, + {"name": "CloudantDetector"}, + {"name": "DiscordBotTokenDetector"}, + {"name": "GitHubTokenDetector"}, + {"name": "GitLabTokenDetector"}, + {"name": "Base64HighEntropyString", "limit": 6.0}, + {"name": "HexHighEntropyString", "limit": 3.0}, + {"name": "IbmCloudIamDetector"}, + {"name": "IbmCosHmacDetector"}, + # {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885 + {"name": "JwtTokenDetector"}, + {"name": "KeywordDetector"}, + {"name": "MailchimpDetector"}, + {"name": "NpmDetector"}, + {"name": "OpenAIDetector"}, + {"name": "PrivateKeyDetector"}, + {"name": "PypiTokenDetector"}, + {"name": "SendGridDetector"}, + {"name": "SlackDetector"}, + {"name": "SoftlayerDetector"}, + {"name": "SquareOAuthDetector"}, + {"name": "StripeDetector"}, + # {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878 + {"name": "TwilioKeyDetector"}, +] + def open_file(input_file: str, mode: str = "r") -> TextIOWrapper: """open_file returns a handler to the file using the specified mode.""" @@ -82,13 +112,17 @@ def hash_sha512(string: str) -> str: def detect_secrets_scan( - data: str = None, file=None, excluded_secrets: list[str] = None + data: str = None, + file=None, + excluded_secrets: list[str] = None, + detect_secrets_plugins: dict = None, ) -> list[dict[str, str]]: """detect_secrets_scan scans the data or file for secrets using the detect-secrets library. Args: data (str): The data to scan for secrets. file (str): The file to scan for secrets. excluded_secrets (list): A list of regex patterns to exclude from the scan. + detect_secrets_plugins (dict): The settings to use for the scan. Returns: dict: The secrets found in the Raises: @@ -107,36 +141,11 @@ def detect_secrets_scan( secrets = SecretsCollection() + if not detect_secrets_plugins: + detect_secrets_plugins = default_detect_secrets_plugins + settings = { - "plugins_used": [ - {"name": "ArtifactoryDetector"}, - {"name": "AWSKeyDetector"}, - {"name": "AzureStorageKeyDetector"}, - {"name": "BasicAuthDetector"}, - {"name": "CloudantDetector"}, - {"name": "DiscordBotTokenDetector"}, - {"name": "GitHubTokenDetector"}, - {"name": "GitLabTokenDetector"}, - {"name": "Base64HighEntropyString", "limit": 6.0}, - {"name": "HexHighEntropyString", "limit": 3.0}, - {"name": "IbmCloudIamDetector"}, - {"name": "IbmCosHmacDetector"}, - # {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885 - {"name": "JwtTokenDetector"}, - {"name": "KeywordDetector"}, - {"name": "MailchimpDetector"}, - {"name": "NpmDetector"}, - {"name": "OpenAIDetector"}, - {"name": "PrivateKeyDetector"}, - {"name": "PypiTokenDetector"}, - {"name": "SendGridDetector"}, - {"name": "SlackDetector"}, - {"name": "SoftlayerDetector"}, - {"name": "SquareOAuthDetector"}, - {"name": "StripeDetector"}, - # {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878 - {"name": "TwilioKeyDetector"}, - ], + "plugins_used": detect_secrets_plugins, "filters_used": [ {"path": "detect_secrets.filters.common.is_invalid_file"}, {"path": "detect_secrets.filters.common.is_known_false_positive"}, @@ -144,6 +153,7 @@ def detect_secrets_scan( {"path": "detect_secrets.filters.heuristic.is_potential_secret"}, ], } + if excluded_secrets and len(excluded_secrets) > 0: settings["filters_used"].append( { diff --git a/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py b/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py index eb4af7ea82..88b4678acc 100644 --- a/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py +++ b/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py @@ -46,7 +46,11 @@ class autoscaling_find_secrets_ec2_launch_configuration(Check): continue has_secrets = detect_secrets_scan( - data=user_data, excluded_secrets=secrets_ignore_patterns + data=user_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=autoscaling_client.audit_config.get( + "detect_secrets_plugins" + ), ) if has_secrets: diff --git a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py index b03e339321..b9dc59cebb 100644 --- a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py +++ b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py @@ -34,6 +34,9 @@ class awslambda_function_no_secrets_in_code(Check): detect_secrets_output = detect_secrets_scan( file=f"{tmp_dir_name}/{file}", excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=awslambda_client.audit_config.get( + "detect_secrets_plugins", + ), ) if detect_secrets_output: for ( diff --git a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py index 9ed6065511..0905c752d2 100644 --- a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py +++ b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py @@ -28,6 +28,9 @@ class awslambda_function_no_secrets_in_variables(Check): detect_secrets_output = detect_secrets_scan( data=json.dumps(function.environment, indent=2), excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=awslambda_client.audit_config.get( + "detect_secrets_plugins", + ), ) original_env_vars = {} for name, value in function.environment.items(): diff --git a/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py b/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py index 2bcaa2ac30..d40a8fcfe1 100644 --- a/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py +++ b/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py @@ -31,7 +31,11 @@ class cloudformation_stack_outputs_find_secrets(Check): data += f"{output}\n" detect_secrets_output = detect_secrets_scan( - data=data, excluded_secrets=secrets_ignore_patterns + data=data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=cloudformation_client.audit_config.get( + "detect_secrets_plugins", + ), ) # If secrets are found, update the report status if detect_secrets_output: diff --git a/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py b/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py index 0e350a31f7..b819214592 100644 --- a/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py +++ b/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py @@ -38,6 +38,9 @@ class cloudwatch_log_group_no_secrets_in_logs(Check): log_stream_secrets_output = detect_secrets_scan( data=log_stream_data, excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=logs_client.audit_config.get( + "detect_secrets_plugins", + ), ) if log_stream_secrets_output: @@ -70,7 +73,10 @@ class cloudwatch_log_group_no_secrets_in_logs(Check): # Can get more informative output if there is more than 1 line. # Will rescan just this event to get the type of secret and the line number event_detect_secrets_output = detect_secrets_scan( - data=log_event_data + data=log_event_data, + detect_secrets_plugins=logs_client.audit_config.get( + "detect_secrets_plugins" + ), ) if event_detect_secrets_output: for secret in event_detect_secrets_output: diff --git a/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py b/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py index d21dc660b5..58d58f730e 100644 --- a/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py +++ b/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py @@ -33,6 +33,9 @@ class codebuild_project_no_secrets_in_variables(Check): detect_secrets_output = detect_secrets_scan( data=json.dumps({env_var.name: env_var.value}), excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=codebuild_client.audit_config.get( + "detect_secrets_plugins", + ), ) if detect_secrets_output: secrets_info = [ diff --git a/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py b/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py index 0bb21a93f6..88de6e720f 100644 --- a/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py +++ b/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py @@ -37,7 +37,11 @@ class ec2_instance_secrets_user_data(Check): ) continue detect_secrets_output = detect_secrets_scan( - data=user_data, excluded_secrets=secrets_ignore_patterns + data=user_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ec2_client.audit_config.get( + "detect_secrets_plugins" + ), ) if detect_secrets_output: secrets_string = ", ".join( diff --git a/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py b/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py index 9b0779f705..97844e5253 100644 --- a/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py +++ b/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py @@ -43,7 +43,11 @@ class ec2_launch_template_no_secrets(Check): continue version_secrets = detect_secrets_scan( - data=user_data, excluded_secrets=secrets_ignore_patterns + data=user_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ec2_client.audit_config.get( + "detect_secrets_plugins" + ), ) if version_secrets: diff --git a/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py b/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py index 66ad7a299b..5ecda04b64 100644 --- a/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py +++ b/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py @@ -39,7 +39,11 @@ class ecs_task_definitions_no_environment_secrets(Check): env_data = dumps(dump_env_vars, indent=2) detect_secrets_output = detect_secrets_scan( - data=env_data, excluded_secrets=secrets_ignore_patterns + data=env_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ecs_client.audit_config.get( + "detect_secrets_plugins", + ), ) if detect_secrets_output: secrets_string = ", ".join( diff --git a/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py b/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py index ec99d163b7..d70cf187e4 100644 --- a/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py +++ b/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py @@ -26,6 +26,9 @@ class ssm_document_secrets(Check): detect_secrets_output = detect_secrets_scan( data=json.dumps(document.content, indent=2), excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ssm_client.audit_config.get( + "detect_secrets_plugins" + ), ) if detect_secrets_output: secrets_string = ", ".join( diff --git a/tests/lib/utils/utils_test.py b/tests/lib/utils/utils_test.py index c3f90e341e..248305e039 100644 --- a/tests/lib/utils/utils_test.py +++ b/tests/lib/utils/utils_test.py @@ -111,7 +111,7 @@ class Test_utils_validate_ip_address: class Test_detect_secrets_scan: def test_detect_secrets_scan_data(self): data = "password=password" - secrets_detected = detect_secrets_scan(data=data) + secrets_detected = detect_secrets_scan(data=data, excluded_secrets=[]) assert type(secrets_detected) is list assert len(secrets_detected) == 1 assert "filename" in secrets_detected[0] @@ -128,7 +128,9 @@ class Test_detect_secrets_scan: temp_data_file = tempfile.NamedTemporaryFile(delete=False) temp_data_file.write(b"password=password") temp_data_file.seek(0) - secrets_detected = detect_secrets_scan(file=temp_data_file.name) + secrets_detected = detect_secrets_scan( + file=temp_data_file.name, excluded_secrets=[] + ) assert type(secrets_detected) is list assert len(secrets_detected) == 1 assert "filename" in secrets_detected[0] diff --git a/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py b/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py index 6499815964..4659d88af7 100644 --- a/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py +++ b/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py @@ -203,6 +203,7 @@ class Test_ec2_launch_template_no_secrets: ) ec2_client.launch_templates = [launch_template] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", @@ -277,6 +278,7 @@ class Test_ec2_launch_template_no_secrets: ) ec2_client.launch_templates = [launch_template] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", @@ -341,6 +343,7 @@ class Test_ec2_launch_template_no_secrets: ) ec2_client.launch_templates = [launch_template] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", @@ -482,6 +485,7 @@ class Test_ec2_launch_template_no_secrets: launch_template_secrets, launch_template_no_secrets, ] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", diff --git a/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py b/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py index cace44e681..24f0a1fdd1 100644 --- a/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py +++ b/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py @@ -27,6 +27,7 @@ class Test_ssm_documents_secrets: document_name = "test-document" document_arn = f"arn:aws:ssm:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:document/{document_name}" ssm_client.audited_account = AWS_ACCOUNT_NUMBER + ssm_client.audit_config = {"detect_secrets_plugins": None} ssm_client.documents = { document_name: Document( arn=document_arn,