diff --git a/prowler/config/config.yaml b/prowler/config/config.yaml index cceb549948..e319f4a029 100644 --- a/prowler/config/config.yaml +++ b/prowler/config/config.yaml @@ -378,6 +378,37 @@ aws: # Minimum retention period in hours for Kinesis streams min_kinesis_stream_retention_hours: 168 # 7 days + # Detect Secrets plugin configuration + detect_secrets_plugins: [ + {"name": "ArtifactoryDetector"}, + {"name": "AWSKeyDetector"}, + {"name": "AzureStorageKeyDetector"}, + {"name": "BasicAuthDetector"}, + {"name": "CloudantDetector"}, + {"name": "DiscordBotTokenDetector"}, + {"name": "GitHubTokenDetector"}, + {"name": "GitLabTokenDetector"}, + {"name": "Base64HighEntropyString", "limit": 6.0}, + {"name": "HexHighEntropyString", "limit": 3.0}, + {"name": "IbmCloudIamDetector"}, + {"name": "IbmCosHmacDetector"}, + # {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885 + {"name": "JwtTokenDetector"}, + {"name": "KeywordDetector"}, + {"name": "MailchimpDetector"}, + {"name": "NpmDetector"}, + {"name": "OpenAIDetector"}, + {"name": "PrivateKeyDetector"}, + {"name": "PypiTokenDetector"}, + {"name": "SendGridDetector"}, + {"name": "SlackDetector"}, + {"name": "SoftlayerDetector"}, + {"name": "SquareOAuthDetector"}, + {"name": "StripeDetector"}, + # {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878 + {"name": "TwilioKeyDetector"}, + ] + # Azure Configuration azure: diff --git a/prowler/lib/utils/utils.py b/prowler/lib/utils/utils.py index dbc735f293..0a5787f7a7 100644 --- a/prowler/lib/utils/utils.py +++ b/prowler/lib/utils/utils.py @@ -26,6 +26,36 @@ from detect_secrets.settings import transient_settings from prowler.config.config import encoding_format_utf_8 from prowler.lib.logger import logger +default_detect_secrets_plugins = [ + {"name": "ArtifactoryDetector"}, + {"name": "AWSKeyDetector"}, + {"name": "AzureStorageKeyDetector"}, + {"name": "BasicAuthDetector"}, + {"name": "CloudantDetector"}, + {"name": "DiscordBotTokenDetector"}, + {"name": "GitHubTokenDetector"}, + {"name": "GitLabTokenDetector"}, + {"name": "Base64HighEntropyString", "limit": 6.0}, + {"name": "HexHighEntropyString", "limit": 3.0}, + {"name": "IbmCloudIamDetector"}, + {"name": "IbmCosHmacDetector"}, + # {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885 + {"name": "JwtTokenDetector"}, + {"name": "KeywordDetector"}, + {"name": "MailchimpDetector"}, + {"name": "NpmDetector"}, + {"name": "OpenAIDetector"}, + {"name": "PrivateKeyDetector"}, + {"name": "PypiTokenDetector"}, + {"name": "SendGridDetector"}, + {"name": "SlackDetector"}, + {"name": "SoftlayerDetector"}, + {"name": "SquareOAuthDetector"}, + {"name": "StripeDetector"}, + # {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878 + {"name": "TwilioKeyDetector"}, +] + def open_file(input_file: str, mode: str = "r") -> TextIOWrapper: """open_file returns a handler to the file using the specified mode.""" @@ -82,13 +112,17 @@ def hash_sha512(string: str) -> str: def detect_secrets_scan( - data: str = None, file=None, excluded_secrets: list[str] = None + data: str = None, + file=None, + excluded_secrets: list[str] = None, + detect_secrets_plugins: dict = None, ) -> list[dict[str, str]]: """detect_secrets_scan scans the data or file for secrets using the detect-secrets library. Args: data (str): The data to scan for secrets. file (str): The file to scan for secrets. excluded_secrets (list): A list of regex patterns to exclude from the scan. + detect_secrets_plugins (dict): The settings to use for the scan. Returns: dict: The secrets found in the Raises: @@ -107,36 +141,11 @@ def detect_secrets_scan( secrets = SecretsCollection() + if not detect_secrets_plugins: + detect_secrets_plugins = default_detect_secrets_plugins + settings = { - "plugins_used": [ - {"name": "ArtifactoryDetector"}, - {"name": "AWSKeyDetector"}, - {"name": "AzureStorageKeyDetector"}, - {"name": "BasicAuthDetector"}, - {"name": "CloudantDetector"}, - {"name": "DiscordBotTokenDetector"}, - {"name": "GitHubTokenDetector"}, - {"name": "GitLabTokenDetector"}, - {"name": "Base64HighEntropyString", "limit": 6.0}, - {"name": "HexHighEntropyString", "limit": 3.0}, - {"name": "IbmCloudIamDetector"}, - {"name": "IbmCosHmacDetector"}, - # {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885 - {"name": "JwtTokenDetector"}, - {"name": "KeywordDetector"}, - {"name": "MailchimpDetector"}, - {"name": "NpmDetector"}, - {"name": "OpenAIDetector"}, - {"name": "PrivateKeyDetector"}, - {"name": "PypiTokenDetector"}, - {"name": "SendGridDetector"}, - {"name": "SlackDetector"}, - {"name": "SoftlayerDetector"}, - {"name": "SquareOAuthDetector"}, - {"name": "StripeDetector"}, - # {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878 - {"name": "TwilioKeyDetector"}, - ], + "plugins_used": detect_secrets_plugins, "filters_used": [ {"path": "detect_secrets.filters.common.is_invalid_file"}, {"path": "detect_secrets.filters.common.is_known_false_positive"}, @@ -144,6 +153,7 @@ def detect_secrets_scan( {"path": "detect_secrets.filters.heuristic.is_potential_secret"}, ], } + if excluded_secrets and len(excluded_secrets) > 0: settings["filters_used"].append( { diff --git a/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py b/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py index d4a75535b9..6595b71085 100644 --- a/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py +++ b/prowler/providers/aws/services/autoscaling/autoscaling_find_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py @@ -43,7 +43,11 @@ class autoscaling_find_secrets_ec2_launch_configuration(Check): continue has_secrets = detect_secrets_scan( - data=user_data, excluded_secrets=secrets_ignore_patterns + data=user_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=autoscaling_client.audit_config.get( + "detect_secrets_plugins" + ), ) if has_secrets: diff --git a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py index 05ffd2bdc5..51c56a2011 100644 --- a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py +++ b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py @@ -32,6 +32,9 @@ class awslambda_function_no_secrets_in_code(Check): detect_secrets_output = detect_secrets_scan( file=f"{tmp_dir_name}/{file}", excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=awslambda_client.audit_config.get( + "detect_secrets_plugins", + ), ) if detect_secrets_output: for ( diff --git a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py index 3be5bafa18..aea6617e16 100644 --- a/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py +++ b/prowler/providers/aws/services/awslambda/awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py @@ -24,6 +24,9 @@ class awslambda_function_no_secrets_in_variables(Check): detect_secrets_output = detect_secrets_scan( data=json.dumps(function.environment, indent=2), excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=awslambda_client.audit_config.get( + "detect_secrets_plugins", + ), ) original_env_vars = {} for name, value in function.environment.items(): diff --git a/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py b/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py index 5a3e3b8889..f9b47932bb 100644 --- a/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py +++ b/prowler/providers/aws/services/cloudformation/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py @@ -27,7 +27,11 @@ class cloudformation_stack_outputs_find_secrets(Check): data += f"{output}\n" detect_secrets_output = detect_secrets_scan( - data=data, excluded_secrets=secrets_ignore_patterns + data=data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=cloudformation_client.audit_config.get( + "detect_secrets_plugins", + ), ) # If secrets are found, update the report status if detect_secrets_output: diff --git a/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py b/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py index 010a5b81fb..5154a5acb7 100644 --- a/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py +++ b/prowler/providers/aws/services/cloudwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py @@ -34,6 +34,9 @@ class cloudwatch_log_group_no_secrets_in_logs(Check): log_stream_secrets_output = detect_secrets_scan( data=log_stream_data, excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=logs_client.audit_config.get( + "detect_secrets_plugins", + ), ) if log_stream_secrets_output: @@ -66,7 +69,10 @@ class cloudwatch_log_group_no_secrets_in_logs(Check): # Can get more informative output if there is more than 1 line. # Will rescan just this event to get the type of secret and the line number event_detect_secrets_output = detect_secrets_scan( - data=log_event_data + data=log_event_data, + detect_secrets_plugins=logs_client.audit_config.get( + "detect_secrets_plugins" + ), ) if event_detect_secrets_output: for secret in event_detect_secrets_output: diff --git a/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py b/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py index d78e0d5f69..8d031cc25a 100644 --- a/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py +++ b/prowler/providers/aws/services/codebuild/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py @@ -29,6 +29,9 @@ class codebuild_project_no_secrets_in_variables(Check): detect_secrets_output = detect_secrets_scan( data=json.dumps({env_var.name: env_var.value}), excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=codebuild_client.audit_config.get( + "detect_secrets_plugins", + ), ) if detect_secrets_output: secrets_info = [ diff --git a/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py b/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py index 3bb4cd3947..3c3864c479 100644 --- a/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py +++ b/prowler/providers/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py @@ -37,7 +37,11 @@ class ec2_instance_secrets_user_data(Check): ) continue detect_secrets_output = detect_secrets_scan( - data=user_data, excluded_secrets=secrets_ignore_patterns + data=user_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ec2_client.audit_config.get( + "detect_secrets_plugins" + ), ) if detect_secrets_output: secrets_string = ", ".join( diff --git a/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py b/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py index 47277647ce..823553bcdf 100644 --- a/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py +++ b/prowler/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py @@ -43,7 +43,11 @@ class ec2_launch_template_no_secrets(Check): continue version_secrets = detect_secrets_scan( - data=user_data, excluded_secrets=secrets_ignore_patterns + data=user_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ec2_client.audit_config.get( + "detect_secrets_plugins" + ), ) if version_secrets: diff --git a/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py b/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py index c1b0a05396..42dc6b437c 100644 --- a/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py +++ b/prowler/providers/aws/services/ecs/ecs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py @@ -38,7 +38,11 @@ class ecs_task_definitions_no_environment_secrets(Check): env_data = dumps(dump_env_vars, indent=2) detect_secrets_output = detect_secrets_scan( - data=env_data, excluded_secrets=secrets_ignore_patterns + data=env_data, + excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ecs_client.audit_config.get( + "detect_secrets_plugins", + ), ) if detect_secrets_output: secrets_string = ", ".join( diff --git a/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py b/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py index 12d983f936..0ec8502bab 100644 --- a/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py +++ b/prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py @@ -22,6 +22,9 @@ class ssm_document_secrets(Check): detect_secrets_output = detect_secrets_scan( data=json.dumps(document.content, indent=2), excluded_secrets=secrets_ignore_patterns, + detect_secrets_plugins=ssm_client.audit_config.get( + "detect_secrets_plugins" + ), ) if detect_secrets_output: secrets_string = ", ".join( diff --git a/tests/lib/utils/utils_test.py b/tests/lib/utils/utils_test.py index c3f90e341e..248305e039 100644 --- a/tests/lib/utils/utils_test.py +++ b/tests/lib/utils/utils_test.py @@ -111,7 +111,7 @@ class Test_utils_validate_ip_address: class Test_detect_secrets_scan: def test_detect_secrets_scan_data(self): data = "password=password" - secrets_detected = detect_secrets_scan(data=data) + secrets_detected = detect_secrets_scan(data=data, excluded_secrets=[]) assert type(secrets_detected) is list assert len(secrets_detected) == 1 assert "filename" in secrets_detected[0] @@ -128,7 +128,9 @@ class Test_detect_secrets_scan: temp_data_file = tempfile.NamedTemporaryFile(delete=False) temp_data_file.write(b"password=password") temp_data_file.seek(0) - secrets_detected = detect_secrets_scan(file=temp_data_file.name) + secrets_detected = detect_secrets_scan( + file=temp_data_file.name, excluded_secrets=[] + ) assert type(secrets_detected) is list assert len(secrets_detected) == 1 assert "filename" in secrets_detected[0] diff --git a/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py b/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py index 6499815964..4659d88af7 100644 --- a/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py +++ b/tests/providers/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py @@ -203,6 +203,7 @@ class Test_ec2_launch_template_no_secrets: ) ec2_client.launch_templates = [launch_template] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", @@ -277,6 +278,7 @@ class Test_ec2_launch_template_no_secrets: ) ec2_client.launch_templates = [launch_template] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", @@ -341,6 +343,7 @@ class Test_ec2_launch_template_no_secrets: ) ec2_client.launch_templates = [launch_template] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", @@ -482,6 +485,7 @@ class Test_ec2_launch_template_no_secrets: launch_template_secrets, launch_template_no_secrets, ] + ec2_client.audit_config = {"detect_secrets_plugins": None} with mock.patch( "prowler.providers.common.provider.Provider.get_global_provider", diff --git a/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py b/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py index cace44e681..24f0a1fdd1 100644 --- a/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py +++ b/tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py @@ -27,6 +27,7 @@ class Test_ssm_documents_secrets: document_name = "test-document" document_arn = f"arn:aws:ssm:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:document/{document_name}" ssm_client.audited_account = AWS_ACCOUNT_NUMBER + ssm_client.audit_config = {"detect_secrets_plugins": None} ssm_client.documents = { document_name: Document( arn=document_arn,