From 3d19a496e58c446fe88bad632c8fcd8e8e92008b Mon Sep 17 00:00:00 2001 From: HugoPBrito Date: Wed, 27 Aug 2025 11:34:17 +0200 Subject: [PATCH] fix: lib/check tests --- prowler/lib/check/models.py | 22 +- tests/lib/check/check_loader_test.py | 72 +- tests/lib/check/compliance_check_test.py | 4 +- .../lib/check/custom_checks_metadata_test.py | 2 +- .../check/fixtures/bulk_checks_metadata.py | 20 +- tests/lib/check/models_test.py | 1094 ++++++++++++++++- 6 files changed, 1160 insertions(+), 54 deletions(-) diff --git a/prowler/lib/check/models.py b/prowler/lib/check/models.py index 78e3cb644c..5679cd3daa 100644 --- a/prowler/lib/check/models.py +++ b/prowler/lib/check/models.py @@ -27,20 +27,26 @@ def _validate_aws_check_type_in_config(check_type: str) -> bool: bool: True if the CheckType path exists in the config hierarchy """ try: - # Get config directly from global provider like custom checks do - from prowler.providers.common.provider import Provider + import json + import os - if not hasattr(Provider, "_global_provider") or not Provider._global_provider: + if not check_type: return False - # Access config directly like: service_client.audit_config.get("key", default) - hierarchy = Provider._global_provider.audit_config.get("aws", {}).get( - "valid_check_types", {} + # Get the path to the AWS CheckTypes configuration + current_dir = os.path.dirname(os.path.abspath(__file__)) + check_types_file = os.path.join( + current_dir, "..", "..", "providers", "aws", "config", "check_types.json" ) + check_types_file = os.path.normpath(check_types_file) - if not check_type or not hierarchy: + # Load the CheckTypes hierarchy from JSON file + if not os.path.exists(check_types_file): return False + with open(check_types_file, "r") as f: + hierarchy = json.load(f) + # Split the path by '/' to get each level path_parts = check_type.split("/") @@ -53,7 +59,7 @@ def _validate_aws_check_type_in_config(check_type: str) -> bool: return True - except (KeyError, AttributeError): + except (KeyError, AttributeError, FileNotFoundError, json.JSONDecodeError): return False diff --git a/tests/lib/check/check_loader_test.py b/tests/lib/check/check_loader_test.py index d61c7149d5..aa1f9a5f25 100644 --- a/tests/lib/check/check_loader_test.py +++ b/tests/lib/check/check_loader_test.py @@ -1,4 +1,4 @@ -from mock import patch +from unittest.mock import patch from prowler.lib.check.checks_loader import ( load_checks_to_execute, @@ -30,7 +30,9 @@ class TestCheckLoader: Provider="aws", CheckID=S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME, CheckTitle="Check S3 Bucket Level Public Access Block.", - CheckType=["Data Protection"], + CheckType=[ + "Software and Configuration Checks/AWS Security Best Practices/Network Reachability" + ], CheckAliases=[S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME_CUSTOM_ALIAS], ServiceName=S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME_SERVICE, SubServiceName="", @@ -64,7 +66,9 @@ class TestCheckLoader: Provider="aws", CheckID=IAM_USER_NO_MFA_NAME, CheckTitle="Check IAM User No MFA.", - CheckType=["Data Protection"], + CheckType=[ + "Software and Configuration Checks/Industry and Regulatory Standards/CIS AWS Foundations Benchmark" + ], CheckAliases=[IAM_USER_NO_MFA_NAME_CUSTOM_ALIAS], ServiceName=IAM_USER_NO_MFA_NAME_SERVICE, SubServiceName="", @@ -98,7 +102,7 @@ class TestCheckLoader: Provider="aws", CheckID=CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME, CheckTitle="Ensure there are no potential enumeration threats in CloudTrail", - CheckType=[], + CheckType=["TTPs/Discovery"], ServiceName="cloudtrail", SubServiceName="", ResourceIdTemplate="arn:partition:service:region:account-id:resource-id", @@ -122,55 +126,55 @@ class TestCheckLoader: ) def test_load_checks_to_execute(self): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, provider=self.provider, ) def test_load_checks_to_execute_with_check_list(self): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } check_list = [S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME] assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, check_list=check_list, provider=self.provider, ) def test_load_checks_to_execute_with_severities(self): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } severities = [S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_SEVERITY] assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, severities=severities, provider=self.provider, ) def test_load_checks_to_execute_with_severities_and_services(self): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } service_list = [S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME_SERVICE] severities = [S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_SEVERITY] assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, service_list=service_list, severities=severities, provider=self.provider, ) def test_load_checks_to_execute_with_severities_and_services_multiple(self): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata(), IAM_USER_NO_MFA_NAME: self.get_custom_check_iam_metadata(), } @@ -181,7 +185,7 @@ class TestCheckLoader: S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME, IAM_USER_NO_MFA_NAME, } == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, service_list=service_list, severities=severities, provider=self.provider, @@ -190,14 +194,14 @@ class TestCheckLoader: def test_load_checks_to_execute_with_severities_and_services_not_within_severity( self, ): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } service_list = ["ec2"] severities = [S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_SEVERITY] assert set() == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, service_list=service_list, severities=severities, provider=self.provider, @@ -206,7 +210,7 @@ class TestCheckLoader: def test_load_checks_to_execute_with_checks_file( self, ): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } checks_file = "path/to/test_file" @@ -215,7 +219,7 @@ class TestCheckLoader: return_value={S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME}, ): assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, checks_file=checks_file, provider=self.provider, ) @@ -223,13 +227,13 @@ class TestCheckLoader: def test_load_checks_to_execute_with_service_list( self, ): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } service_list = [S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME_SERVICE] assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, service_list=service_list, provider=self.provider, ) @@ -237,7 +241,7 @@ class TestCheckLoader: def test_load_checks_to_execute_with_compliance_frameworks( self, ): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } bulk_compliance_frameworks = { @@ -259,7 +263,7 @@ class TestCheckLoader: compliance_frameworks = ["soc2_aws"] assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, bulk_compliance_frameworks=bulk_compliance_frameworks, compliance_frameworks=compliance_frameworks, provider=self.provider, @@ -268,24 +272,24 @@ class TestCheckLoader: def test_load_checks_to_execute_with_categories( self, ): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } categories = {"internet-exposed"} assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, categories=categories, provider=self.provider, ) def test_load_checks_to_execute_no_bulk_checks_metadata(self): - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } with patch( "prowler.lib.check.checks_loader.CheckMetadata.get_bulk", - return_value=bulk_checks_metatada, + return_value=bulk_checks_metadata, ): assert {S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME} == load_checks_to_execute( provider=self.provider, @@ -311,13 +315,13 @@ class TestCheckLoader: compliance_frameworks = ["soc2_aws"] - bulk_checks_metatada = { + bulk_checks_metadata = { S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME: self.get_custom_check_s3_metadata() } with ( patch( "prowler.lib.check.checks_loader.CheckMetadata.get_bulk", - return_value=bulk_checks_metatada, + return_value=bulk_checks_metadata, ), patch( "prowler.lib.check.checks_loader.Compliance.get_bulk", @@ -344,38 +348,38 @@ class TestCheckLoader: ) def test_threat_detection_category(self): - bulk_checks_metatada = { + bulk_checks_metadata = { CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME: self.get_threat_detection_check_metadata() } categories = {"threat-detection"} assert {CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, categories=categories, provider=self.provider, ) def test_discard_threat_detection_checks(self): - bulk_checks_metatada = { + bulk_checks_metadata = { CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME: self.get_threat_detection_check_metadata() } categories = {} assert set() == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, categories=categories, provider=self.provider, ) def test_threat_detection_single_check(self): - bulk_checks_metatada = { + bulk_checks_metadata = { CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME: self.get_threat_detection_check_metadata() } categories = {} check_list = [CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME] assert {CLOUDTRAIL_THREAT_DETECTION_ENUMERATION_NAME} == load_checks_to_execute( - bulk_checks_metadata=bulk_checks_metatada, + bulk_checks_metadata=bulk_checks_metadata, check_list=check_list, categories=categories, provider=self.provider, diff --git a/tests/lib/check/compliance_check_test.py b/tests/lib/check/compliance_check_test.py index f7a58f04d0..3ebc0ed642 100644 --- a/tests/lib/check/compliance_check_test.py +++ b/tests/lib/check/compliance_check_test.py @@ -192,7 +192,7 @@ class TestCompliance: Provider="aws", CheckID="accessanalyzer_enabled", CheckTitle="Check 1", - CheckType=["type1"], + CheckType=["TTPs/Initial Access"], ServiceName="accessanalyzer", SubServiceName="subservice1", ResourceIdTemplate="template1", @@ -220,7 +220,7 @@ class TestCompliance: Provider="aws", CheckID="iam_user_mfa_enabled_console_access", CheckTitle="Check 2", - CheckType=["type2"], + CheckType=["TTPs/Credential Access"], ServiceName="iam", SubServiceName="subservice2", ResourceIdTemplate="template2", diff --git a/tests/lib/check/custom_checks_metadata_test.py b/tests/lib/check/custom_checks_metadata_test.py index 69f27880a8..32c57199bc 100644 --- a/tests/lib/check/custom_checks_metadata_test.py +++ b/tests/lib/check/custom_checks_metadata_test.py @@ -36,7 +36,7 @@ class TestCustomChecksMetadata: Provider="aws", CheckID=S3_BUCKET_LEVEL_PUBLIC_ACCESS_BLOCK_NAME, CheckTitle="Check S3 Bucket Level Public Access Block.", - CheckType=["Data Protection"], + CheckType=["Sensitive Data Identifications/PII"], CheckAliases=[], ServiceName="s3", SubServiceName="", diff --git a/tests/lib/check/fixtures/bulk_checks_metadata.py b/tests/lib/check/fixtures/bulk_checks_metadata.py index d0eee8bc66..2de2ca2a2a 100644 --- a/tests/lib/check/fixtures/bulk_checks_metadata.py +++ b/tests/lib/check/fixtures/bulk_checks_metadata.py @@ -5,7 +5,9 @@ test_bulk_checks_metadata = { Provider="aws", CheckID="vpc_peering_routing_tables_with_least_privilege", CheckTitle="Ensure routing tables for VPC peering are least access.", - CheckType=["Infrastructure Security"], + CheckType=[ + "Software and Configuration Checks/AWS Security Best Practices/Network Reachability" + ], ServiceName="vpc", SubServiceName="route_table", ResourceIdTemplate="arn:partition:service:region:account-id:resource-id", @@ -36,7 +38,9 @@ test_bulk_checks_metadata = { Provider="aws", CheckID="vpc_subnet_different_az", CheckTitle="Ensure all vpc has subnets in more than one availability zone", - CheckType=["Infrastructure Security"], + CheckType=[ + "Software and Configuration Checks/AWS Security Best Practices/Network Reachability" + ], ServiceName="vpc", SubServiceName="subnet", ResourceIdTemplate="arn:partition:service:region:account-id:resource-id", @@ -64,7 +68,9 @@ test_bulk_checks_metadata = { Provider="aws", CheckID="vpc_subnet_separate_private_public", CheckTitle="Ensure all vpc has public and private subnets defined", - CheckType=["Infrastructure Security"], + CheckType=[ + "Software and Configuration Checks/AWS Security Best Practices/Network Reachability" + ], ServiceName="vpc", SubServiceName="subnet", ResourceIdTemplate="arn:partition:service:region:account-id:resource-id", @@ -91,7 +97,9 @@ test_bulk_checks_metadata = { Provider="aws", CheckID="workspaces_volume_encryption_enabled", CheckTitle="Ensure that your Amazon WorkSpaces storage volumes are encrypted in order to meet security and compliance requirements", - CheckType=[], + CheckType=[ + "Software and Configuration Checks/AWS Security Best Practices/Runtime Behavior Analysis" + ], ServiceName="workspaces", SubServiceName="", ResourceIdTemplate="arn:aws:workspaces:region:account-id:workspace", @@ -122,7 +130,9 @@ test_bulk_checks_metadata = { Provider="aws", CheckID="workspaces_vpc_2private_1public_subnets_nat", CheckTitle="Ensure that the Workspaces VPC are deployed following the best practices using 1 public subnet and 2 private subnets with a NAT Gateway attached", - CheckType=[], + CheckType=[ + "Software and Configuration Checks/AWS Security Best Practices/Runtime Behavior Analysis" + ], ServiceName="workspaces", SubServiceName="", ResourceIdTemplate="arn:aws:workspaces:region:account-id:workspace", diff --git a/tests/lib/check/models_test.py b/tests/lib/check/models_test.py index 83b294f171..d052de9086 100644 --- a/tests/lib/check/models_test.py +++ b/tests/lib/check/models_test.py @@ -1,13 +1,16 @@ from unittest import mock +import pytest +from pydantic.v1 import ValidationError + from prowler.lib.check.models import CheckMetadata from tests.lib.check.compliance_check_test import custom_compliance_metadata mock_metadata = CheckMetadata( - Provider="aws", + Provider="azure", # Using non-AWS provider to avoid config validation issues CheckID="accessanalyzer_enabled", CheckTitle="Check 1", - CheckType=["type1"], + CheckType=["Security"], ServiceName="accessanalyzer", SubServiceName="subservice1", ResourceIdTemplate="template1", @@ -33,10 +36,10 @@ mock_metadata = CheckMetadata( ) mock_metadata_lambda = CheckMetadata( - Provider="aws", + Provider="azure", # Using non-AWS provider to avoid config validation issues CheckID="awslambda_function_url_public", CheckTitle="Check 1", - CheckType=["type1"], + CheckType=["Security"], ServiceName="awslambda", SubServiceName="subservice1", ResourceIdTemplate="template1", @@ -325,3 +328,1086 @@ class TestCheckMetada: result = CheckMetadata.list(bulk_checks_metadata=bulk_metadata) assert result == set() + + +class TestCheckMetadataValidators: + """Test class for CheckMetadata validators""" + + def test_valid_category_success(self): + """Test valid category validation with valid categories""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security", "network", "data-protection"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + # Should not raise any validation error + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.Categories == ["security", "network", "data-protection"] + + def test_valid_category_failure_non_string(self): + """Test valid category validation fails with non-string category""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": [123], # Invalid: number instead of string + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "Categories must be a list of strings" in str(exc_info.value) + + def test_valid_category_failure_invalid_format(self): + """Test valid category validation fails with invalid format""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["invalid_category!"], # Invalid: contains special character + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert ( + "Categories can only contain lowercase letters, numbers and hyphen" + in str(exc_info.value) + ) + + def test_severity_to_lower_success(self): + """Test severity validation converts to lowercase""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "HIGH", # Uppercase - should be converted to lowercase + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.Severity == "high" + + def test_valid_cli_command_success(self): + """Test CLI command validation with valid command""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "aws iam create-role --role-name test", # Valid CLI command + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + # Should not raise any validation error + check_metadata = CheckMetadata(**valid_metadata) + assert "aws iam create-role" in check_metadata.Remediation.Code.CLI + + def test_valid_cli_command_failure_url(self): + """Test CLI command validation fails with URL""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "https://example.com/command", # Invalid: URL instead of command + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "CLI command cannot be an URL" in str(exc_info.value) + + def test_valid_resource_type_success(self): + """Test resource type validation with valid resource type""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "AWS::IAM::Role", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.ResourceType == "AWS::IAM::Role" + + def test_valid_resource_type_failure_empty(self): + """Test resource type validation fails with empty string""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "", # Invalid: empty string + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "ResourceType must be a non-empty string" in str(exc_info.value) + + def test_validate_service_name_success(self): + """Test service name validation with valid service name matching CheckID""" + valid_metadata = { + "Provider": "azure", + "CheckID": "s3_bucket_public_read", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "s3", # Matches first part of CheckID + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "AWS::S3::Bucket", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.ServiceName == "s3" + + def test_validate_service_name_failure_mismatch(self): + """Test service name validation fails when not matching CheckID""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "s3_bucket_public_read", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "ec2", # Does not match first part of CheckID + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "AWS::S3::Bucket", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert ( + "ServiceName ec2 does not belong to CheckID s3_bucket_public_read" + in str(exc_info.value) + ) + + def test_validate_service_name_failure_uppercase(self): + """Test service name validation fails with uppercase""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "S3_bucket_public_read", + "CheckTitle": "Test Check", + "CheckType": ["TTPs/Discovery"], + "ServiceName": "S3", # Invalid: uppercase + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "AWS::S3::Bucket", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "ServiceName S3 must be in lowercase" in str(exc_info.value) + + def test_validate_service_name_iac_provider_success(self): + """Test service name validation allows any service name for IAC provider""" + valid_metadata = { + "Provider": "iac", + "CheckID": "custom_check_id", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "CustomService", # Valid for IAC provider + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.ServiceName == "CustomService" + + def test_valid_check_id_success(self): + """Test CheckID validation with valid check ID""" + valid_metadata = { + "Provider": "azure", + "CheckID": "s3_bucket_public_read_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "s3", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "AWS::S3::Bucket", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.CheckID == "s3_bucket_public_read_check" + + def test_valid_check_id_failure_empty(self): + """Test CheckID validation fails with empty string""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "", # Invalid: empty string + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "CheckID must be a non-empty string" in str(exc_info.value) + + def test_valid_check_id_failure_hyphen(self): + """Test CheckID validation fails with hyphen""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "s3-bucket-public-read", # Invalid: contains hyphens + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "s3", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "AWS::S3::Bucket", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert ( + "CheckID s3-bucket-public-read contains a hyphen, which is not allowed" + in str(exc_info.value) + ) + + def test_validate_check_title_success(self): + """Test CheckTitle validation with valid title""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "A" * 150, # Exactly 150 characters + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert len(check_metadata.CheckTitle) == 150 + + def test_validate_check_title_failure_too_long(self): + """Test CheckTitle validation fails when too long""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "A" * 151, # Too long: 151 characters + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "CheckTitle must not exceed 150 characters, got 151 characters" in str( + exc_info.value + ) + + def test_validate_check_type_success(self): + """Test CheckType validation with valid check types""" + valid_metadata = { + "Provider": "azure", # Using non-AWS provider to avoid config validation + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security", "Network"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.CheckType == ["Security", "Network"] + + def test_validate_check_type_failure_empty_string(self): + """Test CheckType validation fails with empty string in list""" + invalid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security", ""], # Invalid: empty string in list + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "CheckType list cannot contain empty strings" in str(exc_info.value) + + def test_validate_description_success(self): + """Test Description validation with valid description""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "A" * 400, # Exactly 400 characters + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert len(check_metadata.Description) == 400 + + def test_validate_description_failure_too_long(self): + """Test Description validation fails when too long""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "A" * 401, # Too long: 401 characters + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "Description must not exceed 400 characters, got 401 characters" in str( + exc_info.value + ) + + def test_validate_risk_success(self): + """Test Risk validation with valid risk""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "A" * 400, # Exactly 400 characters + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert len(check_metadata.Risk) == 400 + + def test_validate_risk_failure_too_long(self): + """Test Risk validation fails when too long""" + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security"], + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "A" * 401, # Too long: 401 characters + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "Risk must not exceed 400 characters, got 401 characters" in str( + exc_info.value + ) + + def test_validate_check_type_aws_invalid_type(self): + """Test CheckType validation fails with invalid AWS CheckType""" + + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["InvalidType"], # Invalid: not in AWS config + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "Invalid CheckType: 'InvalidType'" in str(exc_info.value) + + def test_validate_check_type_aws_valid_hierarchy_path(self): + """Test CheckType validation succeeds with valid AWS CheckType hierarchy path""" + + valid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["TTPs/Initial Access"], # Valid: partial path in hierarchy + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.CheckType == ["TTPs/Initial Access"] + + def test_validate_check_type_non_aws_provider(self): + """Test CheckType validation doesn't apply AWS rules to non-AWS providers""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["CustomType"], # Valid for non-AWS provider + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.CheckType == ["CustomType"] + + def test_validate_check_type_aws_validation_called(self): + """Test that AWS CheckType validation function works for AWS provider""" + + valid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Effects/Data Exposure"], # Valid AWS CheckType + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.CheckType == ["Effects/Data Exposure"] + + def test_validate_check_type_multiple_types_all_valid(self): + """Test CheckType validation with multiple valid types for non-AWS provider""" + valid_metadata = { + "Provider": "azure", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["Security", "Network", "Compliance"], # Multiple valid types + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + check_metadata = CheckMetadata(**valid_metadata) + assert check_metadata.CheckType == ["Security", "Network", "Compliance"] + + def test_validate_check_type_aws_multiple_types_mixed_validity(self): + """Test CheckType validation with multiple types where one is invalid for AWS""" + + invalid_metadata = { + "Provider": "aws", + "CheckID": "test_check", + "CheckTitle": "Test Check", + "CheckType": ["TTPs/Discovery", "InvalidType"], # One valid, one invalid + "ServiceName": "test", + "SubServiceName": "subtest", + "ResourceIdTemplate": "template", + "Severity": "high", + "ResourceType": "TestResource", + "Description": "Test description", + "Risk": "Test risk", + "RelatedUrl": "https://example.com", + "Remediation": { + "Code": { + "CLI": "test command", + "NativeIaC": "test native", + "Other": "test other", + "Terraform": "test terraform", + }, + "Recommendation": { + "Text": "test recommendation", + "Url": "https://example.com", + }, + }, + "Categories": ["security"], + "DependsOn": [], + "RelatedTo": [], + "Notes": "Test notes", + } + + with pytest.raises(ValidationError) as exc_info: + CheckMetadata(**invalid_metadata) + assert "Invalid CheckType: 'InvalidType'" in str(exc_info.value)