From 47f7555d05fe1a685693fb81e6b9986d7f2d2977 Mon Sep 17 00:00:00 2001 From: Pepe Fagoaga Date: Mon, 5 Aug 2024 18:59:30 +0200 Subject: [PATCH] refactor(mutelist): Remove re.match and improve docs (#4637) Co-authored-by: Sergio --- docs/tutorials/mutelist.md | 192 ++++++++++-------- prowler/lib/mutelist/mutelist.py | 17 +- .../aws/lib/mutelist/aws_mutelist_test.py | 68 +++++-- .../lib/mutelist/kubernetes_mutelist_test.py | 37 +++- 4 files changed, 197 insertions(+), 117 deletions(-) diff --git a/docs/tutorials/mutelist.md b/docs/tutorials/mutelist.md index 1e3d468b1e..f67594c0ab 100644 --- a/docs/tutorials/mutelist.md +++ b/docs/tutorials/mutelist.md @@ -7,97 +7,121 @@ Mutelist option works along with other options and will modify the output in the - CSV: `muted` is `True`. The field `status` will keep the original status, `MANUAL`, `PASS` or `FAIL`, of the finding. -You can use `-w`/`--mutelist-file` with the path of your mutelist yaml file: +## How the Mutelist Works + +The Mutelist uses an "ANDed" and "ORed" logic to determine which resources, checks, regions, and tags should be muted. For each check, the Mutelist checks if the account, region, and resource match the specified criteria, using an "ANDed" logic. If tags are specified, the mutelist uses and "ORed" logic to see if at least one tag is present in the resource. + +If any of the criteria do not match, the check is not muted. + +## Mutelist Specification + +???+ note + - For Azure provider, the Account ID is the Subscription Name and the Region is the Location. + - For GCP provider, the Account ID is the Project ID and the Region is the Zone. + - For Kubernetes provider, the Account ID is the Cluster Name and the Region is the Namespace. + +The Mutelist file uses the [YAML](https://en.wikipedia.org/wiki/YAML) format with the following syntax: + +```yaml +### Account, Check and/or Region can be * to apply for all the cases. +### Resources and tags are lists that can have either Regex or Keywords. +### Tags is an optional list that matches on tuples of 'key=value' and are "ANDed" together. +### Use an alternation Regex to match one of multiple tags with "ORed" logic. +### For each check you can except Accounts, Regions, Resources and/or Tags. +########################### MUTELIST EXAMPLE ########################### +Mutelist: + Accounts: + "123456789012": + Checks: + "iam_user_hardware_mfa_enabled": + Regions: + - "us-east-1" + Resources: + - "user-1" # Will ignore user-1 in check iam_user_hardware_mfa_enabled + - "user-2" # Will ignore user-2 in check iam_user_hardware_mfa_enabled + "ec2_*": + Regions: + - "*" + Resources: + - "*" # Will ignore every EC2 check in every account and region + "*": + Regions: + - "*" + Resources: + - "test" + Tags: + - "test=test" # Will ignore every resource containing the string "test" and the tags 'test=test' and + - "project=test|project=stage" # either of ('project=test' OR project=stage) in account 123456789012 and every region + + "*": + Checks: + "s3_bucket_object_versioning": + Regions: + - "eu-west-1" + - "us-east-1" + Resources: + - "ci-logs" # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions + - "logs" # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions + - ".+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions + "ecs_task_definitions_no_environment_secrets": + Regions: + - "*" + Resources: + - "*" + Exceptions: + Accounts: + - "0123456789012" + Regions: + - "eu-west-1" + - "eu-south-2" # Will ignore every resource in check ecs_task_definitions_no_environment_secrets except the ones in account 0123456789012 located in eu-south-2 or eu-west-1 + "*": + Regions: + - "*" + Resources: + - "*" + Tags: + - "environment=dev" # Will ignore every resource containing the tag 'environment=dev' in every account and region + + "123456789012": + Checks: + "*": + Regions: + - "*" + Resources: + - "*" + Exceptions: + Resources: + - "test" + Tags: + - "environment=prod" # Will ignore every resource except in account 123456789012 except the ones containing the string "test" and tag environment=prod +``` + +### Account, Check, Region, Resource, and Tag + +| Field | Description | Logic | +|----------|----------|----------| +| `` | Use `*` to apply the mutelist to all accounts. | `ANDed` | +| `` | The name of the Prowler check. Use `*` to apply the mutelist to all checks. | `ANDed` | +| `` | The region identifier. Use `*` to apply the mutelist to all regions. | `ANDed` | +| `` | The resource identifier. Use `*` to apply the mutelist to all resources. | `ANDed` | +| `` | The tag value. | `ORed` | + + +## How to Use the Mutelist + +To use the Mutelist, you need to specify the path to the Mutelist YAML file using the `-w` or `--mutelist-file` option when running Prowler: + ``` prowler -w mutelist.yaml ``` -## Mutelist YAML File Syntax +Replace `` with the appropriate provider name. -???+ note - For Azure provider, the Account ID is the Subscription Name and the Region is the Location. +## Considerations -???+ note - For GCP provider, the Account ID is the Project ID and the Region is the Zone. +- The Mutelist can be used in combination with other Prowler options, such as the `--service` or `--checks` option, to further customize the scanning process. +- Make sure to review and update the Mutelist regularly to ensure it reflects the desired exclusions and remains up to date with your infrastructure. -???+ note - For Kubernetes provider, the Account ID is the Cluster Name and the Region is the Namespace. - -The Mutelist file is a YAML file with the following syntax: - -```yaml - ### Account, Check and/or Region can be * to apply for all the cases. - ### Resources and tags are lists that can have either Regex or Keywords. - ### Tags is an optional list that matches on tuples of 'key=value' and are "ANDed" together. - ### Use an alternation Regex to match one of multiple tags with "ORed" logic. - ### For each check you can except Accounts, Regions, Resources and/or Tags. - ########################### MUTELIST EXAMPLE ########################### - Mutelist: - Accounts: - "123456789012": - Checks: - "iam_user_hardware_mfa_enabled": - Regions: - - "us-east-1" - Resources: - - "user-1" # Will ignore user-1 in check iam_user_hardware_mfa_enabled - - "user-2" # Will ignore user-2 in check iam_user_hardware_mfa_enabled - "ec2_*": - Regions: - - "*" - Resources: - - "*" # Will ignore every EC2 check in every account and region - "*": - Regions: - - "*" - Resources: - - "test" - Tags: - - "test=test" # Will ignore every resource containing the string "test" and the tags 'test=test' and - - "project=test|project=stage" # either of ('project=test' OR project=stage) in account 123456789012 and every region - - "*": - Checks: - "s3_bucket_object_versioning": - Regions: - - "eu-west-1" - - "us-east-1" - Resources: - - "ci-logs" # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions - - "logs" # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions - - ".+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions - "ecs_task_definitions_no_environment_secrets": - Regions: - - "*" - Resources: - - "*" - Exceptions: - Accounts: - - "0123456789012" - Regions: - - "eu-west-1" - - "eu-south-2" # Will ignore every resource in check ecs_task_definitions_no_environment_secrets except the ones in account 0123456789012 located in eu-south-2 or eu-west-1 - "*": - Regions: - - "*" - Resources: - - "*" - Tags: - - "environment=dev" # Will ignore every resource containing the tag 'environment=dev' in every account and region - - "123456789012": - Checks: - "*": - Regions: - - "*" - Resources: - - "*" - Exceptions: - Resources: - - "test" - Tags: - - "environment=prod" # Will ignore every resource except in account 123456789012 except the ones containing the string "test" and tag environment=prod -``` ## AWS Mutelist ### Mute specific AWS regions diff --git a/prowler/lib/mutelist/mutelist.py b/prowler/lib/mutelist/mutelist.py index c5f9789b39..479847492d 100644 --- a/prowler/lib/mutelist/mutelist.py +++ b/prowler/lib/mutelist/mutelist.py @@ -211,9 +211,7 @@ class Mutelist(ABC): muted_in_resource = self.is_item_matched( muted_resources, finding_resource ) - muted_in_tags = self.is_item_matched( - muted_tags, finding_tags, tag=True - ) + muted_in_tags = self.is_item_matched(muted_tags, finding_tags) # For a finding to be muted requires the following set to True: # - muted_in_check -> True @@ -281,9 +279,7 @@ class Mutelist(ABC): ) excepted_tags = exceptions.get("Tags", []) - is_tag_excepted = self.is_item_matched( - excepted_tags, finding_tags, tag=True - ) + is_tag_excepted = self.is_item_matched(excepted_tags, finding_tags) if ( not is_account_excepted @@ -307,7 +303,7 @@ class Mutelist(ABC): return False @staticmethod - def is_item_matched(matched_items, finding_items, tag=False): + def is_item_matched(matched_items, finding_items): """ Check if any of the items in matched_items are present in finding_items. @@ -321,15 +317,10 @@ class Mutelist(ABC): try: is_item_matched = False if matched_items and (finding_items or finding_items == ""): - # If we use tags, we need to use re.search instead of re.match because we need to match the tags in the format key1=value1 | key2=value2 - if tag: - operation = re.search - else: - operation = re.match for item in matched_items: if item.startswith("*"): item = ".*" + item[1:] - if operation(item, finding_items): + if re.search(item, finding_items): is_item_matched = True break return is_item_matched diff --git a/tests/providers/aws/lib/mutelist/aws_mutelist_test.py b/tests/providers/aws/lib/mutelist/aws_mutelist_test.py index 60fb2e502b..a7902abdec 100644 --- a/tests/providers/aws/lib/mutelist/aws_mutelist_test.py +++ b/tests/providers/aws/lib/mutelist/aws_mutelist_test.py @@ -871,6 +871,46 @@ class TestAWSMutelist: mutelist.is_muted(AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test", "") ) + def test_is_muted_search(self): + # Mutelist + mutelist_content = { + "Accounts": { + AWS_ACCOUNT_NUMBER: { + "Checks": { + "check_test": { + "Regions": ["*"], + "Resources": ["prowler"], + } + } + } + } + } + mutelist = AWSMutelist(mutelist_content=mutelist_content) + + assert mutelist.is_muted( + AWS_ACCOUNT_NUMBER, + "check_test", + AWS_REGION_US_EAST_1, + "prowler", + "", + ) + + assert mutelist.is_muted( + AWS_ACCOUNT_NUMBER, + "check_test", + AWS_REGION_US_EAST_1, + "resource-prowler", + "", + ) + + assert mutelist.is_muted( + AWS_ACCOUNT_NUMBER, + "check_test", + AWS_REGION_US_EAST_1, + "prowler-resource", + "", + ) + def test_is_muted_in_region(self): muted_regions = [AWS_REGION_US_EAST_1, AWS_REGION_EU_WEST_1] finding_region = AWS_REGION_US_EAST_1 @@ -1223,49 +1263,43 @@ class TestAWSMutelist: def test_is_muted_in_tags(self): mutelist_tags = ["environment=dev", "project=prowler"] - assert AWSMutelist.is_item_matched(mutelist_tags, "environment=dev", tag=True) + assert AWSMutelist.is_item_matched(mutelist_tags, "environment=dev") assert AWSMutelist.is_item_matched( - mutelist_tags, "environment=dev | project=prowler", tag=True + mutelist_tags, "environment=dev | project=prowler" ) assert AWSMutelist.is_item_matched( - mutelist_tags, "environment=pro | project=prowler", tag=True + mutelist_tags, "environment=pro | project=prowler" ) - assert not ( - AWSMutelist.is_item_matched(mutelist_tags, "environment=pro", tag=True) - ) + assert not (AWSMutelist.is_item_matched(mutelist_tags, "environment=pro")) def test_is_muted_in_tags_with_piped_tags(self): mutelist_tags = ["environment=dev|project=prowler"] - assert AWSMutelist.is_item_matched(mutelist_tags, "environment=dev", tag=True) + assert AWSMutelist.is_item_matched(mutelist_tags, "environment=dev") assert AWSMutelist.is_item_matched( - mutelist_tags, "environment=dev | project=prowler", tag=True + mutelist_tags, "environment=dev | project=prowler" ) assert AWSMutelist.is_item_matched( - mutelist_tags, "environment=pro | project=prowler", tag=True + mutelist_tags, "environment=pro | project=prowler" ) - assert not ( - AWSMutelist.is_item_matched(mutelist_tags, "environment=pro", tag=True) - ) + assert not (AWSMutelist.is_item_matched(mutelist_tags, "environment=pro")) def test_is_muted_in_tags_regex(self): mutelist_tags = ["environment=(dev|test)", ".*=prowler"] assert AWSMutelist.is_item_matched( - mutelist_tags, "environment=test | proj=prowler", tag=True + mutelist_tags, "environment=test | proj=prowler" ) - assert AWSMutelist.is_item_matched( - mutelist_tags, "env=prod | project=prowler", tag=True - ) + assert AWSMutelist.is_item_matched(mutelist_tags, "env=prod | project=prowler") assert not AWSMutelist.is_item_matched( - mutelist_tags, "environment=prod | project=myproj", tag=True + mutelist_tags, "environment=prod | project=myproj" ) def test_is_muted_in_tags_with_no_tags_in_finding(self): diff --git a/tests/providers/kubernetes/lib/mutelist/kubernetes_mutelist_test.py b/tests/providers/kubernetes/lib/mutelist/kubernetes_mutelist_test.py index e60516e2ce..5f6c8197dc 100644 --- a/tests/providers/kubernetes/lib/mutelist/kubernetes_mutelist_test.py +++ b/tests/providers/kubernetes/lib/mutelist/kubernetes_mutelist_test.py @@ -67,17 +67,17 @@ class TestKubernetesMutelist: assert mutelist.is_finding_muted(finding, "cluster_1") - def test_is_finding_muted_etcd_star_within_check_name(self): + def test_is_finding_muted_apiserver_star_within_check_name_with_exception(self): # Mutelist mutelist_content = { "Accounts": { "*": { "Checks": { - "etcd_*": { + "apiserver_*": { "Regions": ["*"], "Resources": ["*"], "Exceptions": { - "Accounts": ["k8s-cluster-2"], + "Accounts": ["cluster_1"], "Regions": ["namespace1", "namespace2"], }, } @@ -97,3 +97,34 @@ class TestKubernetesMutelist: finding.resource_tags = [] assert not mutelist.is_finding_muted(finding, "cluster_1") + + def test_is_finding_muted_apiserver_star_within_check_name(self): + # Mutelist + mutelist_content = { + "Accounts": { + "*": { + "Checks": { + "apiserver_*": { + "Regions": ["*"], + "Resources": ["*"], + "Exceptions": { + "Accounts": ["k8s-cluster-1"], + "Regions": ["namespace1", "namespace2"], + }, + } + } + } + } + } + + mutelist = KubernetesMutelist(mutelist_content=mutelist_content) + + finding = MagicMock + finding.check_metadata = MagicMock + finding.check_metadata.CheckID = "apiserver_etcd_cafile_set" + finding.status = "FAIL" + finding.resource_name = "test_resource" + finding.namespace = "namespace1" + finding.resource_tags = [] + + assert mutelist.is_finding_muted(finding, "cluster_1")