feat(aws): support the ISO partitions for region resolution and scanning (#12759)

Co-authored-by: pedrooot <pedromarting3@gmail.com>

.github/workflows/sdk-refresh-aws-services-regions.yml

@@ -44,7 +44,10 @@ jobs:
           cache: 'pip'
 
       - name: Install dependencies
-        run: pip install boto3
+        # Pinned to the versions in pyproject.toml: the ISO partitions region
+        # data comes from the endpoints.json bundled with botocore, so the
+        # botocore version is itself a data source and must be deterministic
+        run: pip install boto3==1.40.61 botocore==1.40.61
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1

prowler/changelog.d/aws-iso-partitions.added.md

@@ -0,0 +1 @@
+AWS ISO partitions (`aws-iso`, `aws-iso-b`, `aws-iso-e` and `aws-iso-f`) to the AWS service region matrix, generated from the endpoints data bundled with botocore

prowler/changelog.d/aws-iso-partitions.fixed.md

@@ -0,0 +1 @@
+`AwsProvider.get_global_region()` now returns a real region for each ISO partition instead of the `aws-iso-global` pseudo endpoint, which collapsed the four partitions into one answer

prowler/changelog.d/aws-regional-clients-empty-dict.fixed.md

@@ -0,0 +1 @@
+`AwsProvider.generate_regional_clients()` now returns an empty dict instead of `None` when the regional clients cannot be built, a failure that surfaced later as `AttributeError: 'NoneType' object has no attribute 'values'`

prowler/changelog.d/aws-service-regions-unknown-partition.fixed.md

@@ -0,0 +1 @@
+`AwsProvider.get_available_aws_service_regions()` now returns an empty set for an unknown service or partition instead of raising `KeyError`, so a service unavailable in the audited partition is skipped

prowler/providers/aws/aws_provider.py

@@ -921,6 +921,9 @@ class AwsProvider(Provider):
             logger.error(
                 f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
             )
+            # Return an empty dict, as promised by the signature, so the service
+            # is simply not scanned instead of the callers failing later on a None
+            return {}
 
     @staticmethod
     def get_available_aws_service_regions(
@@ -936,9 +939,13 @@ class AwsProvider(Provider):
 
         Returns:
             - A set of strings representing the available regions for the given service and partition.
+              A service or a partition not present in the regions file yields an empty set, the same
+              outcome as a service explicitly recorded as unavailable in the partition.
         """
         data = read_aws_regions_file()
-        json_regions = set(data["services"][service]["regions"][partition])
+        json_regions = set(
+            data["services"].get(service, {}).get("regions", {}).get(partition, [])
+        )
         if audited_regions:
             # Get common regions between input and json
             regions = json_regions.intersection(audited_regions)
@@ -1145,16 +1152,14 @@ class AwsProvider(Provider):
         Example:
             global_region = get_global_region()a
         """
-        global_region = "us-east-1"
-        if self._identity.partition == "aws-cn":
-            global_region = "cn-north-1"
-        elif self._identity.partition == "aws-eusc":
-            global_region = "eusc-de-east-1"
-        elif self._identity.partition == "aws-us-gov":
-            global_region = "us-gov-east-1"
-        elif "aws-iso" in self._identity.partition:
-            global_region = "aws-iso-global"
-        return global_region
+        # The first region of the partition is the one of its global STS endpoint,
+        # which is always a real region, never a pseudo endpoint like "aws-iso-global"
+        partition_regions = get_botocore_partition_regions().get(
+            self._identity.partition
+        )
+        if partition_regions:
+            return partition_regions[0]
+        return "us-east-1"
 
     @staticmethod
     def input_role_mfa_token_and_code() -> AWSMFAInfo:

tests/providers/aws/aws_provider_test.py

@@ -59,6 +59,7 @@ from tests.providers.aws.utils import (
     AWS_EUSC_PARTITION,
     AWS_GOV_CLOUD_ACCOUNT_ARN,
     AWS_GOV_CLOUD_PARTITION,
+    AWS_ISO_B_PARTITION,
     AWS_ISO_PARTITION,
     AWS_REGION_CN_NORTH_1,
     AWS_REGION_CN_NORTHWEST_1,
@@ -67,7 +68,9 @@ from tests.providers.aws.utils import (
     AWS_REGION_EUSC_DE_EAST_1,
     AWS_REGION_GOV_CLOUD_US_EAST_1,
     AWS_REGION_GOV_CLOUD_US_WEST_1,
-    AWS_REGION_ISO_GLOBAL,
+    AWS_REGION_ISO_B_EAST_1,
+    AWS_REGION_ISO_EAST_1,
+    AWS_REGION_ISO_WEST_1,
     AWS_REGION_US_EAST_1,
     AWS_REGION_US_EAST_2,
     EXAMPLE_AMI_ID,
@@ -1192,6 +1195,13 @@ aws:
             == AWS_REGION_EU_WEST_1
         )
 
+    @mock_aws
+    def test_aws_get_global_region(self):
+        aws_provider = AwsProvider()
+        aws_provider._identity.partition = AWS_COMMERCIAL_PARTITION
+
+        assert aws_provider.get_global_region() == AWS_REGION_US_EAST_1
+
     @mock_aws
     def test_aws_gov_get_global_region(self):
         aws_provider = AwsProvider()
@@ -1211,7 +1221,21 @@ aws:
         aws_provider = AwsProvider()
         aws_provider._identity.partition = AWS_ISO_PARTITION
 
-        assert aws_provider.get_global_region() == AWS_REGION_ISO_GLOBAL
+        assert aws_provider.get_global_region() == AWS_REGION_ISO_EAST_1
+
+    @mock_aws
+    def test_aws_iso_b_get_global_region(self):
+        aws_provider = AwsProvider()
+        aws_provider._identity.partition = AWS_ISO_B_PARTITION
+
+        assert aws_provider.get_global_region() == AWS_REGION_ISO_B_EAST_1
+
+    @mock_aws
+    def test_get_global_region_for_an_unknown_partition(self):
+        aws_provider = AwsProvider()
+        aws_provider._identity.partition = "aws-unknown"
+
+        assert aws_provider.get_global_region() == AWS_REGION_US_EAST_1
 
     @mock_aws
     def test_aws_eusc_get_global_region(self):
@@ -1299,6 +1323,88 @@ aws:
                 len(aws_provider.get_available_aws_service_regions("ec2", "aws")) == 17
             )
 
+    @mock_aws
+    def test_get_available_aws_service_regions_commercial_and_gov_cloud(self):
+        aws_provider = AwsProvider()
+
+        assert AWS_REGION_US_EAST_1 in aws_provider.get_available_aws_service_regions(
+            "ec2", AWS_COMMERCIAL_PARTITION
+        )
+        assert (
+            AWS_REGION_GOV_CLOUD_US_EAST_1
+            in aws_provider.get_available_aws_service_regions(
+                "ec2", AWS_GOV_CLOUD_PARTITION
+            )
+        )
+        # A service recorded as unavailable in the partition yields an empty set
+        assert (
+            aws_provider.get_available_aws_service_regions(
+                "bedrock-agent", AWS_CHINA_PARTITION
+            )
+            == set()
+        )
+
+    @mock_aws
+    def test_get_available_aws_service_regions_iso_partitions(self):
+        aws_provider = AwsProvider()
+
+        assert aws_provider.get_available_aws_service_regions(
+            "ec2", AWS_ISO_PARTITION
+        ) == {
+            AWS_REGION_ISO_EAST_1,
+            AWS_REGION_ISO_WEST_1,
+        }
+        assert aws_provider.get_available_aws_service_regions(
+            "guardduty", AWS_ISO_B_PARTITION
+        ) == {AWS_REGION_ISO_B_EAST_1}
+        # Every service carries every ISO partition, empty when not available
+        assert (
+            aws_provider.get_available_aws_service_regions(
+                "bedrock", AWS_ISO_B_PARTITION
+            )
+            == set()
+        )
+
+    @mock_aws
+    def test_get_available_aws_service_regions_unknown_partition(self):
+        aws_provider = AwsProvider()
+
+        assert (
+            aws_provider.get_available_aws_service_regions("ec2", "aws-unknown")
+            == set()
+        )
+
+    @mock_aws
+    def test_get_available_aws_service_regions_unknown_service(self):
+        aws_provider = AwsProvider()
+
+        assert (
+            aws_provider.get_available_aws_service_regions(
+                "unknown-service", AWS_COMMERCIAL_PARTITION
+            )
+            == set()
+        )
+
+    @mock_aws
+    def test_generate_regional_clients_service_not_in_partition(self):
+        aws_provider = AwsProvider()
+        aws_provider._identity.partition = AWS_ISO_PARTITION
+
+        response = aws_provider.generate_regional_clients("bedrock")
+
+        assert response == {}
+
+    @mock_aws
+    def test_generate_regional_clients_returns_empty_dict_on_error(self):
+        aws_provider = AwsProvider()
+
+        with patch.object(
+            AwsProvider,
+            "get_available_aws_service_regions",
+            side_effect=Exception("boom"),
+        ):
+            assert aws_provider.generate_regional_clients("ec2") == {}
+
     @mock_aws
     def test_get_tagged_resources(self):
         ec2_client = client("ec2", region_name=AWS_REGION_EU_CENTRAL_1)
@@ -2065,7 +2171,8 @@ aws:
         assert not recovered_regions
 
     def test_get_regions_all_count(self):
-        assert len(AwsProvider.get_regions(partition=None)) == 39
+        # 34 aws + 2 aws-cn + 2 aws-us-gov + 1 aws-eusc + 7 ISO regions
+        assert len(AwsProvider.get_regions(partition=None)) == 46
 
     def test_get_regions_cn_count(self):
         assert len(AwsProvider.get_regions("aws-cn")) == 2
@@ -2073,6 +2180,12 @@ aws:
     def test_get_regions_aws_count(self):
         assert len(AwsProvider.get_regions(partition="aws")) == 34
 
+    def test_get_regions_iso_count(self):
+        assert AwsProvider.get_regions(AWS_ISO_PARTITION) == {
+            AWS_REGION_ISO_EAST_1,
+            AWS_REGION_ISO_WEST_1,
+        }
+
     def test_get_all_regions(self):
         with patch(
             "prowler.providers.aws.aws_provider.read_aws_regions_file",

tests/providers/aws/utils.py

@@ -21,6 +21,7 @@ AWS_GOV_CLOUD_PARTITION = "aws-us-gov"
 AWS_CHINA_PARTITION = "aws-cn"
 AWS_EUSC_PARTITION = "aws-eusc"
 AWS_ISO_PARTITION = "aws-iso"
+AWS_ISO_B_PARTITION = "aws-iso-b"
 
 # Root AWS Account
 AWS_ACCOUNT_NUMBER = "123456789012"
@@ -54,7 +55,9 @@ AWS_REGION_GOV_CLOUD_US_EAST_1 = "us-gov-east-1"
 AWS_REGION_GOV_CLOUD_US_WEST_1 = "us-gov-west-1"
 
 # Iso Regions
-AWS_REGION_ISO_GLOBAL = "aws-iso-global"
+AWS_REGION_ISO_EAST_1 = "us-iso-east-1"
+AWS_REGION_ISO_WEST_1 = "us-iso-west-1"
+AWS_REGION_ISO_B_EAST_1 = "us-isob-east-1"
 
 # European Sovereign Cloud Regions
 AWS_REGION_EUSC_DE_EAST_1 = "eusc-de-east-1"

util/update_aws_services_regions.py

@@ -4,6 +4,7 @@ import os
 import sys
 
 import boto3
+from botocore.session import Session as BotocoreSession
 
 # Logging config
 logging.basicConfig(
@@ -13,6 +14,39 @@ logging.basicConfig(
     level=logging.INFO,
 )
 
+# AWS partitions that the SSM global-infrastructure parameters do not publish.
+# Their availability comes from the endpoints.json bundled with botocore, which
+# is offline data and needs neither credentials nor network access.
+ISO_PARTITIONS = ("aws-iso", "aws-iso-b", "aws-iso-e", "aws-iso-f")
+
+# Cost Explorer: botocore keys it by its endpoint prefix "ce", while the matrix
+# (and the boto3 client name) calls it "costexplorer". Explicit rename override,
+# since no boto3 service model resolves the "ce" prefix.
+ISO_ENDPOINT_PREFIX_RENAMES = {"ce": "costexplorer"}
+
+# "transcribestreaming" is the streaming endpoint of Amazon Transcribe. The
+# "transcribe" prefix is already present in the same partitions with the same
+# regions, so mapping it would only duplicate data. Ignoring it is a deliberate
+# decision, not a resolution failure.
+ISO_IGNORED_ENDPOINT_PREFIXES = {"transcribestreaming"}
+
+# A service whose only endpoint in a partition is the partition-wide pseudo
+# endpoint (for example "aws-iso-global") gets every region of that partition,
+# matching how the matrix already records iam, organizations, route53 and
+# support for aws and aws-us-gov. Cost Explorer is the exception: the matrix
+# records it as a single-region service (aws: us-east-1, aws-cn: cn-northwest-1),
+# so it only gets the region declared in the endpoint's credentialScope.
+ISO_SINGLE_REGION_PARTITION_GLOBAL_SERVICES = {"costexplorer"}
+
+
+def get_regions_by_service_from_ssm() -> dict:
+    """Get the AWS services and their regions for the partitions published in
+    the SSM global-infrastructure parameters: aws, aws-cn, aws-eusc and
+    aws-us-gov.
+
+    Returns:
+        dict: The AWS regions matrix, keyed by service name.
+    """
     regions_by_service = {"services": {}}
 
     logging.info("Recovering AWS Regions by Service")
@@ -25,7 +59,16 @@ for page in get_parameters_by_path_paginator.paginate(
         for service in page["Parameters"]:
             regions_by_service["services"][service["Value"]] = {}
             # Get all AWS Regions for the specific service
-        regions = {"aws": [], "aws-cn": [], "aws-eusc": [], "aws-us-gov": []}
+            regions = {
+                "aws": [],
+                "aws-cn": [],
+                "aws-eusc": [],
+                "aws-us-gov": [],
+                "aws-iso": [],
+                "aws-iso-b": [],
+                "aws-iso-e": [],
+                "aws-iso-f": [],
+            }
             for page in get_parameters_by_path_paginator.paginate(
                 Path="/aws/service/global-infrastructure/services/"
                 + service["Value"]
@@ -47,8 +90,15 @@ for page in get_parameters_by_path_paginator.paginate(
                     regions["aws-us-gov"] = sorted(regions["aws-us-gov"])
             regions_by_service["services"][service["Value"]]["regions"] = regions
 
-# Include the regions for the subservices and the services not present
-logging.info("Updating subservices and the services not present in the original matrix")
+    return regions_by_service
+
+
+def add_subservices_and_missing_services(regions_by_service: dict) -> None:
+    """Include the regions for the subservices and the services not present in
+    the original matrix."""
+    logging.info(
+        "Updating subservices and the services not present in the original matrix"
+    )
     # macie2 --> macie
     regions_by_service["services"]["macie2"] = regions_by_service["services"]["macie"]
     # bedrock-agent is not in SSM, and has different availability than bedrock
@@ -95,9 +145,155 @@ regions_by_service["services"]["wellarchitected"] = regions_by_service["services
     # sesv2 --> ses
     regions_by_service["services"]["sesv2"] = regions_by_service["services"]["ses"]
 
-# Write to file
-parsed_matrix_regions_aws = f"{os.path.dirname(os.path.realpath(__name__))}/prowler/providers/aws/aws_regions_by_service.json"
+
+def get_endpoint_prefix_to_services() -> dict:
+    """Map every botocore endpoint prefix to the set of boto3 service (client)
+    names using it.
+
+    botocore's endpoints.json keys services by endpoint prefix, while the matrix
+    keys them by the boto3/SSM service name. The mapping is derived from the SDK
+    itself instead of being hand-written, so it stays correct as the SDK evolves
+    (monitoring -> cloudwatch, elasticloadbalancing -> elb and elbv2, states ->
+    stepfunctions, api.ecr -> ecr, ...).
+
+    Returns:
+        dict: A dictionary mapping each endpoint prefix to a set of service names.
+    """
+    session = BotocoreSession()
+    endpoint_prefix_to_services = {}
+    for service_name in session.get_available_services():
+        endpoint_prefix = session.get_service_model(service_name).endpoint_prefix
+        endpoint_prefix_to_services.setdefault(endpoint_prefix, set()).add(service_name)
+    return endpoint_prefix_to_services
+
+
+def resolve_matrix_services(
+    endpoint_prefix: str, endpoint_prefix_to_services: dict, services: dict
+) -> set:
+    """Resolve a botocore endpoint prefix to the matrix service names it stands
+    for.
+
+    Args:
+        - endpoint_prefix: The botocore endpoint prefix.
+        - endpoint_prefix_to_services: The map returned by get_endpoint_prefix_to_services.
+        - services: The services of the AWS regions matrix.
+
+    Returns:
+        set: The matrix service names, empty when the prefix does not resolve.
+    """
+    renamed_service = ISO_ENDPOINT_PREFIX_RENAMES.get(endpoint_prefix)
+    if renamed_service:
+        return {renamed_service} & set(services)
+
+    service_names = endpoint_prefix_to_services.get(endpoint_prefix, set()) & set(
+        services
+    )
+    if not service_names and endpoint_prefix in services:
+        service_names = {endpoint_prefix}
+    return service_names
+
+
+def get_partition_global_service_regions(
+    service_names: set, service_data: dict, partition_regions: list
+) -> list:
+    """Get the regions of a service whose only endpoint in the partition is the
+    partition-wide pseudo endpoint (for example "aws-iso-global"), which is not
+    a region and must never be recorded as one.
+
+    Returns:
+        list: Every region of the partition, or only the credentialScope region
+            for the services the matrix records as single-region ones.
+    """
+    partition_endpoint = service_data.get("partitionEndpoint")
+    credential_scope_region = (
+        service_data.get("endpoints", {})
+        .get(partition_endpoint, {})
+        .get("credentialScope", {})
+        .get("region")
+    )
+    if service_names & ISO_SINGLE_REGION_PARTITION_GLOBAL_SERVICES:
+        if credential_scope_region in partition_regions:
+            return [credential_scope_region]
+        return []
+    return list(partition_regions)
+
+
+def add_iso_partitions_regions(regions_by_service: dict) -> None:
+    """Fill the aws-iso, aws-iso-b, aws-iso-e and aws-iso-f regions of every
+    service from the endpoints.json bundled with botocore.
+
+    It runs after the subservices and the services not present in the original
+    matrix have been added, so it sees the final set of services: the aliases
+    sharing a single dict and the hand-written bedrock-agent entry all get their
+    ISO partition keys.
+
+    Raises:
+        ValueError: If an endpoint prefix present in an ISO partition does not
+            resolve to a matrix service and is not explicitly ignored.
+    """
+    logging.info("Updating the ISO partitions regions from the botocore endpoints")
+    services = regions_by_service["services"]
+    endpoints_data = BotocoreSession().get_data("endpoints")
+    endpoint_prefix_to_services = get_endpoint_prefix_to_services()
+
+    # Every service carries every partition key, so the matrix stays rectangular
+    # even for the services with no presence at all in the ISO partitions.
+    for service in services.values():
+        for partition in ISO_PARTITIONS:
+            service["regions"].setdefault(partition, [])
+
+    for partition_data in endpoints_data["partitions"]:
+        partition = partition_data["partition"]
+        if partition not in ISO_PARTITIONS:
+            continue
+        partition_regions = sorted(partition_data.get("regions", {}))
+        for endpoint_prefix, service_data in partition_data.get("services", {}).items():
+            if endpoint_prefix in ISO_IGNORED_ENDPOINT_PREFIXES:
+                continue
+            service_names = resolve_matrix_services(
+                endpoint_prefix, endpoint_prefix_to_services, services
+            )
+            if not service_names:
+                raise ValueError(
+                    f"The botocore endpoint prefix '{endpoint_prefix}', present in the "
+                    f"'{partition}' partition, does not resolve to any service of the "
+                    "AWS regions matrix. Dropping it silently would leave the service "
+                    "out of the scans, so either add the prefix to "
+                    "ISO_ENDPOINT_PREFIX_RENAMES with the matrix service name it "
+                    "corresponds to, or add it to ISO_IGNORED_ENDPOINT_PREFIXES if it "
+                    "must not be mapped."
+                )
+            # Keep only the endpoints that are real regions of the partition,
+            # which drops the fips-* and the partition-wide pseudo endpoints.
+            regions = sorted(
+                set(service_data.get("endpoints", {})) & set(partition_regions)
+            )
+            if not regions:
+                regions = get_partition_global_service_regions(
+                    service_names, service_data, partition_regions
+                )
+            for service_name in service_names:
+                services[service_name]["regions"][partition] = list(regions)
+
+
+def write_regions_by_service(regions_by_service: dict) -> None:
+    """Write the AWS regions matrix to the file read by the AWS provider."""
+    repository_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    parsed_matrix_regions_aws = (
+        f"{repository_root}/prowler/providers/aws/aws_regions_by_service.json"
+    )
     logging.info(f"Writing {parsed_matrix_regions_aws}")
     with open(parsed_matrix_regions_aws, "w") as outfile:
         json.dump(regions_by_service, outfile, indent=2, sort_keys=True)
         outfile.write("\n")
+
+
+def main() -> None:
+    regions_by_service = get_regions_by_service_from_ssm()
+    add_subservices_and_missing_services(regions_by_service)
+    add_iso_partitions_regions(regions_by_service)
+    write_regions_by_service(regions_by_service)
+
+
+if __name__ == "__main__":
+    main()