mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-10-04 02:04:06 +00:00
feat(aws): support the ISO partitions for region resolution and scanning (#12759)
Co-authored-by: pedrooot <pedromarting3@gmail.com>
This commit is contained in:
StylusFrost
@@ -4,6 +4,7 @@ import os
|
||||
import sys
|
||||
|
||||
import boto3
|
||||
from botocore.session import Session as BotocoreSession
|
||||
|
||||
# Logging config
|
||||
logging.basicConfig(
|
||||
@@ -13,91 +14,286 @@ logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
)
|
||||
|
||||
regions_by_service = {"services": {}}
|
||||
# AWS partitions that the SSM global-infrastructure parameters do not publish.
|
||||
# Their availability comes from the endpoints.json bundled with botocore, which
|
||||
# is offline data and needs neither credentials nor network access.
|
||||
ISO_PARTITIONS = ("aws-iso", "aws-iso-b", "aws-iso-e", "aws-iso-f")
|
||||
|
||||
logging.info("Recovering AWS Regions by Service")
|
||||
client = boto3.client("ssm", region_name="us-east-1")
|
||||
get_parameters_by_path_paginator = client.get_paginator("get_parameters_by_path")
|
||||
# Get all AWS Available Services
|
||||
for page in get_parameters_by_path_paginator.paginate(
|
||||
Path="/aws/service/global-infrastructure/services"
|
||||
):
|
||||
for service in page["Parameters"]:
|
||||
regions_by_service["services"][service["Value"]] = {}
|
||||
# Get all AWS Regions for the specific service
|
||||
regions = {"aws": [], "aws-cn": [], "aws-eusc": [], "aws-us-gov": []}
|
||||
for page in get_parameters_by_path_paginator.paginate(
|
||||
Path="/aws/service/global-infrastructure/services/"
|
||||
+ service["Value"]
|
||||
+ "/regions"
|
||||
):
|
||||
for region in page["Parameters"]:
|
||||
if "cn" in region["Value"]:
|
||||
regions["aws-cn"].append(region["Value"])
|
||||
elif "eusc" in region["Value"]:
|
||||
regions["aws-eusc"].append(region["Value"])
|
||||
elif "gov" in region["Value"]:
|
||||
regions["aws-us-gov"].append(region["Value"])
|
||||
else:
|
||||
regions["aws"].append(region["Value"])
|
||||
# Sort regions per partition
|
||||
regions["aws"] = sorted(regions["aws"])
|
||||
regions["aws-cn"] = sorted(regions["aws-cn"])
|
||||
regions["aws-eusc"] = sorted(regions["aws-eusc"])
|
||||
regions["aws-us-gov"] = sorted(regions["aws-us-gov"])
|
||||
regions_by_service["services"][service["Value"]]["regions"] = regions
|
||||
# Cost Explorer: botocore keys it by its endpoint prefix "ce", while the matrix
|
||||
# (and the boto3 client name) calls it "costexplorer". Explicit rename override,
|
||||
# since no boto3 service model resolves the "ce" prefix.
|
||||
ISO_ENDPOINT_PREFIX_RENAMES = {"ce": "costexplorer"}
|
||||
|
||||
# Include the regions for the subservices and the services not present
|
||||
logging.info("Updating subservices and the services not present in the original matrix")
|
||||
# macie2 --> macie
|
||||
regions_by_service["services"]["macie2"] = regions_by_service["services"]["macie"]
|
||||
# bedrock-agent is not in SSM, and has different availability than bedrock
|
||||
# See: https://docs.aws.amazon.com/bedrock/latest/userguide/agents-supported.html
|
||||
regions_by_service["services"]["bedrock-agent"] = {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-south-1",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ca-central-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-west-2",
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-us-gov": [
|
||||
"us-gov-west-1",
|
||||
],
|
||||
# "transcribestreaming" is the streaming endpoint of Amazon Transcribe. The
|
||||
# "transcribe" prefix is already present in the same partitions with the same
|
||||
# regions, so mapping it would only duplicate data. Ignoring it is a deliberate
|
||||
# decision, not a resolution failure.
|
||||
ISO_IGNORED_ENDPOINT_PREFIXES = {"transcribestreaming"}
|
||||
|
||||
# A service whose only endpoint in a partition is the partition-wide pseudo
|
||||
# endpoint (for example "aws-iso-global") gets every region of that partition,
|
||||
# matching how the matrix already records iam, organizations, route53 and
|
||||
# support for aws and aws-us-gov. Cost Explorer is the exception: the matrix
|
||||
# records it as a single-region service (aws: us-east-1, aws-cn: cn-northwest-1),
|
||||
# so it only gets the region declared in the endpoint's credentialScope.
|
||||
ISO_SINGLE_REGION_PARTITION_GLOBAL_SERVICES = {"costexplorer"}
|
||||
|
||||
|
||||
def get_regions_by_service_from_ssm() -> dict:
|
||||
"""Get the AWS services and their regions for the partitions published in
|
||||
the SSM global-infrastructure parameters: aws, aws-cn, aws-eusc and
|
||||
aws-us-gov.
|
||||
|
||||
Returns:
|
||||
dict: The AWS regions matrix, keyed by service name.
|
||||
"""
|
||||
regions_by_service = {"services": {}}
|
||||
|
||||
logging.info("Recovering AWS Regions by Service")
|
||||
client = boto3.client("ssm", region_name="us-east-1")
|
||||
get_parameters_by_path_paginator = client.get_paginator("get_parameters_by_path")
|
||||
# Get all AWS Available Services
|
||||
for page in get_parameters_by_path_paginator.paginate(
|
||||
Path="/aws/service/global-infrastructure/services"
|
||||
):
|
||||
for service in page["Parameters"]:
|
||||
regions_by_service["services"][service["Value"]] = {}
|
||||
# Get all AWS Regions for the specific service
|
||||
regions = {
|
||||
"aws": [],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-us-gov": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
}
|
||||
for page in get_parameters_by_path_paginator.paginate(
|
||||
Path="/aws/service/global-infrastructure/services/"
|
||||
+ service["Value"]
|
||||
+ "/regions"
|
||||
):
|
||||
for region in page["Parameters"]:
|
||||
if "cn" in region["Value"]:
|
||||
regions["aws-cn"].append(region["Value"])
|
||||
elif "eusc" in region["Value"]:
|
||||
regions["aws-eusc"].append(region["Value"])
|
||||
elif "gov" in region["Value"]:
|
||||
regions["aws-us-gov"].append(region["Value"])
|
||||
else:
|
||||
regions["aws"].append(region["Value"])
|
||||
# Sort regions per partition
|
||||
regions["aws"] = sorted(regions["aws"])
|
||||
regions["aws-cn"] = sorted(regions["aws-cn"])
|
||||
regions["aws-eusc"] = sorted(regions["aws-eusc"])
|
||||
regions["aws-us-gov"] = sorted(regions["aws-us-gov"])
|
||||
regions_by_service["services"][service["Value"]]["regions"] = regions
|
||||
|
||||
return regions_by_service
|
||||
|
||||
|
||||
def add_subservices_and_missing_services(regions_by_service: dict) -> None:
|
||||
"""Include the regions for the subservices and the services not present in
|
||||
the original matrix."""
|
||||
logging.info(
|
||||
"Updating subservices and the services not present in the original matrix"
|
||||
)
|
||||
# macie2 --> macie
|
||||
regions_by_service["services"]["macie2"] = regions_by_service["services"]["macie"]
|
||||
# bedrock-agent is not in SSM, and has different availability than bedrock
|
||||
# See: https://docs.aws.amazon.com/bedrock/latest/userguide/agents-supported.html
|
||||
regions_by_service["services"]["bedrock-agent"] = {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-south-1",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ca-central-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-west-2",
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-us-gov": [
|
||||
"us-gov-west-1",
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
# cognito --> cognito-idp
|
||||
regions_by_service["services"]["cognito"] = regions_by_service["services"][
|
||||
"cognito-idp"
|
||||
]
|
||||
# opensearch --> es
|
||||
regions_by_service["services"]["opensearch"] = regions_by_service["services"]["es"]
|
||||
# elbv2 --> elb
|
||||
regions_by_service["services"]["elbv2"] = regions_by_service["services"]["elb"]
|
||||
# wafv2 --> waf
|
||||
regions_by_service["services"]["wafv2"] = regions_by_service["services"]["waf"]
|
||||
# wellarchitected --> wellarchitectedtool
|
||||
regions_by_service["services"]["wellarchitected"] = regions_by_service["services"][
|
||||
"wellarchitectedtool"
|
||||
]
|
||||
# sesv2 --> ses
|
||||
regions_by_service["services"]["sesv2"] = regions_by_service["services"]["ses"]
|
||||
# cognito --> cognito-idp
|
||||
regions_by_service["services"]["cognito"] = regions_by_service["services"][
|
||||
"cognito-idp"
|
||||
]
|
||||
# opensearch --> es
|
||||
regions_by_service["services"]["opensearch"] = regions_by_service["services"]["es"]
|
||||
# elbv2 --> elb
|
||||
regions_by_service["services"]["elbv2"] = regions_by_service["services"]["elb"]
|
||||
# wafv2 --> waf
|
||||
regions_by_service["services"]["wafv2"] = regions_by_service["services"]["waf"]
|
||||
# wellarchitected --> wellarchitectedtool
|
||||
regions_by_service["services"]["wellarchitected"] = regions_by_service["services"][
|
||||
"wellarchitectedtool"
|
||||
]
|
||||
# sesv2 --> ses
|
||||
regions_by_service["services"]["sesv2"] = regions_by_service["services"]["ses"]
|
||||
|
||||
# Write to file
|
||||
parsed_matrix_regions_aws = f"{os.path.dirname(os.path.realpath(__name__))}/prowler/providers/aws/aws_regions_by_service.json"
|
||||
logging.info(f"Writing {parsed_matrix_regions_aws}")
|
||||
with open(parsed_matrix_regions_aws, "w") as outfile:
|
||||
json.dump(regions_by_service, outfile, indent=2, sort_keys=True)
|
||||
outfile.write("\n")
|
||||
|
||||
def get_endpoint_prefix_to_services() -> dict:
|
||||
"""Map every botocore endpoint prefix to the set of boto3 service (client)
|
||||
names using it.
|
||||
|
||||
botocore's endpoints.json keys services by endpoint prefix, while the matrix
|
||||
keys them by the boto3/SSM service name. The mapping is derived from the SDK
|
||||
itself instead of being hand-written, so it stays correct as the SDK evolves
|
||||
(monitoring -> cloudwatch, elasticloadbalancing -> elb and elbv2, states ->
|
||||
stepfunctions, api.ecr -> ecr, ...).
|
||||
|
||||
Returns:
|
||||
dict: A dictionary mapping each endpoint prefix to a set of service names.
|
||||
"""
|
||||
session = BotocoreSession()
|
||||
endpoint_prefix_to_services = {}
|
||||
for service_name in session.get_available_services():
|
||||
endpoint_prefix = session.get_service_model(service_name).endpoint_prefix
|
||||
endpoint_prefix_to_services.setdefault(endpoint_prefix, set()).add(service_name)
|
||||
return endpoint_prefix_to_services
|
||||
|
||||
|
||||
def resolve_matrix_services(
|
||||
endpoint_prefix: str, endpoint_prefix_to_services: dict, services: dict
|
||||
) -> set:
|
||||
"""Resolve a botocore endpoint prefix to the matrix service names it stands
|
||||
for.
|
||||
|
||||
Args:
|
||||
- endpoint_prefix: The botocore endpoint prefix.
|
||||
- endpoint_prefix_to_services: The map returned by get_endpoint_prefix_to_services.
|
||||
- services: The services of the AWS regions matrix.
|
||||
|
||||
Returns:
|
||||
set: The matrix service names, empty when the prefix does not resolve.
|
||||
"""
|
||||
renamed_service = ISO_ENDPOINT_PREFIX_RENAMES.get(endpoint_prefix)
|
||||
if renamed_service:
|
||||
return {renamed_service} & set(services)
|
||||
|
||||
service_names = endpoint_prefix_to_services.get(endpoint_prefix, set()) & set(
|
||||
services
|
||||
)
|
||||
if not service_names and endpoint_prefix in services:
|
||||
service_names = {endpoint_prefix}
|
||||
return service_names
|
||||
|
||||
|
||||
def get_partition_global_service_regions(
|
||||
service_names: set, service_data: dict, partition_regions: list
|
||||
) -> list:
|
||||
"""Get the regions of a service whose only endpoint in the partition is the
|
||||
partition-wide pseudo endpoint (for example "aws-iso-global"), which is not
|
||||
a region and must never be recorded as one.
|
||||
|
||||
Returns:
|
||||
list: Every region of the partition, or only the credentialScope region
|
||||
for the services the matrix records as single-region ones.
|
||||
"""
|
||||
partition_endpoint = service_data.get("partitionEndpoint")
|
||||
credential_scope_region = (
|
||||
service_data.get("endpoints", {})
|
||||
.get(partition_endpoint, {})
|
||||
.get("credentialScope", {})
|
||||
.get("region")
|
||||
)
|
||||
if service_names & ISO_SINGLE_REGION_PARTITION_GLOBAL_SERVICES:
|
||||
if credential_scope_region in partition_regions:
|
||||
return [credential_scope_region]
|
||||
return []
|
||||
return list(partition_regions)
|
||||
|
||||
|
||||
def add_iso_partitions_regions(regions_by_service: dict) -> None:
|
||||
"""Fill the aws-iso, aws-iso-b, aws-iso-e and aws-iso-f regions of every
|
||||
service from the endpoints.json bundled with botocore.
|
||||
|
||||
It runs after the subservices and the services not present in the original
|
||||
matrix have been added, so it sees the final set of services: the aliases
|
||||
sharing a single dict and the hand-written bedrock-agent entry all get their
|
||||
ISO partition keys.
|
||||
|
||||
Raises:
|
||||
ValueError: If an endpoint prefix present in an ISO partition does not
|
||||
resolve to a matrix service and is not explicitly ignored.
|
||||
"""
|
||||
logging.info("Updating the ISO partitions regions from the botocore endpoints")
|
||||
services = regions_by_service["services"]
|
||||
endpoints_data = BotocoreSession().get_data("endpoints")
|
||||
endpoint_prefix_to_services = get_endpoint_prefix_to_services()
|
||||
|
||||
# Every service carries every partition key, so the matrix stays rectangular
|
||||
# even for the services with no presence at all in the ISO partitions.
|
||||
for service in services.values():
|
||||
for partition in ISO_PARTITIONS:
|
||||
service["regions"].setdefault(partition, [])
|
||||
|
||||
for partition_data in endpoints_data["partitions"]:
|
||||
partition = partition_data["partition"]
|
||||
if partition not in ISO_PARTITIONS:
|
||||
continue
|
||||
partition_regions = sorted(partition_data.get("regions", {}))
|
||||
for endpoint_prefix, service_data in partition_data.get("services", {}).items():
|
||||
if endpoint_prefix in ISO_IGNORED_ENDPOINT_PREFIXES:
|
||||
continue
|
||||
service_names = resolve_matrix_services(
|
||||
endpoint_prefix, endpoint_prefix_to_services, services
|
||||
)
|
||||
if not service_names:
|
||||
raise ValueError(
|
||||
f"The botocore endpoint prefix '{endpoint_prefix}', present in the "
|
||||
f"'{partition}' partition, does not resolve to any service of the "
|
||||
"AWS regions matrix. Dropping it silently would leave the service "
|
||||
"out of the scans, so either add the prefix to "
|
||||
"ISO_ENDPOINT_PREFIX_RENAMES with the matrix service name it "
|
||||
"corresponds to, or add it to ISO_IGNORED_ENDPOINT_PREFIXES if it "
|
||||
"must not be mapped."
|
||||
)
|
||||
# Keep only the endpoints that are real regions of the partition,
|
||||
# which drops the fips-* and the partition-wide pseudo endpoints.
|
||||
regions = sorted(
|
||||
set(service_data.get("endpoints", {})) & set(partition_regions)
|
||||
)
|
||||
if not regions:
|
||||
regions = get_partition_global_service_regions(
|
||||
service_names, service_data, partition_regions
|
||||
)
|
||||
for service_name in service_names:
|
||||
services[service_name]["regions"][partition] = list(regions)
|
||||
|
||||
|
||||
def write_regions_by_service(regions_by_service: dict) -> None:
|
||||
"""Write the AWS regions matrix to the file read by the AWS provider."""
|
||||
repository_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
parsed_matrix_regions_aws = (
|
||||
f"{repository_root}/prowler/providers/aws/aws_regions_by_service.json"
|
||||
)
|
||||
logging.info(f"Writing {parsed_matrix_regions_aws}")
|
||||
with open(parsed_matrix_regions_aws, "w") as outfile:
|
||||
json.dump(regions_by_service, outfile, indent=2, sort_keys=True)
|
||||
outfile.write("\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
regions_by_service = get_regions_by_service_from_ssm()
|
||||
add_subservices_and_missing_services(regions_by_service)
|
||||
add_iso_partitions_regions(regions_by_service)
|
||||
write_regions_by_service(regions_by_service)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user