From 7f6b36c4b1397e61e92ba326b3852381969217be Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 17 Oct 2025 12:51:29 +0200 Subject: [PATCH] feat(attack-paths): prowler to cartography data model - WIP - ECS WIP Pt. II --- .../backend/tasks/jobs/cartography/aws/ecs.py | 164 +++++++++++++++--- .../backend/tasks/jobs/cartography/aws/s3.py | 39 ++--- 2 files changed, 154 insertions(+), 49 deletions(-) diff --git a/api/src/backend/tasks/jobs/cartography/aws/ecs.py b/api/src/backend/tasks/jobs/cartography/aws/ecs.py index 9f397f3b79..d284e4db50 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/ecs.py +++ b/api/src/backend/tasks/jobs/cartography/aws/ecs.py @@ -1,12 +1,15 @@ +from asyncio import tasks import json from collections import defaultdict from typing import Any +from httpx import get import neo4j from cartography.intel.aws import ecs as cartography_ecs from celery.utils.log import get_task_logger +from openai import containers from api.db_utils import rls_transaction from api.models import Resource, ResourceScanSummary @@ -33,12 +36,14 @@ def sync_aws_ecs( """ clusters_region_metadata = _get_ecs_clusters_region_metadata(tenant_id, provider_id, scan_id, regions) + tasks_region_cluster_metadata = _get_ecs_tasks_region_cluster_metadata(tenant_id, provider_id, scan_id, regions) # Calling our version of cartography AWS ECS sync return _sync( neo4j_session, account_id, clusters_region_metadata, + tasks_region_cluster_metadata, update_tag, common_job_parameters, ) @@ -57,11 +62,10 @@ def _get_ecs_clusters_region_metadata( with rls_transaction(tenant_id): clusters_qs = Resource.objects.filter( provider_id=provider_id, - service="ecs", - type="AwsEcsCluster", id__in=ResourceScanSummary.objects.filter( scan_id=scan_id, service="ecs", + resource_type="AwsEcsCluster", ).values_list("resource_id", flat=True), region__in=regions, ).only("metadata", "inserted_at") @@ -75,19 +79,56 @@ def _get_ecs_clusters_region_metadata( return clusters_region_metadata +def _get_ecs_tasks_region_cluster_metadata( + tenant_id: str, + provider_id: str, + scan_id: str, + regions: list[str], +) -> dict[str, dict[str, list[dict[str, Any]]]]: + """ + Getting ECS tasks metadata from Prowler DB. + # TODO: We can't filter the tasks by cluster ARN using Prowler data + """ + + with rls_transaction(tenant_id): + tasks_qs = Resource.objects.filter( + provider_id=provider_id, + id__in=ResourceScanSummary.objects.filter( + scan_id=scan_id, + service="ecs", + resource_type="AwsEcsTaskDefinition", + ).values_list("resource_id", flat=True), + region__in=regions, + ).only("metadata", "inserted_at") + + tasks_region_cluster_metadata = defaultdict(defaultdict(list)) + for task in tasks_qs: + task_metadata = json.loads(task.metadata) + task_metadata["inserted_at"] = task.inserted_at + tasks_region_cluster_metadata[ + task_metadata.get("region") + ][ + task_metadata.get("cluster_arn") # TODO: We can't filter the tasks by cluster ARN using Prowler data + ].append(task_metadata) + + return tasks_region_cluster_metadata + + def _sync( neo4j_session: neo4j.Session, account_id: str, clusters_region_metadata: dict[str, list[dict[str, Any]]], + tasks_region_cluster_metadata: dict[str, dict[str, list[dict[str, Any]]]], update_tag: int, common_job_parameters: dict[str, Any], ) -> dict[str, Any]: """ Code based on `cartography.intel.aws.ecs.sync`. """ + for region in clusters_region_metadata.keys(): - clusters_metadata = clusters_region_metadata[region] - cluster_arns = [cluster.get("arn") for cluster in clusters_metadata] + clusters_metadata = clusters_region_metadata.get(region) + tasks_cluster_metadata = tasks_region_cluster_metadata.get(region, {}) _sync_ecs_clusters( neo4j_session, @@ -106,13 +147,17 @@ def _sync( update_tag, ) - _sync_ecs_task_and_container_defns( - neo4j_session, - cluster_metadata, - region, - account_id, - update_tag, - ) + for cluster in tasks_cluster_metadata.keys(): + tasks_metadata = tasks_cluster_metadata.get(cluster, []) + + _sync_ecs_task_and_container_defns( + neo4j_session, + clusters_metadata, + tasks_metadata, + region, + account_id, + update_tag, + ) _sync_ecs_services( neo4j_session, @@ -134,7 +179,9 @@ def _sync_ecs_clusters( update_tag: int, ) -> None: """ - Code based on `cartography.intel.aws.ecs._sync_ecs_cluster_arns` and `cartography.intel.aws.ecs.get_ecs_clusters`. + Code based on `cartography.intel.aws.ecs._sync_ecs_cluster_arns` and + `cartography.intel.aws.ecs.get_ecs_clusters`. + # TODO: There are missing fields to implement """ clusters = [] @@ -146,8 +193,8 @@ def _sync_ecs_clusters( # "status" # TODO # "registeredContainerInstancesCount" # TODO # "pendingTasksCount" # TODO - "activeServicesCount": len(cluster_metadata.get("services")), - "statistics": [], # TODO + "activeServicesCount": len(cluster_metadata.get("services")), # TODO: Check if this is correct + # "statistics" # TODO "tags": cluster_metadata.get("tags"), "settings": cluster_metadata.get("settings"), "capacityProviders": [ @@ -177,15 +224,16 @@ def _sync_ecs_container_instances( """ Code based on `cartography.intel.aws.ecs._sync_ecs_container_instances` and `cartography.intel.aws.ecs.get_ecs_container_instances`. + # TODO: Container instances data is missing from Prowler DB """ cluster_arn = cluster_metadata.get("arn") - cluster_instances = [] # TODO + containers_instances = [] # TODO cartography_ecs.load_ecs_container_instances( neo4j_session, cluster_arn, - cluster_instances, + containers_instances, region, account_id, update_tag, @@ -195,16 +243,57 @@ def _sync_ecs_container_instances( def _sync_ecs_task_and_container_defns( neo4j_session: neo4j.Session, cluster_metadata: dict[str, Any], + tasks_metadata: dict[str, Any], region: str, account_id: str, update_tag: int, ) -> None: """ - Code based on `cartography.intel.aws.ecs._sync_ecs_task_and_container_defns` and - # TODO + Code based on `cartography.intel.aws.ecs._sync_ecs_task_and_container_defns`, + `cartography.intel.aws.ecs.get_ecs_tasks`, `cartography.intel.aws.ecs.transform_ecs_tasks`, + `cartography.intel.aws.ecs._get_containers_from_tasks`, `cartography.intel.aws.ecs.get_ecs_task_definitions` + and `cartography.intel.aws.ecs._get_container_defs_from_task_definitions`. + # TODO: Not implemented yet because with Prowler data we can't know the cluster ARN of a task """ - pass # TODO + cluster_arn = cluster_metadata.get("arn") + tasks = [] + containers = [] # From `tasks` + task_definitions = [] # From `tasks` + container_defs = [] # From `task_definitions` + + cartography_ecs.load_ecs_tasks( + neo4j_session, + cluster_arn, + tasks, + region, + account_id, + update_tag, + ) + + cartography_ecs.load_ecs_containers( + neo4j_session, + containers, + region, + account_id, + update_tag, + ) + + cartography_ecs.load_ecs_task_definitions( + neo4j_session, + task_definitions, + region, + account_id, + update_tag, + ) + + cartography_ecs.load_ecs_container_definitions( + neo4j_session, + container_defs, + region, + account_id, + update_tag, + ) def _sync_ecs_services( @@ -216,11 +305,44 @@ def _sync_ecs_services( ) -> None: """ Code based on `cartography.intel.aws.ecs._sync_ecs_services` and - # TODO + `cartography.intel.aws.ecs.get_ecs_services`. + # TODO: A lot of fields are missing and probably some of them are too important """ cluster_arn = cluster_metadata.get("arn") - services = [] # TODO + services = [ + { + "serviceArn": service.get("arn"), + "serviceName": service.get("name"), + "clusterArn": cluster_arn, + # "loadBalancers" # TODO + # "serviceRegistries" # TODO + # "status" # TODO + # "desiredCount": # TODO + # "runningCount": # TODO + "launcType": service.get("launch_type"), + "platformVersion": service.get("platform_version"), + "platformFamily": service.get("platform_family"), + # "taskDefinition" # TODO + # "deploymentConfiguration" + # "deployments" + # "roleArn" # TODO: Important? + # "events" + "createdAt": cluster_metadata.get("inserted_at"), + # "placementConstraints" # TODO + # "placementStrategy" # TODO + # "networkConfiguration" # TODO + # "healthCheckGracePeriodSeconds" # TODO + # "schedulingStrategy" # TODO + # "deploymentController" # TODO + # "createdBy" # TODO: Important? + # "enableECSManagedTags" # TODO + # "propagateTags" # TODO + # "enableExecuteCommand" # TODO + # "availabilityZoneRebalancing" # TODO + } + for service in cluster_metadata.get("services").values() + ] cartography_ecs.load_ecs_services( neo4j_session, diff --git a/api/src/backend/tasks/jobs/cartography/aws/s3.py b/api/src/backend/tasks/jobs/cartography/aws/s3.py index 73fabd2645..3a5b124862 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/s3.py +++ b/api/src/backend/tasks/jobs/cartography/aws/s3.py @@ -57,11 +57,10 @@ def _get_s3_buckets_metadata( with rls_transaction(tenant_id): buckets_qs = Resource.objects.filter( provider_id=provider_id, - service="s3", - type="AwsS3Bucket", id__in=ResourceScanSummary.objects.filter( scan_id=scan_id, service="s3", + resource_type="AwsS3Bucket", ).values_list("resource_id", flat=True), region__in=regions, ).only("metadata", "inserted_at") @@ -134,9 +133,10 @@ def _get_and_load_s3_bucket_details( ) -> None: """ Code based on `cartography.intel.aws.s3.get_s3_bucket_details` and `cartography.intel.aws.s3.load_s3_details`. - TODO: Versions of the next functions are not implemented yet: - - `cartography.intel.aws.s3.parse_acl` - - `cartography.intel.aws.s3.parse_policy` and `cartography.intel.aws.s3.parse_policy_statements` as a single function, if not two # noqa: E501 + # TODO: Versions of the next functions are not implemented yet: + - `cartography.intel.aws.s3.parse_acl`. + - `cartography.intel.aws.s3.parse_policy` and + `cartography.intel.aws.s3.parse_policy_statements` as a single function, if not two. """ acls: list[dict[str, Any]] = [] # TODO @@ -190,7 +190,7 @@ def _get_and_load_s3_bucket_details( def _parse_s3_bucket_encryption(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None: """ Code based on `cartography.intel.aws.s3.parse_encryption`. - TODO: Keys `encryption_key_id` and `bucket_key_enabled` are implemented yet. + # TODO: Keys `encryption_key_id` and `bucket_key_enabled` are implemented yet. """ if not bucket_metadata.get("encryption"): @@ -200,8 +200,8 @@ def _parse_s3_bucket_encryption(bucket_metadata: dict[str, Any]) -> dict[str, An "bucket": bucket_metadata.get("name"), "default_encryption": True, "encryption_algorithm": bucket_metadata.get("encryption"), # ServerSideEncryptionConfiguration.Rules[-1].ApplyServerSideEncryptionByDefault.SSEAlgorithm # noqa: E501 - "encryption_key_id": None, # ServerSideEncryptionConfiguration.Rules[-1].ApplyServerSideEncryptionByDefault.KMSMasterKeyID # TODO # noqa: E501 - "bucket_key_enabled": None, # ServerSideEncryptionConfiguration.Rules[-1].BucketKeyEnabled # TODO # noqa: E501 + # "encryption_key_id" # TODO: ServerSideEncryptionConfiguration.Rules[-1].ApplyServerSideEncryptionByDefault.KMSMasterKeyID # noqa: E501 + # "bucket_key_enabled" # TODO: ServerSideEncryptionConfiguration.Rules[-1].BucketKeyEnabled } @@ -227,6 +227,7 @@ def _parse_s3_bucket_public_access_block(bucket_metadata: dict[str, Any]) -> dic **bucket_metadata.get("public_access_block"), } + def _parse_s3_bucket_ownership_controls(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None: """ Code based on `cartography.intel.aws.s3.parse_bucket_ownership_controls`. @@ -237,6 +238,7 @@ def _parse_s3_bucket_ownership_controls(bucket_metadata: dict[str, Any]) -> dict "object_ownership": bucket_metadata.get("ownership"), } + def _parse_s3_bucket_bucket_logging(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None: """ Code based on `cartography.intel.aws.s3.parse_bucket_logging`. @@ -249,29 +251,10 @@ def _parse_s3_bucket_bucket_logging(bucket_metadata: dict[str, Any]) -> dict[str } -def _load_s3_details(neo4j_session: neo4j.Session, update_tag: int, aws_account_id: str) -> None: - cartography_s3.run_cleanup_job( - "aws_s3_details.json", - neo4j_session, - {"UPDATE_TAG": update_tag, "AWS_ID": aws_account_id}, - ) - - cartography_s3._load_s3_acls(neo4j_session, acls, aws_account_id, update_tag) - cartography_s3._load_s3_policies(neo4j_session, policies, update_tag) - cartography_s3._load_s3_policy_statements(neo4j_session, statements, update_tag) - cartography_s3._load_s3_encryption(neo4j_session, encryption_configs, update_tag) - cartography_s3._load_s3_versioning(neo4j_session, versioning_configs, update_tag) - cartography_s3._load_s3_public_access_block(neo4j_session, public_access_block_configs, update_tag) - cartography_s3._load_bucket_ownership_controls(neo4j_session, bucket_ownership_controls_configs, update_tag) - cartography_s3._load_bucket_logging(neo4j_session, bucket_logging_configs, update_tag) - - cartography_s3._set_default_values(neo4j_session, aws_account_id) - - -# TODO: Deeper check def _parse_s3_notifications(buckets_metadata: list[dict[str, Any]]) -> list[dict[str, Any]]: """ Code based on `cartography.intel.aws.s3.parse_notification_configuration`. + # TODO: Do a deeper check of this function """ notifications: list[dict[str, Any]] = []