From c362d3eea7ff7ef7e517a18871714dcd03f5744d Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Tue, 21 Oct 2025 17:08:23 +0200 Subject: [PATCH] feat(attack-paths): prowler to cartography data model - WIP - ECS WIP Pt. V --- .../backend/tasks/jobs/cartography/aws/ecs.py | 64 ++++++++++++++----- .../backend/tasks/jobs/cartography/aws/s3.py | 11 ++-- prowler/lib/scan/scan.py | 2 - 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/api/src/backend/tasks/jobs/cartography/aws/ecs.py b/api/src/backend/tasks/jobs/cartography/aws/ecs.py index 8a44194012..10bd2d4018 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/ecs.py +++ b/api/src/backend/tasks/jobs/cartography/aws/ecs.py @@ -96,7 +96,7 @@ def _get_ecs_task_definitions_region_metadata( """ with rls_transaction(tenant_id): - tasks_qs = Resource.objects.filter( + task_definitions_qs = Resource.objects.filter( provider_id=provider_id, id__in=ResourceScanSummary.objects.filter( scan_id=scan_id, @@ -106,17 +106,15 @@ def _get_ecs_task_definitions_region_metadata( region__in=regions, ).only("metadata", "inserted_at") - tasks_region_cluster_metadata = defaultdict(defaultdict(list)) - for task in tasks_qs: - task_metadata = json.loads(task.metadata) - task_metadata["inserted_at"] = task.inserted_at - tasks_region_cluster_metadata[ + task_definitions_region_metadata = defaultdict(list) + for task_definition in task_definitions_qs: + task_metadata = json.loads(task_definition.metadata) + task_metadata["inserted_at"] = task_definition.inserted_at + task_definitions_region_metadata[ task_metadata.get("region") - ][ - task_metadata.get("cluster_arn") # TODO: We can't filter the tasks by cluster ARN using Prowler data ].append(task_metadata) - return tasks_region_cluster_metadata + return task_definitions_region_metadata def _sync( @@ -131,46 +129,70 @@ def _sync( Code based on `cartography.intel.aws.ecs.sync`. """ + n_clusters = 0 + n_container_instances = 0 + n_tasks = 0 + n_containers = 0 + n_task_definitions = 0 + n_container_definitions = 0 + n_services = 0 + for region in clusters_region_metadata.keys(): clusters_metadata = clusters_region_metadata.get(region) task_definitions_metadata = task_definitions_region_metadata.get(region, []) - _sync_ecs_clusters( + clusters = _sync_ecs_clusters( neo4j_session, clusters_metadata, region, account_id, update_tag, ) + n_clusters += len(clusters) for cluster_metadata in clusters_metadata: - _sync_ecs_container_instances( + container_instances = _sync_ecs_container_instances( neo4j_session, cluster_metadata, region, account_id, update_tag, ) + n_container_instances += len(container_instances) - _sync_ecs_task_and_container_defns( + tasks, containers, task_definitions, container_definitions = _sync_ecs_task_and_container_defns( neo4j_session, - clusters_metadata, + cluster_metadata, task_definitions_metadata, region, account_id, update_tag, ) + n_tasks += len(tasks) + n_containers += len(containers) + n_task_definitions += len(task_definitions) + n_container_definitions += len(container_definitions) - _sync_ecs_services( + services = _sync_ecs_services( neo4j_session, cluster_metadata, region, account_id, update_tag, ) + n_services += len(services) cartography_ecs.cleanup_ecs(neo4j_session, common_job_parameters) + return { + "cluster": n_clusters, + "container_instances": n_container_instances, + "tasks": n_tasks, + "containers": n_containers, + "task_definitions": n_task_definitions, + "container_definitions": n_container_definitions, + "services": n_services, + } def _sync_ecs_clusters( @@ -215,6 +237,8 @@ def _sync_ecs_clusters( update_tag, ) + return clusters + def _sync_ecs_container_instances( neo4j_session: neo4j.Session, @@ -230,17 +254,19 @@ def _sync_ecs_container_instances( """ cluster_arn = cluster_metadata.get("arn") - containers_instances = [] # TODO + container_instances = [] # TODO cartography_ecs.load_ecs_container_instances( neo4j_session, cluster_arn, - containers_instances, + container_instances, region, account_id, update_tag, ) + return container_instances + def _sync_ecs_task_and_container_defns( neo4j_session: neo4j.Session, @@ -261,7 +287,7 @@ def _sync_ecs_task_and_container_defns( tasks = [] # TODO: Prowler doesn't save ECS tasks data containers = [] # TODO: Prowler doesn't save ECS tasks' containers data - task_definitions = [] # From `tasks` + task_definitions = get_ecs_task_definitions(task_definitions_metadata) container_defs = cartography_ecs._get_container_defs_from_task_definitions(task_definitions) cartography_ecs.load_ecs_tasks( @@ -297,6 +323,8 @@ def _sync_ecs_task_and_container_defns( update_tag, ) + return tasks, containers, task_definitions, container_defs + def get_ecs_task_definitions(task_definitions_metadata: list[dict[str, Any]]) -> list[dict[str, Any]]: """ @@ -385,3 +413,5 @@ def _sync_ecs_services( account_id, update_tag, ) + + return services diff --git a/api/src/backend/tasks/jobs/cartography/aws/s3.py b/api/src/backend/tasks/jobs/cartography/aws/s3.py index 3a5b124862..ac90cca8d1 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/s3.py +++ b/api/src/backend/tasks/jobs/cartography/aws/s3.py @@ -69,7 +69,9 @@ def _get_s3_buckets_metadata( for bucket in buckets_qs: bucket_metadata = json.loads(bucket.metadata) bucket_metadata["inserted_at"] = bucket.inserted_at - buckets_metadata.append(bucket_metadata) + + if bucket_metadata.get("name"): + buckets_metadata.append(bucket_metadata) return buckets_metadata @@ -85,8 +87,6 @@ def _sync( Code based on `cartography.intel.aws.s3.sync`. """ - logger.info("Syncing AWS S3 for account '%s'", account_id) - bucket_list = _get_s3_bucket_list(buckets_metadata) cartography_s3.load_s3_buckets(neo4j_session, bucket_list, account_id, update_tag) cartography_s3.cleanup_s3_buckets(neo4j_session, common_job_parameters) @@ -106,7 +106,10 @@ def _sync( stat_handler=cartography_s3.stat_handler, ) - return {"buckets": len(buckets_metadata), "notifications": len(bucket_notifications)} + return { + "buckets": len(buckets_metadata), + "notifications": len(bucket_notifications), + } def _get_s3_bucket_list(buckets_metadata: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: diff --git a/prowler/lib/scan/scan.py b/prowler/lib/scan/scan.py index d530bc02a3..00d063725d 100644 --- a/prowler/lib/scan/scan.py +++ b/prowler/lib/scan/scan.py @@ -113,8 +113,6 @@ class Scan: if check not in self._bulk_checks_metadata.keys(): raise ScanInvalidCheckError(f"Invalid check provided: {check}.") - # TODO: Delete this - services = ["iam", "s3", "ecs"] # Validate services if services: for service in services: