ben.carrasco/prowler
1
0 Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-10-29 06:44:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(attack-paths): prowler to cartography data model - WIP - ECS WIP Pt. II

Browse Source
This commit is contained in:
Josema Camacho committed 2025-10-17 12:51:29 +02:00
1 parent 2fe4135051
commit 7f6b36c4b1
2 files changed
+154 -49
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

api/src/backend/tasks/jobs/cartography/aws/ecs.py
+143 -21
View File
@@ -1,12 +1,15 @@
from asyncio import tasks
import json
from collections import defaultdict
from typing import Any
from httpx import get
import neo4j
from cartography.intel.aws import ecs as cartography_ecs
from celery.utils.log import get_task_logger
from openai import containers
from api.db_utils import rls_transaction
from api.models import Resource, ResourceScanSummary
@@ -33,12 +36,14 @@ def sync_aws_ecs(
"""
clusters_region_metadata = _get_ecs_clusters_region_metadata(tenant_id, provider_id, scan_id, regions)
tasks_region_cluster_metadata = _get_ecs_tasks_region_cluster_metadata(tenant_id, provider_id, scan_id, regions)
# Calling our version of cartography AWS ECS sync
return _sync(
neo4j_session,
account_id,
clusters_region_metadata,
tasks_region_cluster_metadata,
update_tag,
common_job_parameters,
)
@@ -57,11 +62,10 @@ def _get_ecs_clusters_region_metadata(
with rls_transaction(tenant_id):
clusters_qs = Resource.objects.filter(
provider_id=provider_id,
service="ecs",
type="AwsEcsCluster",
id__in=ResourceScanSummary.objects.filter(
scan_id=scan_id,
service="ecs",
resource_type="AwsEcsCluster",
).values_list("resource_id", flat=True),
region__in=regions,
).only("metadata", "inserted_at")
@@ -75,19 +79,56 @@ def _get_ecs_clusters_region_metadata(
return clusters_region_metadata
def _get_ecs_tasks_region_cluster_metadata(
tenant_id: str,
provider_id: str,
scan_id: str,
regions: list[str],
) -> dict[str, dict[str, list[dict[str, Any]]]]:
"""
Getting ECS tasks metadata from Prowler DB.
# TODO: We can't filter the tasks by cluster ARN using Prowler data
"""
with rls_transaction(tenant_id):
tasks_qs = Resource.objects.filter(
provider_id=provider_id,
id__in=ResourceScanSummary.objects.filter(
scan_id=scan_id,
service="ecs",
resource_type="AwsEcsTaskDefinition",
).values_list("resource_id", flat=True),
region__in=regions,
).only("metadata", "inserted_at")
tasks_region_cluster_metadata = defaultdict(defaultdict(list))
for task in tasks_qs:
task_metadata = json.loads(task.metadata)
task_metadata["inserted_at"] = task.inserted_at
tasks_region_cluster_metadata[
task_metadata.get("region")
][
task_metadata.get("cluster_arn") # TODO: We can't filter the tasks by cluster ARN using Prowler data
].append(task_metadata)
return tasks_region_cluster_metadata
def _sync(
neo4j_session: neo4j.Session,
account_id: str,
clusters_region_metadata: dict[str, list[dict[str, Any]]],
tasks_region_cluster_metadata: dict[str, dict[str, list[dict[str, Any]]]],
update_tag: int,
common_job_parameters: dict[str, Any],
) -> dict[str, Any]:
"""
Code based on `cartography.intel.aws.ecs.sync`.
"""
for region in clusters_region_metadata.keys():
clusters_metadata = clusters_region_metadata[region]
cluster_arns = [cluster.get("arn") for cluster in clusters_metadata]
clusters_metadata = clusters_region_metadata.get(region)
tasks_cluster_metadata = tasks_region_cluster_metadata.get(region, {})
_sync_ecs_clusters(
neo4j_session,
@@ -106,13 +147,17 @@ def _sync(
update_tag,
)
_sync_ecs_task_and_container_defns(
neo4j_session,
cluster_metadata,
region,
account_id,
update_tag,
)
for cluster in tasks_cluster_metadata.keys():
tasks_metadata = tasks_cluster_metadata.get(cluster, [])
_sync_ecs_task_and_container_defns(
neo4j_session,
clusters_metadata,
tasks_metadata,
region,
account_id,
update_tag,
)
_sync_ecs_services(
neo4j_session,
@@ -134,7 +179,9 @@ def _sync_ecs_clusters(
update_tag: int,
) -> None:
"""
Code based on `cartography.intel.aws.ecs._sync_ecs_cluster_arns` and `cartography.intel.aws.ecs.get_ecs_clusters`.
Code based on `cartography.intel.aws.ecs._sync_ecs_cluster_arns` and
`cartography.intel.aws.ecs.get_ecs_clusters`.
# TODO: There are missing fields to implement
"""
clusters = []
@@ -146,8 +193,8 @@ def _sync_ecs_clusters(
# "status" # TODO
# "registeredContainerInstancesCount" # TODO
# "pendingTasksCount" # TODO
"activeServicesCount": len(cluster_metadata.get("services")),
"statistics": [], # TODO
"activeServicesCount": len(cluster_metadata.get("services")), # TODO: Check if this is correct
# "statistics" # TODO
"tags": cluster_metadata.get("tags"),
"settings": cluster_metadata.get("settings"),
"capacityProviders": [
@@ -177,15 +224,16 @@ def _sync_ecs_container_instances(
"""
Code based on `cartography.intel.aws.ecs._sync_ecs_container_instances` and
`cartography.intel.aws.ecs.get_ecs_container_instances`.
# TODO: Container instances data is missing from Prowler DB
"""
cluster_arn = cluster_metadata.get("arn")
cluster_instances = [] # TODO
containers_instances = [] # TODO
cartography_ecs.load_ecs_container_instances(
neo4j_session,
cluster_arn,
cluster_instances,
containers_instances,
region,
account_id,
update_tag,
@@ -195,16 +243,57 @@ def _sync_ecs_container_instances(
def _sync_ecs_task_and_container_defns(
neo4j_session: neo4j.Session,
cluster_metadata: dict[str, Any],
tasks_metadata: dict[str, Any],
region: str,
account_id: str,
update_tag: int,
) -> None:
"""
Code based on `cartography.intel.aws.ecs._sync_ecs_task_and_container_defns` and
# TODO
Code based on `cartography.intel.aws.ecs._sync_ecs_task_and_container_defns`,
`cartography.intel.aws.ecs.get_ecs_tasks`, `cartography.intel.aws.ecs.transform_ecs_tasks`,
`cartography.intel.aws.ecs._get_containers_from_tasks`, `cartography.intel.aws.ecs.get_ecs_task_definitions`
and `cartography.intel.aws.ecs._get_container_defs_from_task_definitions`.
# TODO: Not implemented yet because with Prowler data we can't know the cluster ARN of a task
"""
pass # TODO
cluster_arn = cluster_metadata.get("arn")
tasks = []
containers = [] # From `tasks`
task_definitions = [] # From `tasks`
container_defs = [] # From `task_definitions`
cartography_ecs.load_ecs_tasks(
neo4j_session,
cluster_arn,
tasks,
region,
account_id,
update_tag,
)
cartography_ecs.load_ecs_containers(
neo4j_session,
containers,
region,
account_id,
update_tag,
)
cartography_ecs.load_ecs_task_definitions(
neo4j_session,
task_definitions,
region,
account_id,
update_tag,
)
cartography_ecs.load_ecs_container_definitions(
neo4j_session,
container_defs,
region,
account_id,
update_tag,
)
def _sync_ecs_services(
@@ -216,11 +305,44 @@ def _sync_ecs_services(
) -> None:
"""
Code based on `cartography.intel.aws.ecs._sync_ecs_services` and
# TODO
`cartography.intel.aws.ecs.get_ecs_services`.
# TODO: A lot of fields are missing and probably some of them are too important
"""
cluster_arn = cluster_metadata.get("arn")
services = [] # TODO
services = [
{
"serviceArn": service.get("arn"),
"serviceName": service.get("name"),
"clusterArn": cluster_arn,
# "loadBalancers" # TODO
# "serviceRegistries" # TODO
# "status" # TODO
# "desiredCount": # TODO
# "runningCount": # TODO
"launcType": service.get("launch_type"),
"platformVersion": service.get("platform_version"),
"platformFamily": service.get("platform_family"),
# "taskDefinition" # TODO
# "deploymentConfiguration"
# "deployments"
# "roleArn" # TODO: Important?
# "events"
"createdAt": cluster_metadata.get("inserted_at"),
# "placementConstraints" # TODO
# "placementStrategy" # TODO
# "networkConfiguration" # TODO
# "healthCheckGracePeriodSeconds" # TODO
# "schedulingStrategy" # TODO
# "deploymentController" # TODO
# "createdBy" # TODO: Important?
# "enableECSManagedTags" # TODO
# "propagateTags" # TODO
# "enableExecuteCommand" # TODO
# "availabilityZoneRebalancing" # TODO
}
for service in cluster_metadata.get("services").values()
]
cartography_ecs.load_ecs_services(
neo4j_session,
api/src/backend/tasks/jobs/cartography/aws/s3.py
+11 -28
View File
@@ -57,11 +57,10 @@ def _get_s3_buckets_metadata(
with rls_transaction(tenant_id):
buckets_qs = Resource.objects.filter(
provider_id=provider_id,
service="s3",
type="AwsS3Bucket",
id__in=ResourceScanSummary.objects.filter(
scan_id=scan_id,
service="s3",
resource_type="AwsS3Bucket",
).values_list("resource_id", flat=True),
region__in=regions,
).only("metadata", "inserted_at")
@@ -134,9 +133,10 @@ def _get_and_load_s3_bucket_details(
) -> None:
"""
Code based on `cartography.intel.aws.s3.get_s3_bucket_details` and `cartography.intel.aws.s3.load_s3_details`.
TODO: Versions of the next functions are not implemented yet:
- `cartography.intel.aws.s3.parse_acl`
- `cartography.intel.aws.s3.parse_policy` and `cartography.intel.aws.s3.parse_policy_statements` as a single function, if not two # noqa: E501
# TODO: Versions of the next functions are not implemented yet:
- `cartography.intel.aws.s3.parse_acl`.
- `cartography.intel.aws.s3.parse_policy` and
`cartography.intel.aws.s3.parse_policy_statements` as a single function, if not two.
"""
acls: list[dict[str, Any]] = [] # TODO
@@ -190,7 +190,7 @@ def _get_and_load_s3_bucket_details(
def _parse_s3_bucket_encryption(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None:
"""
Code based on `cartography.intel.aws.s3.parse_encryption`.
TODO: Keys `encryption_key_id` and `bucket_key_enabled` are implemented yet.
# TODO: Keys `encryption_key_id` and `bucket_key_enabled` are implemented yet.
"""
if not bucket_metadata.get("encryption"):
@@ -200,8 +200,8 @@ def _parse_s3_bucket_encryption(bucket_metadata: dict[str, Any]) -> dict[str, An
"bucket": bucket_metadata.get("name"),
"default_encryption": True,
"encryption_algorithm": bucket_metadata.get("encryption"), # ServerSideEncryptionConfiguration.Rules[-1].ApplyServerSideEncryptionByDefault.SSEAlgorithm # noqa: E501
"encryption_key_id": None, # ServerSideEncryptionConfiguration.Rules[-1].ApplyServerSideEncryptionByDefault.KMSMasterKeyID # TODO # noqa: E501
"bucket_key_enabled": None, # ServerSideEncryptionConfiguration.Rules[-1].BucketKeyEnabled # TODO # noqa: E501
# "encryption_key_id" # TODO: ServerSideEncryptionConfiguration.Rules[-1].ApplyServerSideEncryptionByDefault.KMSMasterKeyID # noqa: E501
# "bucket_key_enabled" # TODO: ServerSideEncryptionConfiguration.Rules[-1].BucketKeyEnabled
}
@@ -227,6 +227,7 @@ def _parse_s3_bucket_public_access_block(bucket_metadata: dict[str, Any]) -> dic
**bucket_metadata.get("public_access_block"),
}
def _parse_s3_bucket_ownership_controls(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None:
"""
Code based on `cartography.intel.aws.s3.parse_bucket_ownership_controls`.
@@ -237,6 +238,7 @@ def _parse_s3_bucket_ownership_controls(bucket_metadata: dict[str, Any]) -> dict
"object_ownership": bucket_metadata.get("ownership"),
}
def _parse_s3_bucket_bucket_logging(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None:
"""
Code based on `cartography.intel.aws.s3.parse_bucket_logging`.
@@ -249,29 +251,10 @@ def _parse_s3_bucket_bucket_logging(bucket_metadata: dict[str, Any]) -> dict[str
}
def _load_s3_details(neo4j_session: neo4j.Session, update_tag: int, aws_account_id: str) -> None:
cartography_s3.run_cleanup_job(
"aws_s3_details.json",
neo4j_session,
{"UPDATE_TAG": update_tag, "AWS_ID": aws_account_id},
)
cartography_s3._load_s3_acls(neo4j_session, acls, aws_account_id, update_tag)
cartography_s3._load_s3_policies(neo4j_session, policies, update_tag)
cartography_s3._load_s3_policy_statements(neo4j_session, statements, update_tag)
cartography_s3._load_s3_encryption(neo4j_session, encryption_configs, update_tag)
cartography_s3._load_s3_versioning(neo4j_session, versioning_configs, update_tag)
cartography_s3._load_s3_public_access_block(neo4j_session, public_access_block_configs, update_tag)
cartography_s3._load_bucket_ownership_controls(neo4j_session, bucket_ownership_controls_configs, update_tag)
cartography_s3._load_bucket_logging(neo4j_session, bucket_logging_configs, update_tag)
cartography_s3._set_default_values(neo4j_session, aws_account_id)
# TODO: Deeper check
def _parse_s3_notifications(buckets_metadata: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""
Code based on `cartography.intel.aws.s3.parse_notification_configuration`.
# TODO: Do a deeper check of this function
"""
notifications: list[dict[str, Any]] = []