From 03c1e22345be5534f922e1475f0417f2f239b1ac Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 24 Oct 2025 13:55:24 +0200 Subject: [PATCH] feat(attack-paths): prowler to cartography data model - WIP - Pt. ... --- .../tasks/jobs/cartography/__init__.py | 4 +- .../tasks/jobs/cartography/aws/__init__.py | 34 ++++++-- .../tasks/jobs/cartography/aws/analysis.py | 82 +++++++++++++++++++ .../backend/tasks/jobs/cartography/aws/ecs.py | 1 - .../backend/tasks/jobs/cartography/aws/iam.py | 27 ++++-- .../backend/tasks/jobs/cartography/aws/s3.py | 47 +++++++---- 6 files changed, 160 insertions(+), 35 deletions(-) create mode 100644 api/src/backend/tasks/jobs/cartography/aws/analysis.py diff --git a/api/src/backend/tasks/jobs/cartography/__init__.py b/api/src/backend/tasks/jobs/cartography/__init__.py index 9c2d14137a..627c83d010 100644 --- a/api/src/backend/tasks/jobs/cartography/__init__.py +++ b/api/src/backend/tasks/jobs/cartography/__init__.py @@ -1,5 +1,5 @@ from operator import ne -from cartography.intel import create_indexes as cartography_indexes + from celery.utils.log import get_task_logger from neo4j import GraphDatabase @@ -23,8 +23,6 @@ def cartography_sync_scan( with GraphDatabase.driver("bolt://neo4j:7687", auth=("neo4j", "neo4j_password")) as driver: with driver.session() as neo4j_session: - cartography_indexes.run(neo4j_session, None) - return sync_aws( # TODO: Depending on the provider type use the appropriate sync function tenant_id=tenant_id, provider_id=provider_id, diff --git a/api/src/backend/tasks/jobs/cartography/aws/__init__.py b/api/src/backend/tasks/jobs/cartography/aws/__init__.py index aeb51ca4fc..efbec49220 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/__init__.py +++ b/api/src/backend/tasks/jobs/cartography/aws/__init__.py @@ -3,13 +3,14 @@ from typing import Any import neo4j -from cartography.intel import aws as cartography_aws - from api.db_utils import rls_transaction from api.models import Provider, ResourceScanSummary -from tasks.jobs.cartography.aws.s3 import sync_aws_s3 +from cartography.intel import create_indexes as cartography_indexes +from cartography.intel.aws import organizations as cartography_organizations +from tasks.jobs.cartography.aws.analysis import perform_aws_analysis from tasks.jobs.cartography.aws.ecs import sync_aws_ecs from tasks.jobs.cartography.aws.iam import sync_aws_iam +from tasks.jobs.cartography.aws.s3 import sync_aws_s3 def sync_aws( @@ -23,28 +24,45 @@ def sync_aws( """ # Getting data from DB - account_id = get_aws_provider_account_id(tenant_id, provider_id) regions = get_aws_provider_regions(tenant_id, scan_id) + provider = get_aws_provider(tenant_id, provider_id) + account_id = provider.uid + provider_alias = provider.alias # Configuring Cartography job parameters update_tag = int(datetime.now(tz=timezone.utc).timestamp() * 1000) common_job_parameters = {"UPDATE_TAG": update_tag, "AWS_ID": account_id} - return { + # Creating Cartography indexes + cartography_indexes.run(neo4j_session, None) + + # Syncing AWS Account + accounts = {provider_alias: account_id} + cartography_organizations.load_aws_accounts(neo4j_session, accounts, update_tag, common_job_parameters) + + # Syncing AWS resources + result = { "iam": sync_aws_iam(tenant_id, provider_id, account_id, scan_id, regions, neo4j_session, update_tag, common_job_parameters), # noqa: E501 "s3": sync_aws_s3(tenant_id, provider_id, account_id, scan_id, regions, neo4j_session, update_tag, common_job_parameters), # noqa: E501 "ecs": sync_aws_ecs(tenant_id, provider_id, account_id, scan_id, regions, neo4j_session, update_tag, common_job_parameters), # noqa: E501 } -def get_aws_provider_account_id(tenant_id: str, provider_id: str) -> str: + # Running AWS analysis + syncs = result.keys() + perform_aws_analysis(account_id, syncs, regions, neo4j_session, update_tag, common_job_parameters) + + return result + + +def get_aws_provider(tenant_id: str, provider_id: str) -> str: """ - Getting AWS account ID from Prowler DB for a provider. + Getting AWS provider from Prowler DB. """ with rls_transaction(tenant_id): provider = Provider.objects.get(pk=provider_id) - return provider.uid + return provider def get_aws_provider_regions(tenant_id: str, scan_id: str) -> list[str]: diff --git a/api/src/backend/tasks/jobs/cartography/aws/analysis.py b/api/src/backend/tasks/jobs/cartography/aws/analysis.py new file mode 100644 index 0000000000..87271b049b --- /dev/null +++ b/api/src/backend/tasks/jobs/cartography/aws/analysis.py @@ -0,0 +1,82 @@ +import neo4j + +from typing import Any + +from cartography.intel import aws as cartography_aws +# from cartography.intel.aws import permission_relationships as cartography_permission_relationships +# from cartography.intel.aws import resourcegroupstaggingapi as cartography_resourcegroupstaggingapi +from celery.utils.log import get_task_logger + +# TODO: Do the rigth logging setup +# logger = get_task_logger(__name__) +import logging +from config.custom_logging import BackendLogger +logger = logging.getLogger(BackendLogger.API) + + +def perform_aws_analysis( + account_id: str, + syncs: list[str], + regions: list[str], + neo4j_session: neo4j.Session, + update_tag: int, + common_job_parameters: dict[str, Any], +): + """ + Code based on `cartography.intel.aws._sync_multiple_accounts` and `cartography.intel.aws._sync_one_account`. + # TODO: Check if we need to run `permission_relationships.sync` + # TODO: Prowler DB doesn't save `resourcegroupstaggingapi` + """ + + # TODO: Check if we need to run `permission_relationships.sync` and with what `permission_relationships_file` + # cartography_permission_relationships.sync( + # neo4j_session, + # None, # `boto3_session` is not needed here + # regions, + # account_id, + # update_tag, + # common_job_parameters, + # ) + + # TODO: As `boto3_session` is needed for + # `boto3_session.client("resourcegroupstaggingapi", region_name=region) + # we can't call this function + # cartography_resourcegroupstaggingapi.sync( + # neo4j_session + # None, # `boto3_session` is REALLY needed here + # regions, + # account_id, + # update_tag, + # common_job_parameters, + # ) + + cartography_aws.run_scoped_analysis_job( + "aws_ec2_iaminstanceprofile.json", + neo4j_session, + common_job_parameters, + ) + + cartography_aws.run_analysis_job( + "aws_lambda_ecr.json", + neo4j_session, + common_job_parameters, + ) + + cartography_aws.merge_module_sync_metadata( + neo4j_session, + group_type="AWSAccount", + group_id=account_id, + synced_type="AWSAccount", + update_tag=update_tag, + stat_handler=cartography_aws.stat_handler, + ) + + cartography_aws.run_cleanup_job( + "aws_post_ingestion_principals_cleanup.json", + neo4j_session, + common_job_parameters, + ) + + stringified_syncs = ",".join(syncs) + requested_syncs = cartography_aws.parse_and_validate_aws_requested_syncs(stringified_syncs) + cartography_aws._perform_aws_analysis(requested_syncs, neo4j_session, common_job_parameters) diff --git a/api/src/backend/tasks/jobs/cartography/aws/ecs.py b/api/src/backend/tasks/jobs/cartography/aws/ecs.py index e5d5092c20..18677adecc 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/ecs.py +++ b/api/src/backend/tasks/jobs/cartography/aws/ecs.py @@ -7,7 +7,6 @@ import neo4j from cartography.intel.aws import ecs as cartography_ecs from celery.utils.log import get_task_logger -from openai import containers from api.db_utils import rls_transaction from api.models import Resource, ResourceScanSummary diff --git a/api/src/backend/tasks/jobs/cartography/aws/iam.py b/api/src/backend/tasks/jobs/cartography/aws/iam.py index 32818f302c..4c681ec5b6 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/iam.py +++ b/api/src/backend/tasks/jobs/cartography/aws/iam.py @@ -1,13 +1,11 @@ import json -from collections import defaultdict from typing import Any import neo4j from cartography.intel.aws import iam as cartography_iam from celery.utils.log import get_task_logger -from openai import containers from api.db_utils import rls_transaction from api.models import Resource, ResourceScanSummary @@ -66,7 +64,7 @@ def _sync( update_tag, ) - _sync_users( + users = _sync_users( tenant_id, provider_id, account_id, @@ -76,7 +74,7 @@ def _sync( update_tag, ) - _sync_groups( + groups = _sync_groups( tenant_id, provider_id, account_id, @@ -86,7 +84,7 @@ def _sync( update_tag, ) - _sync_roles( + roles = _sync_roles( tenant_id, provider_id, account_id, @@ -103,7 +101,7 @@ def _sync( common_job_parameters, ) - _sync_user_access_keys( + access_keys = _sync_user_access_keys( tenant_id, provider_id, account_id, @@ -125,7 +123,12 @@ def _sync( stat_handler=cartography_iam.stat_handler, ) - return {} + return { + "users": len(users), + "groups": len(groups), + "roles": len(roles), + "access_keys": len(access_keys), + } def _sync_users( tenant_id: str, @@ -144,6 +147,8 @@ def _sync_users( _sync_inline_policies(user_data["Users"], neo4j_session, update_tag, account_id) _sync_managed_policies(user_data["Users"], neo4j_session, update_tag, account_id) + return user_data["Users"] + def _get_user_list_data( tenant_id: str, @@ -312,6 +317,8 @@ def _sync_groups( _sync_managed_policies(group_data["Groups"], neo4j_session, update_tag, account_id) + return group_data["Groups"] + def _get_group_list_data( tenant_id: str, @@ -409,6 +416,8 @@ def _sync_roles( _sync_managed_policies(roles_data["Roles"], neo4j_session, update_tag, account_id) + return roles_data["Roles"] + def _get_role_list_data( tenant_id: str, @@ -491,9 +500,11 @@ def _sync_user_access_keys( user_data = _get_user_list_data(tenant_id, provider_id, scan_id, regions) user_access_keys = _pretransform_access_keys(user_data["Users"]) access_key_data = cartography_iam.transform_access_keys(user_access_keys) + cartography_iam.load_access_keys( neo4j_session, access_key_data, update_tag, account_id ) + cartography_iam.GraphJob.from_node_schema( cartography_iam.AccountAccessKeySchema(), common_job_parameters, @@ -501,6 +512,8 @@ def _sync_user_access_keys( neo4j_session, ) + return access_key_data + def _pretransform_access_keys(users: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: """ diff --git a/api/src/backend/tasks/jobs/cartography/aws/s3.py b/api/src/backend/tasks/jobs/cartography/aws/s3.py index 93140a44bd..7861576215 100644 --- a/api/src/backend/tasks/jobs/cartography/aws/s3.py +++ b/api/src/backend/tasks/jobs/cartography/aws/s3.py @@ -136,15 +136,11 @@ def _get_and_load_s3_bucket_details( ) -> None: """ Code based on `cartography.intel.aws.s3.get_s3_bucket_details` and `cartography.intel.aws.s3.load_s3_details`. - # TODO: Versions of the next functions are not implemented yet: - - `cartography.intel.aws.s3.parse_acl`. - - `cartography.intel.aws.s3.parse_policy` and - `cartography.intel.aws.s3.parse_policy_statements` as a single function, if not two. """ - acls: list[dict[str, Any]] = [] # TODO - policies: list[dict[str, Any]] = [] # TODO - statements: list[dict[str, Any]] = [] # TODO + acls: list[dict[str, Any]] = [] + policies: list[dict[str, Any]] = [] + statements: list[dict[str, Any]] = [] encryption_configs: list[dict[str, Any]] = [] versioning_configs: list[dict[str, Any]] = [] public_access_block_configs: list[dict[str, Any]] = [] @@ -152,19 +148,17 @@ def _get_and_load_s3_bucket_details( bucket_logging_configs: list[dict[str, Any]] = [] for bucket_metadata in buckets_metadata: - parsed_acls = None # TODO - parsed_policy = None # TODO - parsed_statements = None # TODO - parsed_acls = _parse_s3_bucket_acl(bucket_metadata, account_id) if parsed_acls is not None: - acls.append(parsed_acls) + acls.extend(parsed_acls) parsed_policy = _parse_s3_bucket_policy(bucket_metadata) if parsed_policy is not None: policies.append(parsed_policy) - # TODO: More work + parsed_statements = _parse_s3_bucket_policy_statements(bucket_metadata) + if parsed_statements is not None: + statements.extend(parsed_statements) parsed_encryption = _parse_s3_bucket_encryption(bucket_metadata) if parsed_encryption is not None: @@ -229,11 +223,33 @@ def _parse_s3_bucket_acl(bucket_metadata: dict[str, Any], account_id: str) -> di "Permission": grantee.get("permission"), }) - return cartography_s3.parse_acl(bucket_metadata.get("name"), bucket_metadata.get("acl_grantees"), account_id) + return cartography_s3.parse_acl(acl, bucket_metadata.get("name"), account_id) def _parse_s3_bucket_policy(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None: - return None # TODO + """ + Code based on `cartography.intel.aws.s3.parse_policy`. + """ + if not bucket_metadata.get("policy"): + return None + + policy = { + "Policy": json.dumps(bucket_metadata.get("policy")), + } + return cartography_s3.parse_policy(bucket_metadata.get("name"), policy) + + +def _parse_s3_bucket_policy_statements(bucket_metadata: dict[str, Any]) -> list[dict[str, Any]] | None: + """ + Code based on `cartography.intel.aws.s3.parse_policy_statements`. + """ + if not bucket_metadata.get("policy"): + return None + + policy = { + "Policy": json.dumps(bucket_metadata.get("policy")), + } + return cartography_s3.parse_policy_statements(bucket_metadata.get("name"), policy) def _parse_s3_bucket_encryption(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None: @@ -303,7 +319,6 @@ def _parse_s3_bucket_bucket_logging(bucket_metadata: dict[str, Any]) -> dict[str def _parse_s3_notifications(buckets_metadata: list[dict[str, Any]]) -> list[dict[str, Any]]: """ Code based on `cartography.intel.aws.s3.parse_notification_configuration`. - # TODO: Do a deeper check of this function """ notifications: list[dict[str, Any]] = []