feat(attack-paths): prowler to cartography data model - WIP - Pt. ...

api/src/backend/tasks/jobs/cartography/__init__.py

@@ -1,5 +1,5 @@
 from operator import ne
-from cartography.intel import create_indexes as cartography_indexes
+
 from celery.utils.log import get_task_logger
 from neo4j import GraphDatabase
 
@@ -23,8 +23,6 @@ def cartography_sync_scan(
     with GraphDatabase.driver("bolt://neo4j:7687", auth=("neo4j", "neo4j_password")) as driver:
         with driver.session() as neo4j_session:
 
-            cartography_indexes.run(neo4j_session, None)
-
             return sync_aws(  # TODO: Depending on the provider type use the appropriate sync function
                 tenant_id=tenant_id,
                 provider_id=provider_id,

api/src/backend/tasks/jobs/cartography/aws/__init__.py

@@ -3,13 +3,14 @@ from typing import Any
 
 import neo4j
 
-from cartography.intel import aws as cartography_aws
-
 from api.db_utils import rls_transaction
 from api.models import Provider, ResourceScanSummary
-from tasks.jobs.cartography.aws.s3 import sync_aws_s3
+from cartography.intel import create_indexes as cartography_indexes
+from cartography.intel.aws import organizations as cartography_organizations
+from tasks.jobs.cartography.aws.analysis import perform_aws_analysis
 from tasks.jobs.cartography.aws.ecs import sync_aws_ecs
 from tasks.jobs.cartography.aws.iam import sync_aws_iam
+from tasks.jobs.cartography.aws.s3 import sync_aws_s3
 
 
 def sync_aws(
@@ -23,28 +24,45 @@ def sync_aws(
     """
 
     # Getting data from DB
-    account_id = get_aws_provider_account_id(tenant_id, provider_id)
     regions = get_aws_provider_regions(tenant_id, scan_id)
+    provider = get_aws_provider(tenant_id, provider_id)
+    account_id = provider.uid
+    provider_alias = provider.alias
 
     # Configuring Cartography job parameters
     update_tag = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
     common_job_parameters = {"UPDATE_TAG": update_tag, "AWS_ID": account_id}
 
-    return {
+    # Creating Cartography indexes
+    cartography_indexes.run(neo4j_session, None)
+
+    # Syncing AWS Account
+    accounts = {provider_alias: account_id}
+    cartography_organizations.load_aws_accounts(neo4j_session, accounts, update_tag, common_job_parameters)
+
+    # Syncing AWS resources
+    result = {
         "iam": sync_aws_iam(tenant_id, provider_id, account_id, scan_id, regions, neo4j_session, update_tag, common_job_parameters),  # noqa: E501
         "s3": sync_aws_s3(tenant_id, provider_id, account_id, scan_id, regions, neo4j_session, update_tag, common_job_parameters),  # noqa: E501
         "ecs": sync_aws_ecs(tenant_id, provider_id, account_id, scan_id, regions, neo4j_session, update_tag, common_job_parameters),  # noqa: E501
     }
 
-def get_aws_provider_account_id(tenant_id: str, provider_id: str) -> str:
+    # Running AWS analysis
+    syncs = result.keys()
+    perform_aws_analysis(account_id, syncs, regions, neo4j_session, update_tag, common_job_parameters)
+
+    return result
+
+
+def get_aws_provider(tenant_id: str, provider_id: str) -> str:
     """
-    Getting AWS account ID from Prowler DB for a provider.
+    Getting AWS provider from Prowler DB.
     """
 
     with rls_transaction(tenant_id):
         provider = Provider.objects.get(pk=provider_id)
 
-    return provider.uid
+    return provider
 
 
 def get_aws_provider_regions(tenant_id: str, scan_id: str) -> list[str]:

api/src/backend/tasks/jobs/cartography/aws/analysis.py

@@ -0,0 +1,82 @@
+import neo4j
+
+from typing import Any
+
+from cartography.intel import aws as cartography_aws
+# from cartography.intel.aws import permission_relationships as cartography_permission_relationships
+# from cartography.intel.aws import resourcegroupstaggingapi as cartography_resourcegroupstaggingapi
+from celery.utils.log import get_task_logger
+
+# TODO: Do the rigth logging setup
+# logger = get_task_logger(__name__)
+import logging
+from config.custom_logging import BackendLogger
+logger = logging.getLogger(BackendLogger.API)
+
+
+def perform_aws_analysis(
+    account_id: str,
+    syncs: list[str],
+    regions: list[str],
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+):
+    """
+    Code based on `cartography.intel.aws._sync_multiple_accounts` and `cartography.intel.aws._sync_one_account`.
+    # TODO: Check if we need to run `permission_relationships.sync`
+    # TODO: Prowler DB doesn't save `resourcegroupstaggingapi`
+    """
+
+    # TODO: Check if we need to run `permission_relationships.sync` and with what `permission_relationships_file`
+    # cartography_permission_relationships.sync(
+    #     neo4j_session,
+    #     None,  # `boto3_session` is not needed here
+    #     regions,
+    #     account_id,
+    #     update_tag,
+    #     common_job_parameters,
+    # )
+
+    # TODO: As `boto3_session` is needed for
+    #       `boto3_session.client("resourcegroupstaggingapi", region_name=region)
+    #       we can't call this function
+    # cartography_resourcegroupstaggingapi.sync(
+    #     neo4j_session
+    #     None,  # `boto3_session` is REALLY needed here
+    #     regions,
+    #     account_id,
+    #     update_tag,
+    #     common_job_parameters,
+    # )
+
+    cartography_aws.run_scoped_analysis_job(
+        "aws_ec2_iaminstanceprofile.json",
+        neo4j_session,
+        common_job_parameters,
+    )
+
+    cartography_aws.run_analysis_job(
+        "aws_lambda_ecr.json",
+        neo4j_session,
+        common_job_parameters,
+    )
+
+    cartography_aws.merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=account_id,
+        synced_type="AWSAccount",
+        update_tag=update_tag,
+        stat_handler=cartography_aws.stat_handler,
+    )
+
+    cartography_aws.run_cleanup_job(
+        "aws_post_ingestion_principals_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
+
+    stringified_syncs = ",".join(syncs)
+    requested_syncs = cartography_aws.parse_and_validate_aws_requested_syncs(stringified_syncs)
+    cartography_aws._perform_aws_analysis(requested_syncs, neo4j_session, common_job_parameters)

api/src/backend/tasks/jobs/cartography/aws/ecs.py

@@ -7,7 +7,6 @@ import neo4j
 
 from cartography.intel.aws import ecs as cartography_ecs
 from celery.utils.log import get_task_logger
-from openai import containers
 
 from api.db_utils import rls_transaction
 from api.models import Resource, ResourceScanSummary

api/src/backend/tasks/jobs/cartography/aws/iam.py

@@ -1,13 +1,11 @@
 import json
 
-from collections import defaultdict
 from typing import Any
 
 import neo4j
 
 from cartography.intel.aws import iam as cartography_iam
 from celery.utils.log import get_task_logger
-from openai import containers
 
 from api.db_utils import rls_transaction
 from api.models import Resource, ResourceScanSummary
@@ -66,7 +64,7 @@ def _sync(
         update_tag,
     )
 
-    _sync_users(
+    users = _sync_users(
         tenant_id,
         provider_id,
         account_id,
@@ -76,7 +74,7 @@ def _sync(
         update_tag,
     )
 
-    _sync_groups(
+    groups = _sync_groups(
         tenant_id,
         provider_id,
         account_id,
@@ -86,7 +84,7 @@ def _sync(
         update_tag,
     )
 
-    _sync_roles(
+    roles = _sync_roles(
         tenant_id,
         provider_id,
         account_id,
@@ -103,7 +101,7 @@ def _sync(
         common_job_parameters,
     )
 
-    _sync_user_access_keys(
+    access_keys = _sync_user_access_keys(
         tenant_id,
         provider_id,
         account_id,
@@ -125,7 +123,12 @@ def _sync(
         stat_handler=cartography_iam.stat_handler,
     )
 
-    return {}
+    return {
+        "users": len(users),
+        "groups": len(groups),
+        "roles": len(roles),
+        "access_keys": len(access_keys),
+    }
 
 def _sync_users(
     tenant_id: str,
@@ -144,6 +147,8 @@ def _sync_users(
     _sync_inline_policies(user_data["Users"], neo4j_session, update_tag, account_id)
     _sync_managed_policies(user_data["Users"], neo4j_session, update_tag, account_id)
 
+    return user_data["Users"]
+
 
 def _get_user_list_data(
     tenant_id: str,
@@ -312,6 +317,8 @@ def _sync_groups(
 
     _sync_managed_policies(group_data["Groups"], neo4j_session, update_tag, account_id)
 
+    return group_data["Groups"]
+
 
 def _get_group_list_data(
     tenant_id: str,
@@ -409,6 +416,8 @@ def _sync_roles(
 
     _sync_managed_policies(roles_data["Roles"], neo4j_session, update_tag, account_id)
 
+    return roles_data["Roles"]
+
 
 def _get_role_list_data(
     tenant_id: str,
@@ -491,9 +500,11 @@ def _sync_user_access_keys(
     user_data = _get_user_list_data(tenant_id, provider_id, scan_id, regions)
     user_access_keys = _pretransform_access_keys(user_data["Users"])
     access_key_data = cartography_iam.transform_access_keys(user_access_keys)
+
     cartography_iam.load_access_keys(
         neo4j_session, access_key_data, update_tag, account_id
     )
+
     cartography_iam.GraphJob.from_node_schema(
         cartography_iam.AccountAccessKeySchema(),
         common_job_parameters,
@@ -501,6 +512,8 @@ def _sync_user_access_keys(
         neo4j_session,
     )
 
+    return access_key_data
+
 
 def _pretransform_access_keys(users: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
     """

api/src/backend/tasks/jobs/cartography/aws/s3.py

@@ -136,15 +136,11 @@ def _get_and_load_s3_bucket_details(
 ) -> None:
     """
     Code based on `cartography.intel.aws.s3.get_s3_bucket_details` and `cartography.intel.aws.s3.load_s3_details`.
-    # TODO: Versions of the next functions are not implemented yet:
-        - `cartography.intel.aws.s3.parse_acl`.
-        - `cartography.intel.aws.s3.parse_policy` and
-          `cartography.intel.aws.s3.parse_policy_statements` as a single function, if not two.
     """
 
-    acls: list[dict[str, Any]] = []  # TODO
-    policies: list[dict[str, Any]] = []  # TODO
-    statements: list[dict[str, Any]] = []  # TODO
+    acls: list[dict[str, Any]] = []
+    policies: list[dict[str, Any]] = []
+    statements: list[dict[str, Any]] = []
     encryption_configs: list[dict[str, Any]] = []
     versioning_configs: list[dict[str, Any]] = []
     public_access_block_configs: list[dict[str, Any]] = []
@@ -152,19 +148,17 @@ def _get_and_load_s3_bucket_details(
     bucket_logging_configs: list[dict[str, Any]] = []
 
     for bucket_metadata in buckets_metadata:
-        parsed_acls = None  # TODO
-        parsed_policy = None  # TODO
-        parsed_statements = None  # TODO
-
         parsed_acls = _parse_s3_bucket_acl(bucket_metadata, account_id)
         if parsed_acls is not None:
-            acls.append(parsed_acls)
+            acls.extend(parsed_acls)
 
         parsed_policy = _parse_s3_bucket_policy(bucket_metadata)
         if parsed_policy is not None:
             policies.append(parsed_policy)
 
-        # TODO: More work
+        parsed_statements = _parse_s3_bucket_policy_statements(bucket_metadata)
+        if parsed_statements is not None:
+            statements.extend(parsed_statements)
 
         parsed_encryption = _parse_s3_bucket_encryption(bucket_metadata)
         if parsed_encryption is not None:
@@ -229,11 +223,33 @@ def _parse_s3_bucket_acl(bucket_metadata: dict[str, Any], account_id: str) -> di
             "Permission": grantee.get("permission"),
         })
 
-    return cartography_s3.parse_acl(bucket_metadata.get("name"), bucket_metadata.get("acl_grantees"), account_id)
+    return cartography_s3.parse_acl(acl, bucket_metadata.get("name"), account_id)
 
 
 def _parse_s3_bucket_policy(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None:
-    return None  # TODO
+    """
+    Code based on `cartography.intel.aws.s3.parse_policy`.
+    """
+    if not bucket_metadata.get("policy"):
+        return None
+
+    policy = {
+        "Policy": json.dumps(bucket_metadata.get("policy")),
+    }
+    return cartography_s3.parse_policy(bucket_metadata.get("name"), policy)
+
+
+def _parse_s3_bucket_policy_statements(bucket_metadata: dict[str, Any]) -> list[dict[str, Any]] | None:
+    """
+    Code based on `cartography.intel.aws.s3.parse_policy_statements`.
+    """
+    if not bucket_metadata.get("policy"):
+        return None
+
+    policy = {
+        "Policy": json.dumps(bucket_metadata.get("policy")),
+    }
+    return cartography_s3.parse_policy_statements(bucket_metadata.get("name"), policy)
 
 
 def _parse_s3_bucket_encryption(bucket_metadata: dict[str, Any]) -> dict[str, Any] | None:
@@ -303,7 +319,6 @@ def _parse_s3_bucket_bucket_logging(bucket_metadata: dict[str, Any]) -> dict[str
 def _parse_s3_notifications(buckets_metadata: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """
     Code based on `cartography.intel.aws.s3.parse_notification_configuration`.
-    # TODO: Do a deeper check of this function
     """
 
     notifications: list[dict[str, Any]] = []