diff --git a/.env b/.env index 3d9b1fdcbc..1fbf3a630a 100644 --- a/.env +++ b/.env @@ -47,6 +47,7 @@ NEO4J_PORT=7687 NEO4J_USER=neo4j NEO4J_PASSWORD=neo4j_password # Neo4j settings +NEO4J_DBMS_MAX__DATABASES=1000000 NEO4J_SERVER_MEMORY_PAGECACHE_SIZE=1G NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE=1G NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE=1G diff --git a/api/poetry.lock b/api/poetry.lock index 88558a76d4..4bf8a3ddae 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1194,7 +1194,7 @@ files = [ [[package]] name = "cartography" -version = "0.0.1.dev1267+g7c7d6d501" +version = "0.0.1.dev1268+gc134846c0" description = "Explore assets and their relationships across your technical infrastructure." optional = false python-versions = ">=3.10" @@ -1258,8 +1258,8 @@ xmltodict = "*" [package.source] type = "git" url = "https://github.com/prowler-cloud/cartography" -reference = "azure-mgmt-sql-3" -resolved_reference = "7c7d6d5014bf079066ee1645e9517c67fb36fe67" +reference = "master" +resolved_reference = "c134846c0db64747340f880cf0b5085f5e473e03" [[package]] name = "celery" @@ -5478,8 +5478,8 @@ tzlocal = "5.3.1" [package.source] type = "git" url = "https://github.com/prowler-cloud/prowler.git" -reference = "PROWLER-510-update-cartography-dependency" -resolved_reference = "221f6afbcf625e4b6334919f165a1419a1e7d5d4" +reference = "attack-paths-demo" +resolved_reference = "95d9e9a59f8e52e4096a5c17bf839e515b918239" [[package]] name = "psutil" @@ -5696,7 +5696,7 @@ description = "PycURL -- A Python Interface To The cURL library" optional = false python-versions = ">=3.5" groups = ["main"] -markers = "sys_platform != \"win32\" and platform_python_implementation == \"CPython\"" +markers = "platform_python_implementation == \"CPython\" and sys_platform != \"win32\"" files = [ {file = "pycurl-7.45.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c31b390f1e2cd4525828f1bb78c1f825c0aab5d1588228ed71b22c4784bdb593"}, {file = "pycurl-7.45.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:942b352b69184cb26920db48e0c5cb95af39874b57dbe27318e60f1e68564e37"}, @@ -5956,7 +5956,7 @@ description = "The MSALRuntime Python Interop Package" optional = false python-versions = ">=3.6" groups = ["main"] -markers = "sys_platform == \"win32\" and (platform_system == \"Windows\" or platform_system == \"Darwin\" or platform_system == \"Linux\")" +markers = "(platform_system == \"Windows\" or platform_system == \"Darwin\" or platform_system == \"Linux\") and sys_platform == \"win32\"" files = [ {file = "pymsalruntime-0.18.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:0c22e2e83faa10de422bbfaacc1bb2887c9025ee8a53f0fc2e4f7db01c4a7b66"}, {file = "pymsalruntime-0.18.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:8ce2944a0f944833d047bb121396091e00287e2b6373716106da86ea99abf379"}, @@ -7806,4 +7806,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "1c2a75921edf35350cd01ff76bc4d4316ffd4f4aba8d02b4454013d8b70a730d" +content-hash = "de57b503d0f96c22ac3f9b1e9845aefac0a5b32089c9d90928a357f991384db3" diff --git a/api/pyproject.toml b/api/pyproject.toml index cdcf59834a..f96fd1f54b 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "drf-spectacular-jsonapi==0.5.1", "gunicorn==23.0.0", "lxml==5.3.2", - "prowler @ git+https://github.com/prowler-cloud/prowler.git@PROWLER-510-update-cartography-dependency", + "prowler @ git+https://github.com/prowler-cloud/prowler.git@attack-paths-demo", "psycopg2-binary==2.9.9", "pytest-celery[redis] (>=1.0.1,<2.0.0)", "sentry-sdk[django] (>=2.20.0,<3.0.0)", @@ -37,7 +37,7 @@ dependencies = [ "matplotlib (>=3.10.6,<4.0.0)", "reportlab (>=4.4.4,<5.0.0)", "neo4j (<6.0.0)", - "cartography @ git+https://github.com/prowler-cloud/cartography@azure-mgmt-sql-3", + "cartography @ git+https://github.com/prowler-cloud/cartography@master", ] description = "Prowler's API (Django/DRF)" license = "Apache-2.0" diff --git a/api/src/backend/tasks/jobs/attack_paths/aws.py b/api/src/backend/tasks/jobs/attack_paths/aws.py index b89524a2de..e244b6cca7 100644 --- a/api/src/backend/tasks/jobs/attack_paths/aws.py +++ b/api/src/backend/tasks/jobs/attack_paths/aws.py @@ -173,7 +173,9 @@ def sync_aws_account( max_progress = ( 87 # `cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"]` - 1 ) - n_steps = len(requested_syncs) - 2 # Excluding `permission_relationships` and `resourcegroupstaggingapi` + n_steps = ( + len(requested_syncs) - 2 + ) # Excluding `permission_relationships` and `resourcegroupstaggingapi` progress_step = (max_progress - current_progress) / n_steps failed_syncs = {} @@ -195,7 +197,9 @@ def sync_aws_account( if func_name == "ecr:image_layers": cartography_aws.RESOURCE_FUNCTIONS[func_name]( neo4j_session=sync_args.get("neo4j_session"), - aioboto3_session=get_aioboto3_session(sync_args.get("boto3_session")), + aioboto3_session=get_aioboto3_session( + sync_args.get("boto3_session") + ), regions=sync_args.get("regions"), current_aws_account_id=sync_args.get("current_aws_account_id"), update_tag=sync_args.get("update_tag"), diff --git a/api/src/backend/tasks/jobs/attack_paths/scan.py b/api/src/backend/tasks/jobs/attack_paths/scan.py index aa7bb7b756..6e1d3e1b64 100644 --- a/api/src/backend/tasks/jobs/attack_paths/scan.py +++ b/api/src/backend/tasks/jobs/attack_paths/scan.py @@ -46,19 +46,35 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: prowler_api_provider = ProwlerAPIProvider.objects.get(scan__pk=scan_id) prowler_sdk_provider = initialize_prowler_provider(prowler_api_provider) - # If the provider is still not supported, just return the current `ingestion_exceptions`, that is empty - if not get_cartography_ingestion_function(prowler_api_provider.provider): + # Attack Paths Scan necessary objects + cartography_ingestion_function = get_cartography_ingestion_function( + prowler_api_provider.provider + ) + attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id) + + # Checks before starting the scan + if not cartography_ingestion_function: + ingestion_exceptions = { + "global_error": f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans" + } + if attack_paths_scan: + db_utils.finish_attack_paths_scan( + attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions + ) + + logger.warning( + f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans" + ) return ingestion_exceptions - # Getting the Attack Paths Scan object and starting it - attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id) - if not attack_paths_scan: - logger.warning( - f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then" - ) - attack_paths_scan = db_utils.create_attack_paths_scan( - tenant_id, scan_id, prowler_api_provider.id - ) + else: + if not attack_paths_scan: + logger.warning( + f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then" + ) + attack_paths_scan = db_utils.create_attack_paths_scan( + tenant_id, scan_id, prowler_api_provider.id + ) # While creating the Cartography configuration, attributes `neo4j_user` and `neo4j_password` are not really needed in this config object cartography_config = CartographyConfig( @@ -92,7 +108,7 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: # The real scan, where iterates over cloud services ingestion_exceptions = _call_within_event_loop( - get_cartography_ingestion_function(prowler_api_provider.provider), + cartography_ingestion_function, neo4j_session, cartography_config, prowler_api_provider, diff --git a/api/src/backend/tasks/tests/test_attack_paths_scan.py b/api/src/backend/tasks/tests/test_attack_paths_scan.py index 334c02df11..df31b8aa67 100644 --- a/api/src/backend/tasks/tests/test_attack_paths_scan.py +++ b/api/src/backend/tasks/tests/test_attack_paths_scan.py @@ -74,6 +74,9 @@ class TestAttackPathsRun: patch( "tasks.jobs.attack_paths.scan.cartography_analysis.run" ) as mock_cartography_analysis, + patch( + "tasks.jobs.attack_paths.scan.cartography_ontology.run" + ) as mock_cartography_ontology, patch( "tasks.jobs.attack_paths.scan.prowler.create_indexes" ) as mock_prowler_indexes, @@ -115,6 +118,7 @@ class TestAttackPathsRun: mock_cartography_indexes.assert_called_once_with(mock_session, config) mock_prowler_indexes.assert_called_once_with(mock_session) mock_cartography_analysis.assert_called_once_with(mock_session, config) + mock_cartography_ontology.assert_called_once_with(mock_session, config) mock_prowler_analysis.assert_called_once_with( mock_session, provider, diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index bc16d52eb1..36b0942bdf 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -91,18 +91,19 @@ services: # Auth - NEO4J_AUTH=${NEO4J_USER}/${NEO4J_PASSWORD} # Memory limits - - NEO4J_server_memory_pagecache_size=${NEO4J_SERVER_MEMORY_PAGECACHE_SIZE} - - NEO4J_server_memory_heap_initial__size=${NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE} - - NEO4J_server_memory_heap_max__size=${NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE} + - NEO4J_dbms_max__databases=${NEO4J_DBMS_MAX__DATABASES:-1000000} + - NEO4J_server_memory_pagecache_size=${NEO4J_SERVER_MEMORY_PAGECACHE_SIZE:-1G} + - NEO4J_server_memory_heap_initial__size=${NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE:-1G} + - NEO4J_server_memory_heap_max__size=${NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE:-1G} # APOC - - apoc.export.file.enabled=${NEO4J_POC_EXPORT_FILE_ENABLED} - - apoc.import.file.enabled=${NEO4J_APOC_IMPORT_FILE_ENABLED} - - apoc.import.file.use_neo4j_config=${NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG} - - NEO4J_PLUGINS=${NEO4J_PLUGINS} - - NEO4J_dbms_security_procedures_allowlist=${NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST} - - NEO4J_dbms_security_procedures_unrestricted=${NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED} + - apoc.export.file.enabled=${NEO4J_POC_EXPORT_FILE_ENABLED:-true} + - apoc.import.file.enabled=${NEO4J_APOC_IMPORT_FILE_ENABLED:-true} + - apoc.import.file.use_neo4j_config=${NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG:-true} + - "NEO4J_PLUGINS=${NEO4J_PLUGINS:-[\"apoc\"]}" + - "NEO4J_dbms_security_procedures_allowlist=${NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST:-apoc.*}" + - "NEO4J_dbms_security_procedures_unrestricted=${NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED:-apoc.*}" # Networking - - dbms.connector.bolt.listen_address=${NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS} + - "dbms.connector.bolt.listen_address=${NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS:-0.0.0.0:7687}" # 7474 is the UI port ports: - 7474:7474 diff --git a/docker-compose.yml b/docker-compose.yml index 5a94d5faa0..ee9c088c3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,18 +73,19 @@ services: # Auth - NEO4J_AUTH=${NEO4J_USER}/${NEO4J_PASSWORD} # Memory limits - - NEO4J_server_memory_pagecache_size=${NEO4J_SERVER_MEMORY_PAGECACHE_SIZE} - - NEO4J_server_memory_heap_initial__size=${NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE} - - NEO4J_server_memory_heap_max__size=${NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE} + - NEO4J_dbms_max__databases=${NEO4J_DBMS_MAX__DATABASES:-1000000} + - NEO4J_server_memory_pagecache_size=${NEO4J_SERVER_MEMORY_PAGECACHE_SIZE:-1G} + - NEO4J_server_memory_heap_initial__size=${NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE:-1G} + - NEO4J_server_memory_heap_max__size=${NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE:-1G} # APOC - - apoc.export.file.enabled=${NEO4J_POC_EXPORT_FILE_ENABLED} - - apoc.import.file.enabled=${NEO4J_APOC_IMPORT_FILE_ENABLED} - - apoc.import.file.use_neo4j_config=${NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG} - - NEO4J_PLUGINS=${NEO4J_PLUGINS} - - NEO4J_dbms_security_procedures_allowlist=${NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST} - - NEO4J_dbms_security_procedures_unrestricted=${NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED} + - apoc.export.file.enabled=${NEO4J_POC_EXPORT_FILE_ENABLED:-true} + - apoc.import.file.enabled=${NEO4J_APOC_IMPORT_FILE_ENABLED:-true} + - apoc.import.file.use_neo4j_config=${NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG:-true} + - "NEO4J_PLUGINS=${NEO4J_PLUGINS:-[\"apoc\"]}" + - "NEO4J_dbms_security_procedures_allowlist=${NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST:-apoc.*}" + - "NEO4J_dbms_security_procedures_unrestricted=${NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED:-apoc.*}" # Networking - - dbms.connector.bolt.listen_address=${NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS} + - "dbms.connector.bolt.listen_address=${NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS:-0.0.0.0:7687}" ports: - ${NEO4J_PORT:-7687}:7687 healthcheck: