feat(attack-paths): add graph_data_ready field to decouple query availability from scan state (#10089)

Co-authored-by: Alan Buscaglia <gentlemanprogramming@gmail.com>
This commit is contained in:
Josema Camacho
2026-02-17 17:29:36 +01:00
committed by GitHub
parent ff25d6a8c2
commit 7698cdce2e
17 changed files with 860 additions and 366 deletions
+1
View File
@@ -21,6 +21,7 @@ All notable changes to the **Prowler API** are documented in this file.
- Support CSA CCM 4.0 for the Alibaba Cloud provider [(#10061)](https://github.com/prowler-cloud/prowler/pull/10061)
- Attack Paths: Mark attack Paths scan as failed when Celery task fails outside job error handling [(#10065)](https://github.com/prowler-cloud/prowler/pull/10065)
- Attack Paths: Remove legacy per-scan `graph_database` and `is_graph_database_deleted` fields from AttackPathsScan model [(#10077)](https://github.com/prowler-cloud/prowler/pull/10077)
- Attack Paths: Add `graph_data_ready` field to decouple query availability from scan state [(#10089)](https://github.com/prowler-cloud/prowler/pull/10089)
### 🔐 Security
@@ -7,6 +7,7 @@
"provider": "b85601a8-4b45-4194-8135-03fb980ef428",
"scan": "01920573-aa9c-73c9-bcda-f2e35c9b19d2",
"state": "completed",
"graph_data_ready": true,
"progress": 100,
"update_tag": 1693586667,
"task": null,
@@ -0,0 +1,17 @@
# Generated by Django 5.1.15 on 2026-02-16 13:55
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0078_remove_attackpathsscan_graph_database_fields"),
]
operations = [
migrations.AddField(
model_name="attackpathsscan",
name="graph_data_ready",
field=models.BooleanField(default=False),
),
]
@@ -0,0 +1,26 @@
# Separate from 0079 because psqlextra's schema editor runs AddField DDL and DML
# on different database connections, causing a deadlock when combined with RunPython
# in the same migration.
from django.db import migrations
from api.db_router import MainRouter
def backfill_graph_data_ready(apps, schema_editor):
"""Set graph_data_ready=True for all completed AttackPathsScan rows."""
AttackPathsScan = apps.get_model("api", "AttackPathsScan")
AttackPathsScan.objects.using(MainRouter.admin_db).filter(
state="completed",
graph_data_ready=False,
).update(graph_data_ready=True)
class Migration(migrations.Migration):
dependencies = [
("api", "0079_attackpathsscan_graph_data_ready"),
]
operations = [
migrations.RunPython(backfill_graph_data_ready, migrations.RunPython.noop),
]
+1
View File
@@ -655,6 +655,7 @@ class AttackPathsScan(RowLevelSecurityProtectedModel):
state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE)
progress = models.IntegerField(default=0)
graph_data_ready = models.BooleanField(default=False)
# Timing
started_at = models.DateTimeField(null=True, blank=True)
+52 -165
View File
@@ -296,6 +296,7 @@ paths:
enum:
- state
- progress
- graph_data_ready
- provider
- provider_alias
- provider_type
@@ -355,7 +356,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -388,7 +389,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -571,6 +572,7 @@ paths:
enum:
- state
- progress
- graph_data_ready
- provider
- provider_alias
- provider_type
@@ -631,6 +633,7 @@ paths:
enum:
- state
- progress
- graph_data_ready
- provider
- provider_alias
- provider_type
@@ -1340,7 +1343,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -1373,7 +1376,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -1934,7 +1937,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -1967,7 +1970,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -2436,7 +2439,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -2469,7 +2472,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -2936,7 +2939,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -2969,7 +2972,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -3424,7 +3427,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -3457,7 +3460,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5253,7 +5256,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5286,7 +5289,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5420,7 +5423,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5453,7 +5456,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5763,7 +5766,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5796,7 +5799,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5964,7 +5967,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -5997,7 +6000,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -6395,7 +6398,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -6428,7 +6431,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -6561,7 +6564,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -6594,7 +6597,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -6751,7 +6754,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -6784,7 +6787,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -7582,7 +7585,7 @@ paths:
name: filter[provider]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -7615,7 +7618,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -7650,7 +7653,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -7683,7 +7686,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -8341,7 +8344,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -8374,7 +8377,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -8847,7 +8850,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -8880,7 +8883,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -9166,7 +9169,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -9199,7 +9202,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -9491,7 +9494,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -9524,7 +9527,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -10350,7 +10353,7 @@ paths:
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -10383,7 +10386,7 @@ paths:
type: array
items:
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
enum:
- alibabacloud
- aws
@@ -12596,16 +12599,16 @@ components:
type: string
description:
type: string
attribution:
allOf:
- $ref: '#/components/schemas/AttackPathsQueryAttribution'
nullable: true
provider:
type: string
parameters:
type: array
items:
$ref: '#/components/schemas/AttackPathsQueryParameter'
attribution:
allOf:
- $ref: '#/components/schemas/AttackPathsQueryAttribution'
nullable: true
required:
- id
- name
@@ -12803,6 +12806,8 @@ components:
type: integer
maximum: 2147483647
minimum: -2147483648
graph_data_ready:
type: boolean
provider_alias:
type: string
readOnly: true
@@ -17516,36 +17521,6 @@ components:
required:
- clouds_yaml_content
- clouds_yaml_cloud
- type: object
title: OpenStack Explicit Credentials
properties:
auth_url:
type: string
description: OpenStack Keystone authentication URL (e.g.,
https://openstack.example.com:5000/v3).
username:
type: string
description: OpenStack username for authentication.
password:
type: string
description: OpenStack password for authentication.
region_name:
type: string
description: OpenStack region name (e.g., RegionOne).
identity_api_version:
type: string
description: 'Keystone API version (default: 3).'
user_domain_name:
type: string
description: 'User domain name (default: Default).'
project_domain_name:
type: string
description: 'Project domain name (default: Default).'
required:
- auth_url
- username
- password
- region_name
writeOnly: true
required:
- secret
@@ -18560,7 +18535,7 @@ components:
* `alibabacloud` - Alibaba Cloud
* `cloudflare` - Cloudflare
* `openstack` - OpenStack
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
uid:
type: string
title: Unique identifier for the provider, set by the provider
@@ -18679,7 +18654,7 @@ components:
- cloudflare
- openstack
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
description: |-
Type of provider to create.
@@ -18745,7 +18720,7 @@ components:
- cloudflare
- openstack
type: string
x-spec-enum-id: 2d8d323e9cc0044b
x-spec-enum-id: 4b8815b179aa7216
description: |-
Type of provider to create.
@@ -19595,35 +19570,6 @@ components:
required:
- clouds_yaml_content
- clouds_yaml_cloud
- type: object
title: OpenStack Explicit Credentials
properties:
auth_url:
type: string
description: OpenStack Keystone authentication URL (e.g., https://openstack.example.com:5000/v3).
username:
type: string
description: OpenStack username for authentication.
password:
type: string
description: OpenStack password for authentication.
region_name:
type: string
description: OpenStack region name (e.g., RegionOne).
identity_api_version:
type: string
description: 'Keystone API version (default: 3).'
user_domain_name:
type: string
description: 'User domain name (default: Default).'
project_domain_name:
type: string
description: 'Project domain name (default: Default).'
required:
- auth_url
- username
- password
- region_name
writeOnly: true
required:
- secret_type
@@ -20024,36 +19970,6 @@ components:
required:
- clouds_yaml_content
- clouds_yaml_cloud
- type: object
title: OpenStack Explicit Credentials
properties:
auth_url:
type: string
description: OpenStack Keystone authentication URL (e.g.,
https://openstack.example.com:5000/v3).
username:
type: string
description: OpenStack username for authentication.
password:
type: string
description: OpenStack password for authentication.
region_name:
type: string
description: OpenStack region name (e.g., RegionOne).
identity_api_version:
type: string
description: 'Keystone API version (default: 3).'
user_domain_name:
type: string
description: 'User domain name (default: Default).'
project_domain_name:
type: string
description: 'Project domain name (default: Default).'
required:
- auth_url
- username
- password
- region_name
writeOnly: true
required:
- secret_type
@@ -20465,35 +20381,6 @@ components:
required:
- clouds_yaml_content
- clouds_yaml_cloud
- type: object
title: OpenStack Explicit Credentials
properties:
auth_url:
type: string
description: OpenStack Keystone authentication URL (e.g., https://openstack.example.com:5000/v3).
username:
type: string
description: OpenStack username for authentication.
password:
type: string
description: OpenStack password for authentication.
region_name:
type: string
description: OpenStack region name (e.g., RegionOne).
identity_api_version:
type: string
description: 'Keystone API version (default: 3).'
user_domain_name:
type: string
description: 'User domain name (default: Default).'
project_domain_name:
type: string
description: 'Project domain name (default: Default).'
required:
- auth_url
- username
- password
- region_name
writeOnly: true
required:
- secret
+112 -2
View File
@@ -3922,6 +3922,7 @@ class TestAttackPathsScanViewSet:
attack_paths_scan = create_attack_paths_scan(
provider,
scan=scans_fixture[0],
graph_data_ready=True,
)
query_definition = AttackPathsQueryDefinition(
id="aws-rds",
@@ -4000,7 +4001,7 @@ class TestAttackPathsScanViewSet:
assert attributes["nodes"] == graph_payload["nodes"]
assert attributes["relationships"] == graph_payload["relationships"]
def test_run_attack_paths_query_requires_completed_scan(
def test_run_attack_paths_query_blocks_when_graph_data_not_ready(
self,
authenticated_client,
providers_fixture,
@@ -4012,6 +4013,7 @@ class TestAttackPathsScanViewSet:
provider,
scan=scans_fixture[0],
state=StateChoices.EXECUTING,
graph_data_ready=False,
)
response = authenticated_client.post(
@@ -4023,7 +4025,113 @@ class TestAttackPathsScanViewSet:
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "must be completed" in response.json()["errors"][0]["detail"]
assert "not available" in response.json()["errors"][0]["detail"]
def test_run_attack_paths_query_allows_executing_scan_when_graph_data_ready(
self,
authenticated_client,
providers_fixture,
scans_fixture,
create_attack_paths_scan,
):
provider = providers_fixture[0]
attack_paths_scan = create_attack_paths_scan(
provider,
scan=scans_fixture[0],
state=StateChoices.EXECUTING,
graph_data_ready=True,
)
query_definition = AttackPathsQueryDefinition(
id="aws-test",
name="Test",
short_description="Test query.",
description="Test query",
provider=provider.provider,
cypher="MATCH (n) RETURN n",
parameters=[],
)
with (
patch("api.v1.views.get_query_by_id", return_value=query_definition),
patch(
"api.v1.views.attack_paths_views_helpers.prepare_query_parameters",
return_value={"provider_uid": provider.uid},
),
patch(
"api.v1.views.attack_paths_views_helpers.execute_attack_paths_query",
return_value={
"nodes": [{"id": "n1", "labels": ["AWSAccount"], "properties": {}}],
"relationships": [],
},
),
patch("api.v1.views.graph_database.clear_cache"),
patch(
"api.v1.views.graph_database.get_database_name", return_value="db-test"
),
):
response = authenticated_client.post(
reverse(
"attack-paths-scans-queries-run",
kwargs={"pk": attack_paths_scan.id},
),
data=self._run_payload("aws-test"),
content_type=API_JSON_CONTENT_TYPE,
)
assert response.status_code == status.HTTP_200_OK
def test_run_attack_paths_query_allows_failed_scan_when_graph_data_ready(
self,
authenticated_client,
providers_fixture,
scans_fixture,
create_attack_paths_scan,
):
provider = providers_fixture[0]
attack_paths_scan = create_attack_paths_scan(
provider,
scan=scans_fixture[0],
state=StateChoices.FAILED,
graph_data_ready=True,
)
query_definition = AttackPathsQueryDefinition(
id="aws-test",
name="Test",
short_description="Test query.",
description="Test query",
provider=provider.provider,
cypher="MATCH (n) RETURN n",
parameters=[],
)
with (
patch("api.v1.views.get_query_by_id", return_value=query_definition),
patch(
"api.v1.views.attack_paths_views_helpers.prepare_query_parameters",
return_value={"provider_uid": provider.uid},
),
patch(
"api.v1.views.attack_paths_views_helpers.execute_attack_paths_query",
return_value={
"nodes": [{"id": "n1", "labels": ["AWSAccount"], "properties": {}}],
"relationships": [],
},
),
patch("api.v1.views.graph_database.clear_cache"),
patch(
"api.v1.views.graph_database.get_database_name", return_value="db-test"
),
):
response = authenticated_client.post(
reverse(
"attack-paths-scans-queries-run",
kwargs={"pk": attack_paths_scan.id},
),
data=self._run_payload("aws-test"),
content_type=API_JSON_CONTENT_TYPE,
)
assert response.status_code == status.HTTP_200_OK
def test_run_attack_paths_query_unknown_query(
self,
@@ -4036,6 +4144,7 @@ class TestAttackPathsScanViewSet:
attack_paths_scan = create_attack_paths_scan(
provider,
scan=scans_fixture[0],
graph_data_ready=True,
)
with patch("api.v1.views.get_query_by_id", return_value=None):
@@ -4062,6 +4171,7 @@ class TestAttackPathsScanViewSet:
attack_paths_scan = create_attack_paths_scan(
provider,
scan=scans_fixture[0],
graph_data_ready=True,
)
query_definition = AttackPathsQueryDefinition(
id="aws-empty",
+1
View File
@@ -1145,6 +1145,7 @@ class AttackPathsScanSerializer(RLSSerializer):
"id",
"state",
"progress",
"graph_data_ready",
"provider",
"provider_alias",
"provider_type",
+2 -2
View File
@@ -2482,10 +2482,10 @@ class AttackPathsScanViewSet(BaseRLSViewSet):
def run_attack_paths_query(self, request, pk=None):
attack_paths_scan = self.get_object()
if attack_paths_scan.state != StateChoices.COMPLETED:
if not attack_paths_scan.graph_data_ready:
raise ValidationError(
{
"detail": "The Attack Paths scan must be completed before running Attack Paths queries"
"detail": "Attack Paths data is not available for querying - a scan must complete at least once before queries can be run"
}
)
@@ -28,12 +28,21 @@ def create_attack_paths_scan(
return None
with rls_transaction(tenant_id):
# Inherit graph_data_ready from the previous scan for this provider,
# so queries remain available while the new scan runs.
previous_data_ready = ProwlerAPIAttackPathsScan.objects.filter(
tenant_id=tenant_id,
provider_id=provider_id,
graph_data_ready=True,
).exists()
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.create(
tenant_id=tenant_id,
provider_id=provider_id,
scan_id=scan_id,
state=StateChoices.SCHEDULED,
started_at=datetime.now(tz=timezone.utc),
graph_data_ready=previous_data_ready,
)
attack_paths_scan.save()
@@ -116,6 +125,32 @@ def update_attack_paths_scan_progress(
attack_paths_scan.save(update_fields=["progress"])
def set_graph_data_ready(
attack_paths_scan: ProwlerAPIAttackPathsScan,
ready: bool,
) -> None:
with rls_transaction(attack_paths_scan.tenant_id):
attack_paths_scan.graph_data_ready = ready
attack_paths_scan.save(update_fields=["graph_data_ready"])
def set_provider_graph_data_ready(
attack_paths_scan: ProwlerAPIAttackPathsScan,
ready: bool,
) -> None:
"""
Set `graph_data_ready` for ALL scans of the same provider.
Used before drop/sync so that older scan IDs cannot bypass the query gate while the graph is being replaced.
"""
with rls_transaction(attack_paths_scan.tenant_id):
ProwlerAPIAttackPathsScan.objects.filter(
tenant_id=attack_paths_scan.tenant_id,
provider_id=attack_paths_scan.provider_id,
).update(graph_data_ready=ready)
attack_paths_scan.refresh_from_db(fields=["graph_data_ready"])
def fail_attack_paths_scan(
tenant_id: str,
scan_id: str,
@@ -169,6 +169,7 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
sync.create_sync_indexes(tenant_neo4j_session)
logger.info(f"Deleting existing provider graph in {tenant_database_name}")
db_utils.set_provider_graph_data_ready(attack_paths_scan, False)
graph_database.drop_subgraph(
database=tenant_database_name,
provider_id=str(prowler_api_provider.id),
@@ -183,6 +184,7 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
target_database=tenant_database_name,
provider_id=str(prowler_api_provider.id),
)
db_utils.set_graph_data_ready(attack_paths_scan, True)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 99)
logger.info(f"Clearing Neo4j cache for database {tenant_database_name}")
@@ -28,6 +28,8 @@ class TestAttackPathsRun:
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
)
@patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready")
@patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready")
@patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan")
@patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress")
@patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan")
@@ -72,6 +74,8 @@ class TestAttackPathsRun:
mock_starting,
mock_update_progress,
mock_finish,
mock_set_provider_graph_data_ready,
mock_set_graph_data_ready,
mock_event_loop,
mock_drop_db,
tenants_fixture,
@@ -159,9 +163,66 @@ class TestAttackPathsRun:
mock_finish.assert_called_once_with(
attack_paths_scan, StateChoices.COMPLETED, ingestion_result
)
mock_set_provider_graph_data_ready.assert_called_once_with(
attack_paths_scan, False
)
mock_set_graph_data_ready.assert_called_once_with(attack_paths_scan, True)
@patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
return_value="Cartography failed: ingestion boom",
)
@patch(
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
)
@patch("tasks.jobs.attack_paths.scan.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan")
@patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready")
@patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready")
@patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress")
@patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan")
@patch("tasks.jobs.attack_paths.scan.findings.analysis")
@patch("tasks.jobs.attack_paths.scan.internet.analysis")
@patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes")
@patch("tasks.jobs.attack_paths.scan.cartography_analysis.run")
@patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run")
@patch("tasks.jobs.attack_paths.scan.graph_database.create_database")
@patch(
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
return_value="db-scan-id",
)
@patch("tasks.jobs.attack_paths.scan.graph_database.get_uri")
@patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(_enabled_regions=["us-east-1"]),
)
@patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
)
def test_run_failure_marks_scan_failed(
self, tenants_fixture, providers_fixture, scans_fixture
self,
mock_init_provider,
mock_get_uri,
mock_get_db_name,
mock_create_db,
mock_cartography_indexes,
mock_cartography_analysis,
mock_findings_indexes,
mock_internet_analysis,
mock_findings_analysis,
mock_starting,
mock_update_progress,
mock_set_provider_graph_data_ready,
mock_set_graph_data_ready,
mock_finish,
mock_drop_db,
mock_event_loop,
mock_stringify,
tenants_fixture,
providers_fixture,
scans_fixture,
):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
@@ -185,53 +246,18 @@ class TestAttackPathsRun:
ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom"))
with (
patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
),
patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(_enabled_regions=["us-east-1"]),
),
patch("tasks.jobs.attack_paths.scan.graph_database.get_uri"),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
return_value="db-scan-id",
),
patch("tasks.jobs.attack_paths.scan.graph_database.create_database"),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_session",
return_value=session_ctx,
),
patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run"),
patch("tasks.jobs.attack_paths.scan.cartography_analysis.run"),
patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes"),
patch("tasks.jobs.attack_paths.scan.internet.analysis"),
patch("tasks.jobs.attack_paths.scan.findings.analysis"),
patch(
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
return_value=attack_paths_scan,
),
patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"),
patch(
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
),
patch(
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
) as mock_finish,
patch("tasks.jobs.attack_paths.scan.graph_database.drop_database"),
patch(
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
return_value=ingestion_fn,
),
patch(
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
),
patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
return_value="Cartography failed: ingestion boom",
),
):
with pytest.raises(RuntimeError, match="ingestion boom"):
attack_paths_run(str(tenant.id), str(scan.id), "task-456")
@@ -241,8 +267,64 @@ class TestAttackPathsRun:
assert failure_args[1] == StateChoices.FAILED
assert failure_args[2] == {"global_error": "Cartography failed: ingestion boom"}
@patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
return_value="Cartography failed: ingestion boom",
)
@patch(
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
)
@patch(
"tasks.jobs.attack_paths.scan.graph_database.drop_database",
side_effect=ConnectionError("neo4j down"),
)
@patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan")
@patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready")
@patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready")
@patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress")
@patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan")
@patch("tasks.jobs.attack_paths.scan.findings.analysis")
@patch("tasks.jobs.attack_paths.scan.internet.analysis")
@patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes")
@patch("tasks.jobs.attack_paths.scan.cartography_analysis.run")
@patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run")
@patch("tasks.jobs.attack_paths.scan.graph_database.create_database")
@patch(
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
return_value="db-scan-id",
)
@patch("tasks.jobs.attack_paths.scan.graph_database.get_uri")
@patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(_enabled_regions=["us-east-1"]),
)
@patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
)
def test_run_failure_marks_scan_failed_even_when_drop_database_fails(
self, tenants_fixture, providers_fixture, scans_fixture
self,
mock_init_provider,
mock_get_uri,
mock_get_db_name,
mock_create_db,
mock_cartography_indexes,
mock_cartography_analysis,
mock_findings_indexes,
mock_internet_analysis,
mock_findings_analysis,
mock_starting,
mock_update_progress,
mock_set_provider_graph_data_ready,
mock_set_graph_data_ready,
mock_finish,
mock_drop_db,
mock_event_loop,
mock_stringify,
tenants_fixture,
providers_fixture,
scans_fixture,
):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
@@ -266,56 +348,18 @@ class TestAttackPathsRun:
ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom"))
with (
patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
),
patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(_enabled_regions=["us-east-1"]),
),
patch("tasks.jobs.attack_paths.scan.graph_database.get_uri"),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
return_value="db-scan-id",
),
patch("tasks.jobs.attack_paths.scan.graph_database.create_database"),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_session",
return_value=session_ctx,
),
patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run"),
patch("tasks.jobs.attack_paths.scan.cartography_analysis.run"),
patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes"),
patch("tasks.jobs.attack_paths.scan.internet.analysis"),
patch("tasks.jobs.attack_paths.scan.findings.analysis"),
patch(
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
return_value=attack_paths_scan,
),
patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"),
patch(
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
),
patch(
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
) as mock_finish,
patch(
"tasks.jobs.attack_paths.scan.graph_database.drop_database",
side_effect=ConnectionError("neo4j down"),
),
patch(
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
return_value=ingestion_fn,
),
patch(
"tasks.jobs.attack_paths.scan.utils.call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
),
patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
return_value="Cartography failed: ingestion boom",
),
):
with pytest.raises(RuntimeError, match="ingestion boom"):
attack_paths_run(str(tenant.id), str(scan.id), "task-789")
@@ -1017,3 +1061,317 @@ class TestInternetAnalysis:
result = internet_module.analysis(mock_session, provider, config)
assert result == 0
@pytest.mark.django_db
class TestAttackPathsDbUtilsGraphDataReady:
"""Tests for db_utils functions related to graph_data_ready lifecycle."""
def test_create_attack_paths_scan_first_scan_defaults_to_false(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import create_attack_paths_scan
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
attack_paths_scan = create_attack_paths_scan(
str(tenant.id), str(scan.id), provider.id
)
assert attack_paths_scan is not None
assert attack_paths_scan.graph_data_ready is False
def test_create_attack_paths_scan_inherits_true_from_previous(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import create_attack_paths_scan
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.COMPLETED,
graph_data_ready=True,
)
new_scan = Scan.objects.create(
name="New Scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
tenant_id=tenant.id,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
attack_paths_scan = create_attack_paths_scan(
str(tenant.id), str(new_scan.id), provider.id
)
assert attack_paths_scan is not None
assert attack_paths_scan.graph_data_ready is True
def test_create_attack_paths_scan_inherits_false_when_no_previous_ready(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import create_attack_paths_scan
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.FAILED,
graph_data_ready=False,
)
new_scan = Scan.objects.create(
name="New Scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
tenant_id=tenant.id,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
attack_paths_scan = create_attack_paths_scan(
str(tenant.id), str(new_scan.id), provider.id
)
assert attack_paths_scan is not None
assert attack_paths_scan.graph_data_ready is False
def test_set_graph_data_ready_updates_field(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import set_graph_data_ready
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
attack_paths_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.EXECUTING,
graph_data_ready=True,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
set_graph_data_ready(attack_paths_scan, False)
attack_paths_scan.refresh_from_db()
assert attack_paths_scan.graph_data_ready is False
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
set_graph_data_ready(attack_paths_scan, True)
attack_paths_scan.refresh_from_db()
assert attack_paths_scan.graph_data_ready is True
def test_finish_attack_paths_scan_does_not_modify_graph_data_ready(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import finish_attack_paths_scan
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
attack_paths_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.EXECUTING,
graph_data_ready=True,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
finish_attack_paths_scan(attack_paths_scan, StateChoices.COMPLETED, {})
attack_paths_scan.refresh_from_db()
assert attack_paths_scan.state == StateChoices.COMPLETED
assert attack_paths_scan.graph_data_ready is True
def test_finish_attack_paths_scan_preserves_graph_data_ready_on_failure(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import finish_attack_paths_scan
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
attack_paths_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.EXECUTING,
graph_data_ready=True,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
finish_attack_paths_scan(
attack_paths_scan,
StateChoices.FAILED,
{"global_error": "boom"},
)
attack_paths_scan.refresh_from_db()
assert attack_paths_scan.state == StateChoices.FAILED
assert attack_paths_scan.graph_data_ready is True
def test_set_provider_graph_data_ready_updates_all_scans_for_provider(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import set_provider_graph_data_ready
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan_a = scans_fixture[0]
scan_a.provider = provider
scan_a.save()
scan_b = Scan.objects.create(
name="Second Scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
tenant_id=tenant.id,
)
old_ap_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan_a,
state=StateChoices.COMPLETED,
graph_data_ready=True,
)
new_ap_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan_b,
state=StateChoices.EXECUTING,
graph_data_ready=True,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
set_provider_graph_data_ready(new_ap_scan, False)
old_ap_scan.refresh_from_db()
new_ap_scan.refresh_from_db()
assert old_ap_scan.graph_data_ready is False
assert new_ap_scan.graph_data_ready is False
def test_set_provider_graph_data_ready_does_not_affect_other_providers(
self, tenants_fixture, providers_fixture, scans_fixture
):
from tasks.jobs.attack_paths.db_utils import set_provider_graph_data_ready
tenant = tenants_fixture[0]
provider_a = providers_fixture[0]
provider_a.provider = Provider.ProviderChoices.AWS
provider_a.save()
provider_b = providers_fixture[1]
provider_b.provider = Provider.ProviderChoices.AWS
provider_b.save()
scan_a = scans_fixture[0]
scan_a.provider = provider_a
scan_a.save()
scan_b = Scan.objects.create(
name="Scan for provider B",
provider=provider_b,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant_id=tenant.id,
)
ap_scan_a = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider_a,
scan=scan_a,
state=StateChoices.EXECUTING,
graph_data_ready=True,
)
ap_scan_b = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider_b,
scan=scan_b,
state=StateChoices.COMPLETED,
graph_data_ready=True,
)
with patch(
"tasks.jobs.attack_paths.db_utils.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
set_provider_graph_data_ready(ap_scan_a, False)
ap_scan_a.refresh_from_db()
ap_scan_b.refresh_from_db()
assert ap_scan_a.graph_data_ready is False
assert ap_scan_b.graph_data_ready is True