mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-10-28 06:14:20 +00:00
2470 lines
101 KiB
Python
2470 lines
101 KiB
Python
import csv
|
||
import io
|
||
import json
|
||
import re
|
||
import time
|
||
import uuid
|
||
from collections import defaultdict
|
||
from collections.abc import Iterable
|
||
from datetime import UTC, datetime
|
||
from typing import Any
|
||
|
||
import sentry_sdk
|
||
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||
from api.constants import SEVERITY_ORDER
|
||
from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||
from api.db_utils import (
|
||
POSTGRES_TENANT_VAR,
|
||
SET_CONFIG_QUERY,
|
||
psycopg_connection,
|
||
rls_transaction,
|
||
)
|
||
from api.exceptions import ProviderConnectionError
|
||
from api.models import (
|
||
AttackSurfaceOverview,
|
||
ComplianceOverviewSummary,
|
||
ComplianceRequirementOverview,
|
||
DailySeveritySummary,
|
||
Finding,
|
||
FindingGroupDailySummary,
|
||
MuteRule,
|
||
Processor,
|
||
Provider,
|
||
Resource,
|
||
ResourceFindingMapping,
|
||
ResourceScanSummary,
|
||
ResourceTag,
|
||
ResourceTagMapping,
|
||
Scan,
|
||
ScanCategorySummary,
|
||
ScanGroupSummary,
|
||
ScanSummary,
|
||
StateChoices,
|
||
)
|
||
from api.models import StatusChoices as FindingStatus
|
||
from api.utils import initialize_prowler_provider, return_prowler_provider
|
||
from api.v1.serializers import ScanTaskSerializer
|
||
from celery.utils.log import get_task_logger
|
||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||
from config.env import env
|
||
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
|
||
from django.db import IntegrityError, OperationalError
|
||
from django.db.models import (
|
||
Case,
|
||
Count,
|
||
Exists,
|
||
IntegerField,
|
||
Max,
|
||
Min,
|
||
OuterRef,
|
||
Q,
|
||
Sum,
|
||
When,
|
||
)
|
||
from django.utils import timezone as django_timezone
|
||
from prowler.lib.check.models import CheckMetadata
|
||
from prowler.lib.outputs.finding import Finding as ProwlerFinding
|
||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||
from tasks.jobs.queries import (
|
||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||
)
|
||
from tasks.utils import CustomEncoder, batched
|
||
|
||
logger = get_task_logger(__name__)
|
||
|
||
# Column order must match `ComplianceRequirementOverview` schema in
|
||
# `api/models.py`. Keep this list minimal but sufficient to populate all
|
||
# non-nullable fields plus the counters we care about.
|
||
COMPLIANCE_REQUIREMENT_COPY_COLUMNS = (
|
||
"id",
|
||
"tenant_id",
|
||
"inserted_at",
|
||
"compliance_id",
|
||
"framework",
|
||
"version",
|
||
"description",
|
||
"region",
|
||
"requirement_id",
|
||
"requirement_status",
|
||
"passed_checks",
|
||
"failed_checks",
|
||
"total_checks",
|
||
"passed_findings",
|
||
"total_findings",
|
||
"scan_id",
|
||
)
|
||
# Controls how many findings we process per micro-batch before flushing to DB writes
|
||
FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=3000)
|
||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres.
|
||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=1000)
|
||
# Throttle scan progress persistence: minimum progress delta (fraction 0-1)
|
||
# between two persisted progress updates.
|
||
PROGRESS_THROTTLE_DELTA = env.float("DJANGO_SCAN_PROGRESS_THROTTLE_DELTA", default=0.01)
|
||
# Throttle scan progress persistence: maximum seconds without persisting progress
|
||
# regardless of delta (so slow checks still show progress in the UI).
|
||
PROGRESS_THROTTLE_SECONDS = env.float(
|
||
"DJANGO_SCAN_PROGRESS_THROTTLE_SECONDS", default=10.0
|
||
)
|
||
|
||
ATTACK_SURFACE_PROVIDER_COMPATIBILITY = {
|
||
"internet-exposed": None, # Compatible with all providers
|
||
"secrets": None, # Compatible with all providers
|
||
"privilege-escalation": ["aws", "kubernetes"],
|
||
"ec2-imdsv1": ["aws"],
|
||
}
|
||
|
||
_ATTACK_SURFACE_MAPPING_CACHE: dict[str, dict] = {}
|
||
|
||
|
||
def aggregate_category_counts(
|
||
categories: list[str],
|
||
severity: str,
|
||
status: str,
|
||
delta: str | None,
|
||
muted: bool,
|
||
cache: dict[tuple[str, str], dict[str, int]],
|
||
) -> None:
|
||
"""
|
||
Increment category counters in-place for a finding.
|
||
|
||
Args:
|
||
categories: List of categories from finding metadata.
|
||
severity: Severity level (e.g., "high", "medium").
|
||
status: Finding status as string ("FAIL", "PASS").
|
||
delta: Delta value as string ("new", "changed") or None.
|
||
muted: Whether the finding is muted.
|
||
cache: Dict {(category, severity): {"total", "failed", "new_failed"}} to update.
|
||
"""
|
||
is_failed = status == "FAIL" and not muted
|
||
is_new_failed = is_failed and delta == "new"
|
||
|
||
for cat in categories:
|
||
key = (cat, severity)
|
||
if key not in cache:
|
||
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
|
||
if not muted:
|
||
cache[key]["total"] += 1
|
||
if is_failed:
|
||
cache[key]["failed"] += 1
|
||
if is_new_failed:
|
||
cache[key]["new_failed"] += 1
|
||
|
||
|
||
def aggregate_resource_group_counts(
|
||
resource_group: str | None,
|
||
severity: str,
|
||
status: str,
|
||
delta: str | None,
|
||
muted: bool,
|
||
resource_uid: str,
|
||
cache: dict[tuple[str, str], dict[str, int]],
|
||
group_resources_cache: dict[str, set],
|
||
) -> None:
|
||
"""
|
||
Increment resource group counters in-place for a finding.
|
||
|
||
Args:
|
||
resource_group: Resource group from check metadata (e.g., "database", "compute").
|
||
severity: Severity level (e.g., "high", "medium").
|
||
status: Finding status as string ("FAIL", "PASS").
|
||
delta: Delta value as string ("new", "changed") or None.
|
||
muted: Whether the finding is muted.
|
||
resource_uid: Unique identifier for the resource to count distinct resources.
|
||
cache: Dict {(resource_group, severity): {"total", "failed", "new_failed"}} to update.
|
||
group_resources_cache: Dict {resource_group: set(resource_uids)} for group-level resource tracking.
|
||
"""
|
||
if not resource_group:
|
||
return
|
||
|
||
is_failed = status == "FAIL" and not muted
|
||
is_new_failed = is_failed and delta == "new"
|
||
|
||
key = (resource_group, severity)
|
||
if key not in cache:
|
||
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
|
||
if not muted:
|
||
cache[key]["total"] += 1
|
||
if is_failed:
|
||
cache[key]["failed"] += 1
|
||
if is_new_failed:
|
||
cache[key]["new_failed"] += 1
|
||
|
||
# Track resources at GROUP level (not per-severity) to avoid over-counting
|
||
if resource_uid and not muted:
|
||
group_resources_cache.setdefault(resource_group, set()).add(resource_uid)
|
||
|
||
|
||
def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
|
||
global _ATTACK_SURFACE_MAPPING_CACHE
|
||
|
||
if provider_type in _ATTACK_SURFACE_MAPPING_CACHE:
|
||
return _ATTACK_SURFACE_MAPPING_CACHE[provider_type]
|
||
|
||
attack_surface_check_mappings = {
|
||
"internet-exposed": None,
|
||
"secrets": None,
|
||
"privilege-escalation": {
|
||
"iam_policy_allows_privilege_escalation",
|
||
"iam_inline_policy_allows_privilege_escalation",
|
||
},
|
||
"ec2-imdsv1": {
|
||
"ec2_instance_imdsv2_enabled",
|
||
"ec2_instance_account_imdsv2_enabled",
|
||
}, # AWS only - instance-level IMDSv1 exposure and account IMDS defaults
|
||
}
|
||
for category_name, check_ids in attack_surface_check_mappings.items():
|
||
if check_ids is None:
|
||
sdk_check_ids = CheckMetadata.list(
|
||
provider=provider_type, category=category_name
|
||
)
|
||
attack_surface_check_mappings[category_name] = sdk_check_ids
|
||
|
||
_ATTACK_SURFACE_MAPPING_CACHE[provider_type] = attack_surface_check_mappings
|
||
return attack_surface_check_mappings
|
||
|
||
|
||
def _create_finding_delta(
|
||
last_status: FindingStatus | None | str, new_status: FindingStatus | None
|
||
) -> Finding.DeltaChoices:
|
||
"""
|
||
Determine the delta status of a finding based on its previous and current status.
|
||
|
||
Args:
|
||
last_status (FindingStatus | None | str): The previous status of the finding. Can be None or a string representation.
|
||
new_status (FindingStatus | None): The current status of the finding.
|
||
|
||
Returns:
|
||
Finding.DeltaChoices: The delta status indicating if the finding is new, changed, or unchanged.
|
||
- Returns `Finding.DeltaChoices.NEW` if `last_status` is None.
|
||
- Returns `Finding.DeltaChoices.CHANGED` if `last_status` and `new_status` are different.
|
||
- Returns `None` if the status hasn't changed.
|
||
"""
|
||
if last_status is None:
|
||
return Finding.DeltaChoices.NEW
|
||
return Finding.DeltaChoices.CHANGED if last_status != new_status else None
|
||
|
||
|
||
def _store_resources(
|
||
finding: ProwlerFinding, tenant_id: str, provider_instance: Provider
|
||
) -> tuple[Resource, tuple[str, str]]:
|
||
"""
|
||
Store resource information from a finding, including tags, in the database.
|
||
|
||
Args:
|
||
finding (ProwlerFinding): The finding object containing resource information.
|
||
tenant_id (str): The ID of the tenant owning the resource.
|
||
provider_instance (Provider): The provider instance associated with the resource.
|
||
|
||
Returns:
|
||
tuple:
|
||
- Resource: The resource instance created or retrieved from the database.
|
||
- tuple[str, str]: A tuple containing the resource UID and region.
|
||
|
||
"""
|
||
with rls_transaction(tenant_id):
|
||
resource_instance, created = Resource.objects.get_or_create(
|
||
tenant_id=tenant_id,
|
||
provider=provider_instance,
|
||
uid=finding.resource_uid,
|
||
defaults={
|
||
"name": finding.resource_name,
|
||
"region": finding.region,
|
||
"service": finding.service_name,
|
||
"type": finding.resource_type,
|
||
},
|
||
)
|
||
|
||
if not created:
|
||
resource_instance.name = finding.resource_name
|
||
resource_instance.region = finding.region
|
||
resource_instance.service = finding.service_name
|
||
resource_instance.type = finding.resource_type
|
||
resource_instance.save()
|
||
with rls_transaction(tenant_id):
|
||
tags = [
|
||
ResourceTag.objects.get_or_create(
|
||
tenant_id=tenant_id, key=key, value=value
|
||
)[0]
|
||
for key, value in finding.resource_tags.items()
|
||
]
|
||
resource_instance.upsert_or_delete_tags(tags=tags)
|
||
return resource_instance, (resource_instance.uid, resource_instance.region)
|
||
|
||
|
||
def _copy_compliance_requirement_rows(
|
||
tenant_id: str, rows: list[dict[str, Any]]
|
||
) -> None:
|
||
"""Stream compliance requirement rows into Postgres using COPY.
|
||
|
||
We leverage the admin connection (when available) to bypass the COPY + RLS
|
||
restriction, writing only the fields required by
|
||
``ComplianceRequirementOverview``.
|
||
|
||
Args:
|
||
tenant_id: Target tenant UUID.
|
||
rows: List of row dictionaries prepared by
|
||
:func:`create_compliance_requirements`.
|
||
"""
|
||
|
||
csv_buffer = io.StringIO()
|
||
writer = csv.writer(csv_buffer)
|
||
|
||
datetime_now = datetime.now(tz=UTC)
|
||
for row in rows:
|
||
writer.writerow(
|
||
[
|
||
str(row.get("id")),
|
||
str(row.get("tenant_id")),
|
||
(row.get("inserted_at") or datetime_now).isoformat(),
|
||
row.get("compliance_id") or "",
|
||
row.get("framework") or "",
|
||
row.get("version") or "",
|
||
row.get("description") or "",
|
||
row.get("region") or "",
|
||
row.get("requirement_id") or "",
|
||
row.get("requirement_status") or "",
|
||
row.get("passed_checks", 0),
|
||
row.get("failed_checks", 0),
|
||
row.get("total_checks", 0),
|
||
row.get("passed_findings", 0),
|
||
row.get("total_findings", 0),
|
||
str(row.get("scan_id")),
|
||
]
|
||
)
|
||
|
||
csv_buffer.seek(0)
|
||
copy_sql = (
|
||
"COPY compliance_requirements_overviews ("
|
||
+ ", ".join(COMPLIANCE_REQUIREMENT_COPY_COLUMNS)
|
||
+ ") FROM STDIN WITH (FORMAT CSV, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '\\N')"
|
||
)
|
||
|
||
try:
|
||
with psycopg_connection(MainRouter.admin_db) as connection:
|
||
connection.autocommit = False
|
||
try:
|
||
with connection.cursor() as cursor:
|
||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||
cursor.copy_expert(copy_sql, csv_buffer)
|
||
connection.commit()
|
||
except Exception:
|
||
connection.rollback()
|
||
raise
|
||
finally:
|
||
csv_buffer.close()
|
||
|
||
|
||
def _persist_compliance_requirement_rows(
|
||
tenant_id: str, rows: Iterable[dict[str, Any]], batch_size: int = 10000
|
||
) -> int:
|
||
"""Persist compliance requirement rows using batched COPY with ORM fallback.
|
||
|
||
``rows`` is consumed lazily in batches, so peak memory stays at ~``batch_size``
|
||
rows instead of the full set. A batch that fails COPY falls back to an ORM
|
||
``bulk_create`` of just that batch.
|
||
|
||
Args:
|
||
tenant_id: Target tenant UUID.
|
||
rows: Iterable of row dictionaries reflecting the compliance overview
|
||
state for a scan.
|
||
batch_size: Number of rows per COPY batch (default: 10000).
|
||
|
||
Returns:
|
||
int: total number of rows persisted.
|
||
"""
|
||
total_rows = 0
|
||
batch_num = 0
|
||
|
||
for batch, _is_last in batched(rows, batch_size):
|
||
if not batch:
|
||
continue
|
||
batch_num += 1
|
||
try:
|
||
_copy_compliance_requirement_rows(tenant_id, batch)
|
||
except Exception as error:
|
||
logger.exception(
|
||
f"COPY bulk insert for compliance requirements batch {batch_num} "
|
||
"failed; falling back to ORM bulk_create for this batch",
|
||
exc_info=error,
|
||
)
|
||
fallback_objects = [
|
||
ComplianceRequirementOverview(
|
||
id=row["id"],
|
||
tenant_id=row["tenant_id"],
|
||
inserted_at=row["inserted_at"],
|
||
compliance_id=row["compliance_id"],
|
||
framework=row["framework"],
|
||
version=row["version"],
|
||
description=row["description"],
|
||
region=row["region"],
|
||
requirement_id=row["requirement_id"],
|
||
requirement_status=row["requirement_status"],
|
||
passed_checks=row["passed_checks"],
|
||
failed_checks=row["failed_checks"],
|
||
total_checks=row["total_checks"],
|
||
passed_findings=row.get("passed_findings", 0),
|
||
total_findings=row.get("total_findings", 0),
|
||
scan_id=row["scan_id"],
|
||
)
|
||
for row in batch
|
||
]
|
||
with rls_transaction(tenant_id):
|
||
ComplianceRequirementOverview.objects.bulk_create(
|
||
fallback_objects, batch_size=500
|
||
)
|
||
|
||
total_rows += len(batch)
|
||
logger.info(
|
||
f"Compliance COPY batch {batch_num}: inserted {len(batch)} rows "
|
||
f"({total_rows} total)"
|
||
)
|
||
|
||
return total_rows
|
||
|
||
|
||
def _create_compliance_summaries(
|
||
tenant_id: str, scan_id: str, requirement_statuses: dict
|
||
) -> None:
|
||
"""
|
||
Create pre-aggregated compliance summaries from pre-computed requirement statuses.
|
||
|
||
This computes the overall compliance scores across all regions for fast
|
||
lookup in the main compliance overview endpoint.
|
||
|
||
Args:
|
||
tenant_id: Target tenant UUID
|
||
scan_id: Scan UUID
|
||
requirement_statuses: Pre-computed dict of {(compliance_id, requirement_id): {fail_count, pass_count, total_count}}
|
||
"""
|
||
# Determine per-requirement status and aggregate to compliance level
|
||
compliance_summaries = defaultdict(
|
||
lambda: {
|
||
"total_requirements": 0,
|
||
"requirements_passed": 0,
|
||
"requirements_failed": 0,
|
||
"requirements_manual": 0,
|
||
}
|
||
)
|
||
|
||
for (compliance_id, requirement_id), counts in requirement_statuses.items():
|
||
# Apply business rule: any FAIL → requirement fails
|
||
if counts["fail_count"] > 0:
|
||
req_status = "FAIL"
|
||
elif counts["pass_count"] == counts["total_count"]:
|
||
req_status = "PASS"
|
||
else:
|
||
req_status = "MANUAL"
|
||
|
||
# Aggregate to compliance level
|
||
compliance_summaries[compliance_id]["total_requirements"] += 1
|
||
if req_status == "PASS":
|
||
compliance_summaries[compliance_id]["requirements_passed"] += 1
|
||
elif req_status == "FAIL":
|
||
compliance_summaries[compliance_id]["requirements_failed"] += 1
|
||
else:
|
||
compliance_summaries[compliance_id]["requirements_manual"] += 1
|
||
|
||
# Create summary objects
|
||
summary_objects = []
|
||
for compliance_id, data in compliance_summaries.items():
|
||
summary_objects.append(
|
||
ComplianceOverviewSummary(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
compliance_id=compliance_id,
|
||
requirements_passed=data["requirements_passed"],
|
||
requirements_failed=data["requirements_failed"],
|
||
requirements_manual=data["requirements_manual"],
|
||
total_requirements=data["total_requirements"],
|
||
)
|
||
)
|
||
|
||
# Idempotent re-run: clear this scan's prior summaries before re-inserting, so a
|
||
# recovered scan-compliance-overviews run reflects its own re-derived rows instead
|
||
# of keeping a stale one (bulk_create ignore_conflicts alone would keep the old).
|
||
with rls_transaction(tenant_id):
|
||
ComplianceOverviewSummary.objects.filter(scan_id=scan_id).delete()
|
||
if summary_objects:
|
||
ComplianceOverviewSummary.objects.bulk_create(
|
||
summary_objects, batch_size=500, ignore_conflicts=True
|
||
)
|
||
|
||
|
||
def _normalized_compliance_key(framework: str | None, version: str | None) -> str:
|
||
"""Return normalized identifier used to group compliance totals."""
|
||
|
||
def _normalize(value: str | None) -> str:
|
||
if not value:
|
||
return ""
|
||
return re.sub(r"[^a-z0-9]", "", value.lower())
|
||
|
||
return f"{_normalize(framework)}{_normalize(version)}"
|
||
|
||
|
||
def _process_finding_micro_batch(
|
||
tenant_id: str,
|
||
findings_batch: list[ProwlerFinding],
|
||
scan_instance: Scan,
|
||
provider_instance: Provider,
|
||
resource_cache: dict,
|
||
tag_cache: dict,
|
||
last_status_cache: dict,
|
||
resource_failed_findings_cache: dict,
|
||
unique_resources: set,
|
||
scan_resource_cache: set,
|
||
mute_rules_cache: dict,
|
||
scan_categories_cache: dict[tuple[str, str], dict[str, int]],
|
||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]],
|
||
group_resources_cache: dict[str, set],
|
||
) -> None:
|
||
"""
|
||
Process a micro-batch of findings and persist them using bulk operations.
|
||
|
||
Each batch reuses caches (resources, tags, last statuses, mute rules) to avoid
|
||
redundant queries, updates denormalized resource data, and writes findings plus
|
||
resource mappings in as few transactions as possible.
|
||
|
||
Args:
|
||
tenant_id: Tenant owning the scan.
|
||
findings_batch: Findings yielded by the Prowler scanner for this slice.
|
||
scan_instance: Scan ORM instance being updated.
|
||
provider_instance: Provider tied to the scan.
|
||
resource_cache: In-memory cache of provider resources indexed by UID.
|
||
tag_cache: Cache of `ResourceTag` instances keyed by (key, value).
|
||
last_status_cache: Cache of prior finding statuses keyed by finding UID.
|
||
resource_failed_findings_cache: Mutable counter of failed findings per resource.
|
||
unique_resources: Set tracking (uid, region) pairs seen in the scan.
|
||
scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
|
||
mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
|
||
scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
|
||
scan_resource_groups_cache: Dict tracking resource group counts {(resource_group, severity): {"total", "failed", "new_failed"}}.
|
||
group_resources_cache: Dict tracking unique resources per group {resource_group: set(resource_uids)}.
|
||
"""
|
||
# Accumulate objects for bulk operations
|
||
findings_to_create = []
|
||
dirty_resources = {}
|
||
resources_with_new_tag_mappings: set[str] = set()
|
||
resource_denormalized_data = [] # (finding_instance, resource_instance) pairs
|
||
tag_mappings_to_create: list[ResourceTagMapping] = []
|
||
skipped_findings_count = 0 # Track findings skipped due to UID length
|
||
|
||
# Separate findings into those persistable (uid <= 300) and over-limit.
|
||
# Resources/tags ARE still resolved for over-limit findings to preserve the
|
||
# original behavior (resources are persisted even when their finding is dropped).
|
||
non_null_findings = [f for f in findings_batch if f is not None]
|
||
persistable_findings = [f for f in non_null_findings if len(f.uid) <= 300]
|
||
skipped_findings_count = len(non_null_findings) - len(persistable_findings)
|
||
none_count = len(findings_batch) - len(non_null_findings)
|
||
if none_count:
|
||
logger.error(
|
||
f"{none_count} None finding(s) detected on scan {scan_instance.id}."
|
||
)
|
||
|
||
# Prefetch last statuses for all persistable findings in this batch (read replica)
|
||
finding_uids = [f.uid for f in persistable_findings]
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
last_statuses = {
|
||
item["uid"]: (item["status"], item["first_seen_at"])
|
||
for item in Finding.all_objects.filter(
|
||
tenant_id=tenant_id, uid__in=finding_uids
|
||
)
|
||
.values("uid", "status", "first_seen_at")
|
||
.order_by("uid", "-inserted_at")
|
||
.distinct("uid")
|
||
}
|
||
for uid, data in last_statuses.items():
|
||
if uid not in last_status_cache:
|
||
last_status_cache[uid] = data
|
||
|
||
# All DB writes for this micro-batch run inside ONE rls_transaction,
|
||
# with deadlock-retry at micro-batch granularity instead of per-finding.
|
||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||
try:
|
||
with rls_transaction(tenant_id):
|
||
# 1) Pre-resolve Resources in bulk
|
||
# Collect all uids referenced by this batch that are not in cache yet.
|
||
# NOTE: we intentionally include empty-string uids here. The SDK
|
||
# explicitly emits findings with `resource_uid=""` for some flows
|
||
# (IaC scans, some Azure/GCP/K8s checks). The original
|
||
# `get_or_create` behavior was to create/share a Resource with
|
||
# uid="" for these findings rather than dropping them. Preserve
|
||
# that behavior; do NOT filter by truthiness.
|
||
batch_resource_uids: set[str] = set()
|
||
for f in non_null_findings:
|
||
if f.resource_uid not in resource_cache:
|
||
batch_resource_uids.add(f.resource_uid)
|
||
|
||
if batch_resource_uids:
|
||
existing_resources = {
|
||
r.uid: r
|
||
for r in Resource.objects.filter(
|
||
tenant_id=tenant_id,
|
||
provider_id=provider_instance.id,
|
||
uid__in=batch_resource_uids,
|
||
)
|
||
}
|
||
missing_uids = batch_resource_uids - existing_resources.keys()
|
||
if missing_uids:
|
||
# Build defaults from the first finding referencing each uid.
|
||
first_finding_per_uid: dict[str, ProwlerFinding] = {}
|
||
for f in non_null_findings:
|
||
if f.resource_uid in missing_uids:
|
||
first_finding_per_uid.setdefault(f.resource_uid, f)
|
||
resources_to_create = []
|
||
for uid in missing_uids:
|
||
f = first_finding_per_uid[uid]
|
||
check_metadata = f.get_metadata()
|
||
group = check_metadata.get("resourcegroup") or None
|
||
resources_to_create.append(
|
||
Resource(
|
||
tenant_id=tenant_id,
|
||
provider=provider_instance,
|
||
uid=uid,
|
||
region=f.region,
|
||
service=f.service_name,
|
||
type=f.resource_type,
|
||
name=f.resource_name,
|
||
groups=[group] if group else None,
|
||
)
|
||
)
|
||
Resource.objects.bulk_create(
|
||
resources_to_create,
|
||
batch_size=SCAN_DB_BATCH_SIZE,
|
||
ignore_conflicts=True,
|
||
unique_fields=["tenant_id", "provider_id", "uid"],
|
||
)
|
||
# Re-fetch to obtain instances we just created AND any
|
||
# created concurrently by another scan against the same provider.
|
||
existing_resources.update(
|
||
{
|
||
r.uid: r
|
||
for r in Resource.objects.filter(
|
||
tenant_id=tenant_id,
|
||
provider_id=provider_instance.id,
|
||
uid__in=missing_uids,
|
||
)
|
||
}
|
||
)
|
||
for uid, r in existing_resources.items():
|
||
resource_cache[uid] = r
|
||
resource_failed_findings_cache.setdefault(uid, 0)
|
||
|
||
# 2) Pre-resolve ResourceTags in bulk
|
||
batch_tag_kv: set[tuple[str, str]] = set()
|
||
for f in non_null_findings:
|
||
for k, v in f.resource_tags.items():
|
||
if (k, v) not in tag_cache:
|
||
batch_tag_kv.add((k, v))
|
||
|
||
if batch_tag_kv:
|
||
keys_to_query = {k for k, _ in batch_tag_kv}
|
||
existing_tags = {
|
||
(t.key, t.value): t
|
||
for t in ResourceTag.objects.filter(
|
||
tenant_id=tenant_id, key__in=keys_to_query
|
||
)
|
||
if (t.key, t.value) in batch_tag_kv
|
||
}
|
||
missing_kv = batch_tag_kv - existing_tags.keys()
|
||
if missing_kv:
|
||
ResourceTag.objects.bulk_create(
|
||
[
|
||
ResourceTag(tenant_id=tenant_id, key=k, value=v)
|
||
for k, v in missing_kv
|
||
],
|
||
batch_size=SCAN_DB_BATCH_SIZE,
|
||
ignore_conflicts=True,
|
||
unique_fields=["tenant_id", "key", "value"],
|
||
)
|
||
existing_tags.update(
|
||
{
|
||
(t.key, t.value): t
|
||
for t in ResourceTag.objects.filter(
|
||
tenant_id=tenant_id,
|
||
key__in={k for k, _ in missing_kv},
|
||
)
|
||
if (t.key, t.value) in missing_kv
|
||
}
|
||
)
|
||
tag_cache.update(existing_tags)
|
||
|
||
# 3) Per-finding in-memory processing
|
||
for finding in non_null_findings:
|
||
resource_uid = finding.resource_uid
|
||
resource_instance = resource_cache.get(resource_uid)
|
||
if resource_instance is None:
|
||
# Should be unreachable after the pre-resolve step. Defensive log.
|
||
logger.error(
|
||
f"Resource {resource_uid} missing from cache after pre-resolve "
|
||
f"on scan {scan_instance.id}; skipping finding."
|
||
)
|
||
continue
|
||
|
||
# Detect resource field changes (defer save until end-of-batch bulk_update).
|
||
check_metadata = finding.get_metadata()
|
||
group = check_metadata.get("resourcegroup") or None
|
||
updated = False
|
||
if finding.region and resource_instance.region != finding.region:
|
||
resource_instance.region = finding.region
|
||
updated = True
|
||
if (
|
||
finding.resource_name
|
||
and resource_instance.name != finding.resource_name
|
||
):
|
||
resource_instance.name = finding.resource_name
|
||
updated = True
|
||
if resource_instance.service != finding.service_name:
|
||
resource_instance.service = finding.service_name
|
||
updated = True
|
||
if resource_instance.type != finding.resource_type:
|
||
resource_instance.type = finding.resource_type
|
||
updated = True
|
||
if resource_instance.metadata != finding.resource_metadata:
|
||
resource_instance.metadata = json.dumps(
|
||
finding.resource_metadata, cls=CustomEncoder
|
||
)
|
||
updated = True
|
||
if resource_instance.details != finding.resource_details:
|
||
resource_instance.details = finding.resource_details
|
||
updated = True
|
||
if resource_instance.partition != finding.partition:
|
||
resource_instance.partition = finding.partition
|
||
updated = True
|
||
if group and (
|
||
not resource_instance.groups
|
||
or group not in resource_instance.groups
|
||
):
|
||
resource_instance.groups = (resource_instance.groups or []) + [
|
||
group
|
||
]
|
||
updated = True
|
||
|
||
if updated:
|
||
dirty_resources[resource_uid] = resource_instance
|
||
|
||
# Accumulate ResourceTagMapping rows; bulk_create at end of block.
|
||
for k, v in finding.resource_tags.items():
|
||
tag_instance = tag_cache.get((k, v))
|
||
if tag_instance is None:
|
||
# Should not happen after pre-resolve; skip defensively.
|
||
continue
|
||
tag_mappings_to_create.append(
|
||
ResourceTagMapping(
|
||
tenant_id=tenant_id,
|
||
resource=resource_instance,
|
||
tag=tag_instance,
|
||
)
|
||
)
|
||
|
||
unique_resources.add(
|
||
(resource_instance.uid, resource_instance.region)
|
||
)
|
||
|
||
# TEMPORARY WORKAROUND: Skip findings with UID > 300 chars
|
||
# TODO: Remove this after implementing text field migration for finding.uid
|
||
if len(finding.uid) > 300:
|
||
logger.warning(
|
||
f"Skipping finding with UID exceeding 300 characters. "
|
||
f"Length: {len(finding.uid)}, "
|
||
f"Check: {finding.check_id}, "
|
||
f"Resource: {finding.resource_name}, "
|
||
f"UID: {finding.uid}"
|
||
)
|
||
continue
|
||
|
||
finding_uid = finding.uid
|
||
last_status, last_first_seen_at = last_status_cache.get(
|
||
finding_uid, (None, None)
|
||
)
|
||
|
||
status = FindingStatus[finding.status]
|
||
delta = _create_finding_delta(last_status, status)
|
||
|
||
if not last_first_seen_at:
|
||
last_first_seen_at = datetime.now(tz=UTC)
|
||
|
||
# Determine if finding should be muted and why
|
||
# Priority: mutelist processor (highest) > manual mute rules
|
||
is_muted = False
|
||
muted_reason = None
|
||
if finding.muted:
|
||
is_muted = True
|
||
muted_reason = "Muted by mutelist"
|
||
elif finding_uid in mute_rules_cache:
|
||
is_muted = True
|
||
muted_reason = mute_rules_cache[finding_uid]
|
||
|
||
if status == FindingStatus.FAIL and not is_muted:
|
||
resource_failed_findings_cache[resource_uid] += 1
|
||
|
||
check_metadata["compliance"] = finding.compliance
|
||
finding_instance = Finding(
|
||
tenant_id=tenant_id,
|
||
uid=finding_uid,
|
||
delta=delta,
|
||
check_metadata=check_metadata,
|
||
status=status,
|
||
status_extended=finding.status_extended,
|
||
severity=finding.severity,
|
||
impact=finding.severity,
|
||
raw_result=finding.raw,
|
||
check_id=finding.check_id,
|
||
scan=scan_instance,
|
||
first_seen_at=last_first_seen_at,
|
||
muted=is_muted,
|
||
muted_at=datetime.now(tz=UTC) if is_muted else None,
|
||
muted_reason=muted_reason,
|
||
compliance=finding.compliance,
|
||
categories=check_metadata.get("categories", []) or [],
|
||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||
# Denormalized resource arrays populated directly on insert
|
||
# (was previously a separate bulk_update; saves a CASE WHEN
|
||
# over thousands of rows per micro-batch).
|
||
resource_regions=[resource_instance.region]
|
||
if resource_instance.region
|
||
else [],
|
||
resource_services=[resource_instance.service]
|
||
if resource_instance.service
|
||
else [],
|
||
resource_types=[resource_instance.type]
|
||
if resource_instance.type
|
||
else [],
|
||
)
|
||
findings_to_create.append(finding_instance)
|
||
resource_denormalized_data.append(
|
||
(finding_instance, resource_instance)
|
||
)
|
||
|
||
scan_resource_cache.add(
|
||
(
|
||
str(resource_instance.id),
|
||
resource_instance.service,
|
||
resource_instance.region,
|
||
resource_instance.type,
|
||
)
|
||
)
|
||
|
||
aggregate_category_counts(
|
||
categories=check_metadata.get("categories", []) or [],
|
||
severity=finding.severity.value,
|
||
status=status.value,
|
||
delta=delta.value if delta else None,
|
||
muted=is_muted,
|
||
cache=scan_categories_cache,
|
||
)
|
||
|
||
aggregate_resource_group_counts(
|
||
resource_group=check_metadata.get("resourcegroup") or None,
|
||
severity=finding.severity.value,
|
||
status=status.value,
|
||
delta=delta.value if delta else None,
|
||
muted=is_muted,
|
||
resource_uid=resource_instance.uid if resource_instance else "",
|
||
cache=scan_resource_groups_cache,
|
||
group_resources_cache=group_resources_cache,
|
||
)
|
||
|
||
# 4) Bulk create ResourceTagMappings
|
||
# Replaces the original per-resource `upsert_or_delete_tags`
|
||
# (which did one `update_or_create` + SELECT FOR UPDATE per mapping).
|
||
if tag_mappings_to_create:
|
||
# Pre-SELECT existing pairs: `bulk_create(ignore_conflicts=True)`
|
||
# does not populate `pk`, so we cannot tell new vs existing from
|
||
# the result; we need that to bump `updated_at` only on resources
|
||
# that actually gain a mapping.
|
||
candidate_resource_ids = {
|
||
m.resource_id for m in tag_mappings_to_create
|
||
}
|
||
candidate_tag_ids = {m.tag_id for m in tag_mappings_to_create}
|
||
existing_pairs = set(
|
||
ResourceTagMapping.objects.filter(
|
||
tenant_id=tenant_id,
|
||
resource_id__in=candidate_resource_ids,
|
||
tag_id__in=candidate_tag_ids,
|
||
).values_list("resource_id", "tag_id")
|
||
)
|
||
resource_uid_by_id = {
|
||
str(r.id): uid for uid, r in resource_cache.items()
|
||
}
|
||
for m in tag_mappings_to_create:
|
||
if (m.resource_id, m.tag_id) not in existing_pairs:
|
||
uid = resource_uid_by_id.get(str(m.resource_id))
|
||
if uid is not None:
|
||
resources_with_new_tag_mappings.add(uid)
|
||
|
||
ResourceTagMapping.objects.bulk_create(
|
||
tag_mappings_to_create,
|
||
batch_size=SCAN_DB_BATCH_SIZE,
|
||
ignore_conflicts=True,
|
||
unique_fields=["tenant_id", "resource_id", "tag_id"],
|
||
)
|
||
|
||
# 5) Bulk create Findings
|
||
if findings_to_create:
|
||
Finding.objects.bulk_create(
|
||
findings_to_create, batch_size=SCAN_DB_BATCH_SIZE
|
||
)
|
||
|
||
# 6) Bulk create ResourceFindingMapping rows
|
||
mappings_to_create = [
|
||
ResourceFindingMapping(
|
||
tenant_id=tenant_id,
|
||
resource=resource_instance,
|
||
finding=finding_instance,
|
||
)
|
||
for finding_instance, resource_instance in resource_denormalized_data
|
||
]
|
||
if mappings_to_create:
|
||
created_mappings = ResourceFindingMapping.objects.bulk_create(
|
||
mappings_to_create,
|
||
batch_size=SCAN_DB_BATCH_SIZE,
|
||
ignore_conflicts=True,
|
||
unique_fields=["tenant_id", "resource_id", "finding_id"],
|
||
)
|
||
inserted = sum(1 for m in created_mappings if m.pk)
|
||
if inserted != len(mappings_to_create):
|
||
logger.error(
|
||
f"scan {scan_instance.id}: expected "
|
||
f"{len(mappings_to_create)} ResourceFindingMapping rows, "
|
||
f"inserted {inserted}. Rolling back micro-batch."
|
||
)
|
||
|
||
# 7) Bulk update Resources
|
||
# Union of:
|
||
# - resources whose fields changed (dirty_resources)
|
||
# - resources that got new tag mappings (need updated_at bump,
|
||
# preserving the original `self.save(update_fields=["updated_at"])`
|
||
# behavior of `upsert_or_delete_tags`)
|
||
all_resource_uids_to_touch = (
|
||
set(dirty_resources.keys()) | resources_with_new_tag_mappings
|
||
)
|
||
if all_resource_uids_to_touch:
|
||
now_utc = datetime.now(tz=UTC)
|
||
resources_to_bulk_update = []
|
||
for uid in all_resource_uids_to_touch:
|
||
# Use the instance from dirty_resources if present (has mutated
|
||
# fields), otherwise the cached one (for updated_at bump only).
|
||
r = dirty_resources.get(uid) or resource_cache.get(uid)
|
||
if r is None:
|
||
continue
|
||
# Manually bump updated_at since bulk_update bypasses auto_now.
|
||
r.updated_at = now_utc
|
||
resources_to_bulk_update.append(r)
|
||
if resources_to_bulk_update:
|
||
Resource.objects.bulk_update(
|
||
resources_to_bulk_update,
|
||
[
|
||
"name",
|
||
"metadata",
|
||
"details",
|
||
"partition",
|
||
"region",
|
||
"service",
|
||
"type",
|
||
"groups",
|
||
"updated_at",
|
||
],
|
||
batch_size=1000,
|
||
)
|
||
# Successful execution: leave deadlock retry loop.
|
||
break
|
||
except (OperationalError, IntegrityError) as db_err:
|
||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||
logger.warning(
|
||
f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} "
|
||
f"on micro-batch for scan {scan_instance.id}. Retrying (attempt {attempt + 1})..."
|
||
)
|
||
time.sleep(0.1 * (2**attempt))
|
||
# Clear accumulators that we appended to inside the failed transaction
|
||
# so the retry produces consistent results.
|
||
findings_to_create.clear()
|
||
resource_denormalized_data.clear()
|
||
tag_mappings_to_create.clear()
|
||
dirty_resources.clear()
|
||
resources_with_new_tag_mappings.clear()
|
||
continue
|
||
raise
|
||
|
||
# Log skipped findings summary
|
||
if skipped_findings_count > 0:
|
||
logger.warning(
|
||
f"Scan {scan_instance.id}: Skipped {skipped_findings_count} finding(s) "
|
||
f"due to UID length exceeding 300 characters in this micro-batch."
|
||
)
|
||
|
||
|
||
def perform_prowler_scan(
|
||
tenant_id: str,
|
||
scan_id: str,
|
||
provider_id: str,
|
||
checks_to_execute: list[str] | None = None,
|
||
):
|
||
"""
|
||
Run a Prowler scan and persist all generated resources, findings, and summaries.
|
||
|
||
The scan stream is processed in micro-batches to keep memory bounded while still
|
||
benefiting from bulk writes. When the scan completes we also derive
|
||
`ResourceScanSummary` rows and return the serialized `Scan` payload used by the
|
||
API layer.
|
||
|
||
Args:
|
||
tenant_id: Tenant that owns the scan.
|
||
scan_id: UUID of the `Scan` row being executed.
|
||
provider_id: Provider to authenticate against.
|
||
checks_to_execute: Optional subset of check IDs to run.
|
||
|
||
Returns:
|
||
Serialized `ScanTaskSerializer` data for the updated scan.
|
||
|
||
Raises:
|
||
ProviderConnectionError: If the provider cannot be validated before scanning.
|
||
Exception: Any downstream persistence/processing error (re-raised after cleanup).
|
||
"""
|
||
exception = None
|
||
unique_resources = set()
|
||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||
group_resources_cache: dict[str, set] = {}
|
||
start_time = time.time()
|
||
exc = None
|
||
|
||
with rls_transaction(tenant_id):
|
||
provider_instance = Provider.objects.get(pk=provider_id)
|
||
scan_instance = Scan.objects.get(pk=scan_id)
|
||
scan_instance.state = StateChoices.EXECUTING
|
||
scan_instance.started_at = datetime.now(tz=UTC)
|
||
scan_instance.save(update_fields=["state", "started_at", "updated_at"])
|
||
|
||
# Find the mutelist processor if it exists
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
try:
|
||
mutelist_processor = Processor.objects.get(
|
||
tenant_id=tenant_id, processor_type=Processor.ProcessorChoices.MUTELIST
|
||
)
|
||
except Processor.DoesNotExist:
|
||
mutelist_processor = None
|
||
except Exception as e:
|
||
logger.error(f"Error processing mutelist rules: {e}")
|
||
mutelist_processor = None
|
||
|
||
# Load enabled mute rules for this tenant
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
try:
|
||
active_mute_rules = MuteRule.objects.filter(
|
||
tenant_id=tenant_id, enabled=True
|
||
).values_list("finding_uids", "reason")
|
||
|
||
mute_rules_cache = {}
|
||
for finding_uids, reason in active_mute_rules:
|
||
for uid in finding_uids:
|
||
mute_rules_cache[uid] = reason
|
||
except Exception as e:
|
||
logger.error(f"Error loading mute rules: {e}")
|
||
mute_rules_cache = {}
|
||
|
||
try:
|
||
with rls_transaction(tenant_id):
|
||
try:
|
||
prowler_provider = initialize_prowler_provider(
|
||
provider_instance, mutelist_processor
|
||
)
|
||
provider_instance.connected = True
|
||
except Exception as e:
|
||
provider_instance.connected = False
|
||
exc = ProviderConnectionError(
|
||
f"Provider {provider_instance.provider} is not connected: {e}"
|
||
)
|
||
finally:
|
||
provider_instance.connection_last_checked_at = datetime.now(tz=UTC)
|
||
provider_instance.save(
|
||
update_fields=[
|
||
"connected",
|
||
"connection_last_checked_at",
|
||
"updated_at",
|
||
]
|
||
)
|
||
|
||
# If the provider is not connected, raise an exception outside the transaction.
|
||
# If raised within the transaction, the transaction will be rolled back and the provider will not be marked
|
||
# as not connected.
|
||
if exc:
|
||
raise exc
|
||
|
||
prowler_scan = ProwlerScan(provider=prowler_provider, checks=checks_to_execute)
|
||
|
||
resource_cache = {}
|
||
tag_cache = {}
|
||
last_status_cache = {}
|
||
resource_failed_findings_cache = defaultdict(int)
|
||
|
||
# Throttle scan_instance progress writes to avoid hammering the writer:
|
||
# only persist when progress moves by at least `PROGRESS_THROTTLE_DELTA`
|
||
# OR `PROGRESS_THROTTLE_SECONDS` have elapsed. The final progress (1.0)
|
||
# always persists in the `finally` block below.
|
||
last_persisted_progress = -1.0
|
||
last_persisted_progress_at = 0.0
|
||
|
||
for progress, findings in prowler_scan.scan():
|
||
# Process findings in micro-batches
|
||
findings_list = list(findings)
|
||
total_findings = len(findings_list)
|
||
|
||
# Chunk findings into micro-batches
|
||
for i in range(0, total_findings, FINDINGS_MICRO_BATCH_SIZE):
|
||
micro_batch = findings_list[i : i + FINDINGS_MICRO_BATCH_SIZE]
|
||
|
||
_process_finding_micro_batch(
|
||
tenant_id=tenant_id,
|
||
findings_batch=micro_batch,
|
||
scan_instance=scan_instance,
|
||
provider_instance=provider_instance,
|
||
resource_cache=resource_cache,
|
||
tag_cache=tag_cache,
|
||
last_status_cache=last_status_cache,
|
||
resource_failed_findings_cache=resource_failed_findings_cache,
|
||
unique_resources=unique_resources,
|
||
scan_resource_cache=scan_resource_cache,
|
||
mute_rules_cache=mute_rules_cache,
|
||
scan_categories_cache=scan_categories_cache,
|
||
scan_resource_groups_cache=scan_resource_groups_cache,
|
||
group_resources_cache=group_resources_cache,
|
||
)
|
||
|
||
# Throttled progress save (the final save in the `finally` block
|
||
# below always runs regardless of throttle).
|
||
now = time.time()
|
||
progress_delta = progress - last_persisted_progress
|
||
elapsed = now - last_persisted_progress_at
|
||
if (
|
||
progress_delta >= PROGRESS_THROTTLE_DELTA
|
||
or elapsed >= PROGRESS_THROTTLE_SECONDS
|
||
):
|
||
with rls_transaction(tenant_id):
|
||
scan_instance.progress = progress
|
||
scan_instance.save(update_fields=["progress", "updated_at"])
|
||
last_persisted_progress = progress
|
||
last_persisted_progress_at = now
|
||
|
||
scan_instance.state = StateChoices.COMPLETED
|
||
|
||
# Update failed_findings_count for all resources in batches if scan completed successfully
|
||
if resource_failed_findings_cache:
|
||
resources_to_update = []
|
||
for resource_uid, failed_count in resource_failed_findings_cache.items():
|
||
if resource_uid in resource_cache:
|
||
resource_instance = resource_cache[resource_uid]
|
||
resource_instance.failed_findings_count = failed_count
|
||
resources_to_update.append(resource_instance)
|
||
|
||
if resources_to_update:
|
||
# Single rls_transaction wrapping the bulk_update (previously
|
||
# `update_objects_in_batches` opened one rls_transaction per
|
||
# chunk; for tenants with many resources this collapsed N
|
||
# BEGINs/COMMITs into 1).
|
||
with rls_transaction(tenant_id):
|
||
Resource.objects.bulk_update(
|
||
resources_to_update,
|
||
["failed_findings_count"],
|
||
batch_size=SCAN_DB_BATCH_SIZE,
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error performing scan {scan_id}: {e}")
|
||
exception = e
|
||
scan_instance.state = StateChoices.FAILED
|
||
|
||
finally:
|
||
with rls_transaction(tenant_id):
|
||
scan_instance.duration = time.time() - start_time
|
||
scan_instance.completed_at = datetime.now(tz=UTC)
|
||
scan_instance.unique_resource_count = len(unique_resources)
|
||
scan_instance.save(
|
||
update_fields=[
|
||
"state",
|
||
"duration",
|
||
"completed_at",
|
||
"unique_resource_count",
|
||
"progress",
|
||
"updated_at",
|
||
]
|
||
)
|
||
|
||
if exception is not None:
|
||
raise exception
|
||
|
||
try:
|
||
resource_scan_summaries = [
|
||
ResourceScanSummary(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
resource_id=resource_id,
|
||
service=service,
|
||
region=region,
|
||
resource_type=resource_type,
|
||
)
|
||
for resource_id, service, region, resource_type in scan_resource_cache
|
||
]
|
||
with rls_transaction(tenant_id):
|
||
ResourceScanSummary.objects.bulk_create(
|
||
resource_scan_summaries, batch_size=500, ignore_conflicts=True
|
||
)
|
||
except Exception as filter_exception:
|
||
sentry_sdk.capture_exception(filter_exception)
|
||
logger.error(
|
||
f"Error storing filter values for scan {scan_id}: {filter_exception}"
|
||
)
|
||
|
||
try:
|
||
if scan_categories_cache:
|
||
category_summaries = [
|
||
ScanCategorySummary(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
category=category,
|
||
severity=severity,
|
||
total_findings=counts["total"],
|
||
failed_findings=counts["failed"],
|
||
new_failed_findings=counts["new_failed"],
|
||
)
|
||
for (category, severity), counts in scan_categories_cache.items()
|
||
]
|
||
with rls_transaction(tenant_id):
|
||
ScanCategorySummary.objects.bulk_create(
|
||
category_summaries, batch_size=500, ignore_conflicts=True
|
||
)
|
||
except Exception as cat_exception:
|
||
sentry_sdk.capture_exception(cat_exception)
|
||
logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
|
||
|
||
try:
|
||
if scan_resource_groups_cache:
|
||
# Compute group-level resource counts (same value for all severity rows in a group)
|
||
group_resource_counts = {
|
||
grp: len(uids) for grp, uids in group_resources_cache.items()
|
||
}
|
||
resource_group_summaries = [
|
||
ScanGroupSummary(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
resource_group=grp,
|
||
severity=severity,
|
||
total_findings=counts["total"],
|
||
failed_findings=counts["failed"],
|
||
new_failed_findings=counts["new_failed"],
|
||
resources_count=group_resource_counts.get(grp, 0),
|
||
)
|
||
for (
|
||
grp,
|
||
severity,
|
||
), counts in scan_resource_groups_cache.items()
|
||
]
|
||
with rls_transaction(tenant_id):
|
||
ScanGroupSummary.objects.bulk_create(
|
||
resource_group_summaries, batch_size=500, ignore_conflicts=True
|
||
)
|
||
except Exception as rg_exception:
|
||
sentry_sdk.capture_exception(rg_exception)
|
||
logger.error(
|
||
f"Error storing resource groups for scan {scan_id}: {rg_exception}"
|
||
)
|
||
|
||
serializer = ScanTaskSerializer(instance=scan_instance)
|
||
return serializer.data
|
||
|
||
|
||
def aggregate_findings(tenant_id: str, scan_id: str):
|
||
"""
|
||
Aggregate findings for a scan and populate `ScanSummary` rows.
|
||
|
||
We group findings by check/service/severity/region and compute pass/fail/muted
|
||
totals plus delta counts (new/changed/unchanged). The summary dataset feeds the
|
||
overview API and dashboards, so it is recomputed every time a scan finishes.
|
||
|
||
Args:
|
||
tenant_id: Tenant that owns the scan.
|
||
scan_id: Scan UUID whose findings should be aggregated.
|
||
"""
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
findings = Finding.objects.filter(tenant_id=tenant_id, scan_id=scan_id)
|
||
|
||
aggregation = findings.values(
|
||
"check_id",
|
||
"resources__service",
|
||
"severity",
|
||
"resources__region",
|
||
).annotate(
|
||
fail=Sum(
|
||
Case(
|
||
When(status="FAIL", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
_pass=Sum(
|
||
Case(
|
||
When(status="PASS", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
muted_count=Sum(
|
||
Case(
|
||
When(muted=True, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
total=Count("id"),
|
||
new=Sum(
|
||
Case(
|
||
When(delta="new", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
changed=Sum(
|
||
Case(
|
||
When(delta="changed", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
unchanged=Sum(
|
||
Case(
|
||
When(delta__isnull=True, muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
fail_new=Sum(
|
||
Case(
|
||
When(delta="new", status="FAIL", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
fail_changed=Sum(
|
||
Case(
|
||
When(delta="changed", status="FAIL", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
pass_new=Sum(
|
||
Case(
|
||
When(delta="new", status="PASS", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
pass_changed=Sum(
|
||
Case(
|
||
When(delta="changed", status="PASS", muted=False, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
muted_new=Sum(
|
||
Case(
|
||
When(delta="new", muted=True, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
muted_changed=Sum(
|
||
Case(
|
||
When(delta="changed", muted=True, then=1),
|
||
default=0,
|
||
output_field=IntegerField(),
|
||
)
|
||
),
|
||
)
|
||
|
||
with rls_transaction(tenant_id):
|
||
scan_aggregations = {
|
||
ScanSummary(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
check_id=agg["check_id"],
|
||
service=agg["resources__service"],
|
||
severity=agg["severity"],
|
||
region=agg["resources__region"],
|
||
fail=agg["fail"],
|
||
_pass=agg["_pass"],
|
||
muted=agg["muted_count"],
|
||
total=agg["total"],
|
||
new=agg["new"],
|
||
changed=agg["changed"],
|
||
unchanged=agg["unchanged"],
|
||
fail_new=agg["fail_new"],
|
||
fail_changed=agg["fail_changed"],
|
||
pass_new=agg["pass_new"],
|
||
pass_changed=agg["pass_changed"],
|
||
muted_new=agg["muted_new"],
|
||
muted_changed=agg["muted_changed"],
|
||
)
|
||
for agg in aggregation
|
||
if agg["resources__service"] is not None
|
||
and agg["resources__region"] is not None
|
||
}
|
||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||
# `unique_scan_summary`; race-safe under concurrent writers.
|
||
ScanSummary.objects.bulk_create(
|
||
scan_aggregations,
|
||
batch_size=3000,
|
||
update_conflicts=True,
|
||
unique_fields=[
|
||
"tenant",
|
||
"scan",
|
||
"check_id",
|
||
"service",
|
||
"severity",
|
||
"region",
|
||
],
|
||
update_fields=[
|
||
"_pass",
|
||
"fail",
|
||
"muted",
|
||
"total",
|
||
"new",
|
||
"changed",
|
||
"unchanged",
|
||
"fail_new",
|
||
"fail_changed",
|
||
"pass_new",
|
||
"pass_changed",
|
||
"muted_new",
|
||
"muted_changed",
|
||
],
|
||
)
|
||
|
||
|
||
def _aggregate_findings_by_region(
|
||
tenant_id: str, scan_id: str, modeled_threatscore_compliance_id: str
|
||
) -> tuple[dict, dict]:
|
||
"""
|
||
Aggregate findings by region using streaming, column-scoped ORM reads.
|
||
|
||
Reads only the consumed columns as tuples via ``values_list`` and streams
|
||
them with ``.iterator()``, using the denormalized ``resource_regions`` array
|
||
instead of ``prefetch_related("resources")``. ``resource_regions`` mirrors the
|
||
regions of a finding's related resources, so it yields the same per-region
|
||
tally without joining the resource table.
|
||
|
||
Args:
|
||
tenant_id: Tenant UUID
|
||
scan_id: Scan UUID
|
||
modeled_threatscore_compliance_id: ID for ThreatScore compliance framework
|
||
|
||
Returns:
|
||
tuple: (check_status_by_region, findings_count_by_compliance)
|
||
- check_status_by_region: {region: {check_id: status}}
|
||
- findings_count_by_compliance: {region: {normalized_id: {requirement_id: {total, pass}}}}
|
||
"""
|
||
check_status_by_region: dict = {}
|
||
findings_count_by_compliance: dict = {}
|
||
|
||
normalized_id = re.sub(r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower())
|
||
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
findings = (
|
||
Finding.all_objects.filter(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
muted=False,
|
||
status__in=["PASS", "FAIL"],
|
||
)
|
||
.values_list("check_id", "status", "resource_regions", "compliance")
|
||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||
)
|
||
|
||
for check_id, status, resource_regions, compliance in findings:
|
||
threatscore_requirements = (compliance or {}).get(
|
||
modeled_threatscore_compliance_id
|
||
)
|
||
|
||
for region in resource_regions or ():
|
||
# Priority: FAIL > any other status
|
||
current_status = check_status_by_region.setdefault(region, {})
|
||
if current_status.get(check_id) != "FAIL":
|
||
current_status[check_id] = status
|
||
|
||
# Aggregate ThreatScore compliance counts
|
||
if threatscore_requirements:
|
||
compliance_key = findings_count_by_compliance.setdefault(
|
||
region, {}
|
||
).setdefault(normalized_id, {})
|
||
|
||
for requirement_id in threatscore_requirements:
|
||
requirement_stats = compliance_key.setdefault(
|
||
requirement_id, {"total": 0, "pass": 0}
|
||
)
|
||
requirement_stats["total"] += 1
|
||
if status == "PASS":
|
||
requirement_stats["pass"] += 1
|
||
|
||
return check_status_by_region, findings_count_by_compliance
|
||
|
||
|
||
def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||
"""
|
||
Materialize per-requirement compliance rows (and summaries) for a scan.
|
||
|
||
Using the provider’s compliance template plus the scan’s findings, we compute a
|
||
row per (region, compliance, requirement) and write it to
|
||
`ComplianceRequirementOverview` via COPY. The same pass tally requirement
|
||
statuses so we can persist `ComplianceOverviewSummary` records for the fast
|
||
overview endpoint.
|
||
|
||
Args:
|
||
tenant_id: Tenant running the scan.
|
||
scan_id: Scan identifier whose findings should be translated into compliance data.
|
||
|
||
Returns:
|
||
dict: Counts/metadata about the generated rows (e.g., frameworks touched, regions processed).
|
||
"""
|
||
try:
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
scan_instance = Scan.objects.get(pk=scan_id)
|
||
provider_instance = scan_instance.provider
|
||
return_prowler_provider(provider_instance)
|
||
|
||
compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[
|
||
provider_instance.provider
|
||
]
|
||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||
|
||
requirement_lookup: dict[str, list[tuple[str, str]]] = {}
|
||
for compliance_id, compliance in compliance_template.items():
|
||
for requirement_id, requirement in compliance["requirements"].items():
|
||
for check_id in requirement["checks"].keys():
|
||
requirement_lookup.setdefault(check_id, []).append(
|
||
(compliance_id, requirement_id)
|
||
)
|
||
|
||
regions = []
|
||
requirements_created = 0
|
||
requirement_statuses = defaultdict(
|
||
lambda: {"fail_count": 0, "pass_count": 0, "total_count": 0}
|
||
)
|
||
|
||
# Skip if provider has no compliance frameworks
|
||
if compliance_template:
|
||
# Aggregate findings by region using SQL for optimal performance
|
||
check_status_by_region, findings_count_by_compliance = (
|
||
_aggregate_findings_by_region(
|
||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||
)
|
||
)
|
||
|
||
# Only process regions that have findings (optimization: reduces row count)
|
||
regions = list(check_status_by_region.keys())
|
||
|
||
region_requirement_stats: dict[
|
||
str, dict[str, dict[str, dict[str, int]]]
|
||
] = defaultdict(lambda: defaultdict(dict))
|
||
for region, check_status in check_status_by_region.items():
|
||
for check_name, status in check_status.items():
|
||
targets = requirement_lookup.get(check_name)
|
||
if not targets:
|
||
continue
|
||
status_lower = (status or "").lower()
|
||
if status_lower not in {"pass", "fail"}:
|
||
continue
|
||
for compliance_id, requirement_id in targets:
|
||
compliance_stats = region_requirement_stats[region].setdefault(
|
||
compliance_id, {}
|
||
)
|
||
requirement_stats = compliance_stats.setdefault(
|
||
requirement_id, {"passed_checks": 0, "failed_checks": 0}
|
||
)
|
||
if status_lower == "pass":
|
||
requirement_stats["passed_checks"] += 1
|
||
else:
|
||
requirement_stats["failed_checks"] += 1
|
||
|
||
utc_datetime_now = datetime.now(tz=UTC)
|
||
tenant_id_str = str(tenant_id)
|
||
scan_id_str = str(scan_instance.id)
|
||
|
||
# Per-framework constants that don't depend on the region.
|
||
compliance_plan = []
|
||
for compliance_id, compliance in compliance_template.items():
|
||
modeled_compliance_id = _normalized_compliance_key(
|
||
compliance["framework"], compliance["version"]
|
||
)
|
||
framework = compliance["framework"]
|
||
version = compliance["version"] or ""
|
||
requirements = [
|
||
(
|
||
requirement_id,
|
||
requirement.get("description") or "",
|
||
len(requirement["checks"]),
|
||
)
|
||
for requirement_id, requirement in compliance[
|
||
"requirements"
|
||
].items()
|
||
]
|
||
compliance_plan.append(
|
||
(
|
||
compliance_id,
|
||
framework,
|
||
version,
|
||
modeled_compliance_id,
|
||
requirements,
|
||
)
|
||
)
|
||
|
||
# Yield rows lazily (consumed batch-by-batch by COPY) so peak memory
|
||
# stays bounded; tally requirement_statuses in the same pass.
|
||
def _iter_compliance_requirement_rows():
|
||
for region in regions:
|
||
region_stats = region_requirement_stats.get(region, {})
|
||
region_findings = findings_count_by_compliance.get(region, {})
|
||
for (
|
||
compliance_id,
|
||
framework,
|
||
version,
|
||
modeled_compliance_id,
|
||
requirements,
|
||
) in compliance_plan:
|
||
compliance_stats = region_stats.get(compliance_id, {})
|
||
compliance_findings = region_findings.get(
|
||
modeled_compliance_id, {}
|
||
)
|
||
for requirement_id, description, total_checks in requirements:
|
||
stats = compliance_stats.get(requirement_id)
|
||
if stats:
|
||
passed_checks = stats["passed_checks"]
|
||
failed_checks = stats["failed_checks"]
|
||
else:
|
||
passed_checks = 0
|
||
failed_checks = 0
|
||
if total_checks == 0:
|
||
requirement_status = "MANUAL"
|
||
elif failed_checks > 0:
|
||
requirement_status = "FAIL"
|
||
else:
|
||
requirement_status = "PASS"
|
||
|
||
finding_counts = compliance_findings.get(requirement_id)
|
||
if finding_counts:
|
||
passed_findings = finding_counts.get("pass", 0)
|
||
total_findings = finding_counts.get("total", 0)
|
||
else:
|
||
passed_findings = 0
|
||
total_findings = 0
|
||
|
||
key = (compliance_id, requirement_id)
|
||
requirement_statuses[key]["total_count"] += 1
|
||
if requirement_status == "FAIL":
|
||
requirement_statuses[key]["fail_count"] += 1
|
||
elif requirement_status == "PASS":
|
||
requirement_statuses[key]["pass_count"] += 1
|
||
|
||
yield {
|
||
"id": uuid.uuid4(),
|
||
"tenant_id": tenant_id_str,
|
||
"inserted_at": utc_datetime_now,
|
||
"compliance_id": compliance_id,
|
||
"framework": framework,
|
||
"version": version,
|
||
"description": description,
|
||
"region": region,
|
||
"requirement_id": requirement_id,
|
||
"requirement_status": requirement_status,
|
||
"passed_checks": passed_checks,
|
||
"failed_checks": failed_checks,
|
||
"total_checks": total_checks,
|
||
"scan_id": scan_id_str,
|
||
"passed_findings": passed_findings,
|
||
"total_findings": total_findings,
|
||
}
|
||
|
||
# Idempotent re-run: clear this scan's rows before re-inserting.
|
||
with rls_transaction(tenant_id):
|
||
ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete()
|
||
|
||
requirements_created = _persist_compliance_requirement_rows(
|
||
tenant_id, _iter_compliance_requirement_rows()
|
||
)
|
||
|
||
# Create pre-aggregated summaries for fast compliance overview lookups
|
||
_create_compliance_summaries(tenant_id, scan_id, requirement_statuses)
|
||
|
||
return {
|
||
"requirements_created": requirements_created,
|
||
"regions_processed": list(regions),
|
||
"compliance_frameworks": (
|
||
list(compliance_template.keys()) if regions else []
|
||
),
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error creating compliance requirements for scan {scan_id}: {e}")
|
||
raise e
|
||
|
||
|
||
def aggregate_attack_surface(tenant_id: str, scan_id: str):
|
||
"""
|
||
Aggregate findings into attack surface overview records.
|
||
|
||
Creates one AttackSurfaceOverview record per attack surface type
|
||
for the given scan, based on check_id mappings.
|
||
|
||
Args:
|
||
tenant_id: Tenant that owns the scan.
|
||
scan_id: Scan UUID whose findings should be aggregated.
|
||
"""
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
scan_instance = Scan.all_objects.select_related("provider").get(pk=scan_id)
|
||
provider_type = scan_instance.provider.provider
|
||
|
||
provider_attack_surface_mapping = _get_attack_surface_mapping_from_provider(
|
||
provider_type=provider_type
|
||
)
|
||
|
||
# Filter out attack surfaces that are not compatible or have no resolved check IDs
|
||
supported_mappings: dict[str, list[str]] = {}
|
||
for attack_surface_type, check_ids in provider_attack_surface_mapping.items():
|
||
compatible_providers = ATTACK_SURFACE_PROVIDER_COMPATIBILITY.get(
|
||
attack_surface_type
|
||
)
|
||
if (
|
||
compatible_providers is not None
|
||
and provider_type not in compatible_providers
|
||
):
|
||
logger.info(
|
||
f"Skipping {attack_surface_type} - not supported for {provider_type}"
|
||
)
|
||
continue
|
||
|
||
if not check_ids:
|
||
logger.info(
|
||
f"Skipping {attack_surface_type} - no check IDs resolved for {provider_type}"
|
||
)
|
||
continue
|
||
|
||
supported_mappings[attack_surface_type] = list(check_ids)
|
||
|
||
if not supported_mappings:
|
||
logger.info(
|
||
f"No attack surface mappings available for scan {scan_id} and provider {provider_type}"
|
||
)
|
||
logger.info(f"No attack surface overview records created for scan {scan_id}")
|
||
return
|
||
|
||
# Map every check_id to its attack surface, so we can aggregate with a single query
|
||
check_id_to_surface: dict[str, str] = {}
|
||
for attack_surface_type, check_ids in supported_mappings.items():
|
||
for check_id in check_ids:
|
||
check_id_to_surface[check_id] = attack_surface_type
|
||
|
||
aggregated_counts = {
|
||
attack_surface_type: {"total": 0, "failed": 0, "muted": 0}
|
||
for attack_surface_type in supported_mappings.keys()
|
||
}
|
||
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
finding_stats = (
|
||
Finding.all_objects.filter(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
check_id__in=list(check_id_to_surface.keys()),
|
||
)
|
||
.values("check_id")
|
||
.annotate(
|
||
total=Count("id"),
|
||
failed=Count("id", filter=Q(status="FAIL", muted=False)),
|
||
muted=Count("id", filter=Q(status="FAIL", muted=True)),
|
||
)
|
||
)
|
||
|
||
for stats in finding_stats:
|
||
attack_surface_type = check_id_to_surface.get(stats["check_id"])
|
||
if not attack_surface_type:
|
||
continue
|
||
|
||
aggregated_counts[attack_surface_type]["total"] += stats["total"] or 0
|
||
aggregated_counts[attack_surface_type]["failed"] += stats["failed"] or 0
|
||
aggregated_counts[attack_surface_type]["muted"] += stats["muted"] or 0
|
||
|
||
overview_objects = []
|
||
for attack_surface_type, counts in aggregated_counts.items():
|
||
total = counts["total"]
|
||
if not total:
|
||
continue
|
||
|
||
overview_objects.append(
|
||
AttackSurfaceOverview(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
attack_surface_type=attack_surface_type,
|
||
total_findings=total,
|
||
failed_findings=counts["failed"],
|
||
muted_failed_findings=counts["muted"],
|
||
)
|
||
)
|
||
|
||
if overview_objects:
|
||
with rls_transaction(tenant_id):
|
||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||
# `unique_attack_surface_per_scan`; race-safe under concurrent writers.
|
||
AttackSurfaceOverview.objects.bulk_create(
|
||
overview_objects,
|
||
batch_size=500,
|
||
update_conflicts=True,
|
||
unique_fields=["tenant_id", "scan_id", "attack_surface_type"],
|
||
update_fields=[
|
||
"total_findings",
|
||
"failed_findings",
|
||
"muted_failed_findings",
|
||
],
|
||
)
|
||
logger.info(
|
||
f"Upserted {len(overview_objects)} attack surface overview records for scan {scan_id}"
|
||
)
|
||
else:
|
||
logger.info(f"No attack surface overview records created for scan {scan_id}")
|
||
|
||
|
||
def aggregate_daily_severity(tenant_id: str, scan_id: str):
|
||
"""Aggregate scan severity counts into DailySeveritySummary (one record per provider/day)."""
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
scan = Scan.objects.filter(
|
||
tenant_id=tenant_id,
|
||
id=scan_id,
|
||
state=StateChoices.COMPLETED,
|
||
).first()
|
||
|
||
if not scan:
|
||
logger.warning(f"Scan {scan_id} not found or not completed")
|
||
return {"status": "scan is not completed"}
|
||
|
||
provider_id = scan.provider_id
|
||
scan_date = scan.completed_at.date()
|
||
|
||
severity_totals = (
|
||
ScanSummary.objects.filter(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
)
|
||
.values("severity")
|
||
.annotate(total_fail=Sum("fail"), total_muted=Sum("muted"))
|
||
)
|
||
|
||
severity_data = {
|
||
"critical": 0,
|
||
"high": 0,
|
||
"medium": 0,
|
||
"low": 0,
|
||
"informational": 0,
|
||
"muted": 0,
|
||
}
|
||
|
||
for row in severity_totals:
|
||
severity = row["severity"]
|
||
if severity in severity_data:
|
||
severity_data[severity] = row["total_fail"] or 0
|
||
severity_data["muted"] += row["total_muted"] or 0
|
||
|
||
with rls_transaction(tenant_id):
|
||
summary, created = DailySeveritySummary.objects.update_or_create(
|
||
tenant_id=tenant_id,
|
||
provider_id=provider_id,
|
||
date=scan_date,
|
||
defaults={
|
||
"scan_id": scan_id,
|
||
"critical": severity_data["critical"],
|
||
"high": severity_data["high"],
|
||
"medium": severity_data["medium"],
|
||
"low": severity_data["low"],
|
||
"informational": severity_data["informational"],
|
||
"muted": severity_data["muted"],
|
||
},
|
||
)
|
||
|
||
action = "created" if created else "updated"
|
||
logger.info(
|
||
f"Daily severity summary {action} for provider {provider_id} on {scan_date}"
|
||
)
|
||
|
||
return {
|
||
"status": action,
|
||
"provider_id": str(provider_id),
|
||
"date": str(scan_date),
|
||
"severity_data": severity_data,
|
||
}
|
||
|
||
|
||
def update_provider_compliance_scores(tenant_id: str, scan_id: str):
|
||
"""
|
||
Update ProviderComplianceScore with requirement statuses from a completed scan.
|
||
|
||
Uses atomic SQL upsert with ON CONFLICT for concurrency safety. Only updates
|
||
if the new scan is more recent than existing data. Also cleans up stale
|
||
requirements that no longer exist in the new scan.
|
||
|
||
Reads from primary DB (not replica) to avoid replication lag issues since
|
||
this runs immediately after create_compliance_requirements_task.
|
||
|
||
Args:
|
||
tenant_id: Tenant that owns the scan.
|
||
scan_id: Scan UUID whose compliance data should be materialized.
|
||
|
||
Returns:
|
||
dict: Statistics about the upsert operation.
|
||
"""
|
||
with rls_transaction(tenant_id):
|
||
scan = (
|
||
Scan.all_objects.filter(
|
||
tenant_id=tenant_id,
|
||
id=scan_id,
|
||
state=StateChoices.COMPLETED,
|
||
)
|
||
.select_related("provider")
|
||
.first()
|
||
)
|
||
|
||
if not scan:
|
||
logger.warning(
|
||
f"Scan {scan_id} not found or not completed for compliance score update"
|
||
)
|
||
return {"status": "skipped", "reason": "scan not completed"}
|
||
|
||
if not scan.completed_at:
|
||
logger.warning(f"Scan {scan_id} has no completed_at timestamp")
|
||
return {"status": "skipped", "reason": "no completed_at"}
|
||
|
||
provider_id = str(scan.provider_id)
|
||
scan_completed_at = scan.completed_at
|
||
|
||
delete_stale_sql = """
|
||
DELETE FROM provider_compliance_scores pcs
|
||
WHERE pcs.tenant_id = %s
|
||
AND pcs.provider_id = %s
|
||
AND pcs.scan_completed_at < %s
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM compliance_requirements_overviews cro
|
||
WHERE cro.tenant_id = pcs.tenant_id
|
||
AND cro.scan_id = %s
|
||
AND cro.compliance_id = pcs.compliance_id
|
||
AND cro.requirement_id = pcs.requirement_id
|
||
)
|
||
RETURNING compliance_id
|
||
"""
|
||
|
||
compliance_ids_sql = """
|
||
SELECT DISTINCT compliance_id
|
||
FROM compliance_requirements_overviews
|
||
WHERE tenant_id = %s AND scan_id = %s
|
||
"""
|
||
|
||
try:
|
||
with psycopg_connection(MainRouter.default_db) as connection:
|
||
connection.autocommit = False
|
||
try:
|
||
with connection.cursor() as cursor:
|
||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||
|
||
# Update requirement-level scores per provider
|
||
cursor.execute(
|
||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, [tenant_id, scan_id]
|
||
)
|
||
upserted_count = cursor.rowcount
|
||
|
||
cursor.execute(compliance_ids_sql, [tenant_id, scan_id])
|
||
scan_rows = cursor.fetchall()
|
||
if not isinstance(scan_rows, (list, tuple)):
|
||
scan_rows = []
|
||
scan_compliance_ids = {row[0] for row in scan_rows}
|
||
|
||
cursor.execute(
|
||
delete_stale_sql,
|
||
[tenant_id, provider_id, scan_completed_at, scan_id],
|
||
)
|
||
deleted_rows = cursor.fetchall()
|
||
if not isinstance(deleted_rows, (list, tuple)):
|
||
deleted_rows = []
|
||
deleted_ids = {row[0] for row in deleted_rows}
|
||
stale_deleted = len(deleted_ids)
|
||
|
||
impacted_compliance_ids = sorted(scan_compliance_ids | deleted_ids)
|
||
|
||
if impacted_compliance_ids:
|
||
# Advisory lock on tenant to prevent race conditions when
|
||
# multiple scans complete simultaneously for the same tenant
|
||
cursor.execute(
|
||
"SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id]
|
||
)
|
||
|
||
# Recalculate tenant-level summary (FAIL-dominant across all providers)
|
||
cursor.execute(
|
||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||
[tenant_id, tenant_id, impacted_compliance_ids],
|
||
)
|
||
tenant_summary_count = cursor.rowcount
|
||
else:
|
||
tenant_summary_count = 0
|
||
|
||
connection.commit()
|
||
except Exception:
|
||
connection.rollback()
|
||
raise
|
||
|
||
logger.info(
|
||
f"Provider compliance scores updated for scan {scan_id}: "
|
||
f"{upserted_count} upserted, {stale_deleted} stale deleted, "
|
||
f"{tenant_summary_count} tenant summaries upserted"
|
||
)
|
||
|
||
return {
|
||
"status": "completed",
|
||
"scan_id": str(scan_id),
|
||
"provider_id": provider_id,
|
||
"upserted": upserted_count,
|
||
"stale_deleted": stale_deleted,
|
||
"tenant_summary_count": tenant_summary_count,
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(
|
||
f"Error updating provider compliance scores for scan {scan_id}: {e}"
|
||
)
|
||
raise
|
||
|
||
|
||
def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||
"""
|
||
Aggregate finding group summaries for a completed scan.
|
||
|
||
Creates or updates FindingGroupDailySummary records for each unique check_id
|
||
found in the scan's findings. These pre-aggregated summaries enable efficient
|
||
queries over date ranges without scanning millions of findings.
|
||
|
||
Args:
|
||
tenant_id: Tenant that owns the scan.
|
||
scan_id: Scan UUID whose findings should be aggregated.
|
||
|
||
Returns:
|
||
dict: Statistics about the aggregation operation.
|
||
"""
|
||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||
scan = Scan.objects.filter(
|
||
tenant_id=tenant_id,
|
||
id=scan_id,
|
||
state=StateChoices.COMPLETED,
|
||
).first()
|
||
|
||
if not scan:
|
||
logger.warning(
|
||
f"Scan {scan_id} not found or not completed for finding group summary"
|
||
)
|
||
return {"status": "skipped", "reason": "scan not completed"}
|
||
|
||
if not scan.provider:
|
||
logger.warning(f"Scan {scan_id} has no provider for finding group summary")
|
||
return {"status": "skipped", "reason": "scan has no provider"}
|
||
|
||
summary_timestamp = scan.completed_at
|
||
if django_timezone.is_naive(summary_timestamp):
|
||
summary_timestamp = django_timezone.make_aware(summary_timestamp, UTC)
|
||
summary_timestamp = summary_timestamp.replace(
|
||
hour=0, minute=0, second=0, microsecond=0
|
||
)
|
||
provider_id = scan.provider_id
|
||
|
||
# Build severity Case/When expression
|
||
severity_case = Case(
|
||
*[
|
||
When(severity=severity, then=order)
|
||
for severity, order in SEVERITY_ORDER.items()
|
||
],
|
||
output_field=IntegerField(),
|
||
)
|
||
|
||
# Aggregate findings by check_id for this scan.
|
||
# `pass_count`, `fail_count` and `manual_count` only count non-muted
|
||
# findings. Muted findings are tracked separately via the
|
||
# `*_muted_count` fields.
|
||
aggregated = (
|
||
Finding.objects.filter(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
)
|
||
.values("check_id")
|
||
.annotate(
|
||
severity_order=Max(severity_case),
|
||
pass_count=Count("id", filter=Q(status="PASS", muted=False)),
|
||
fail_count=Count("id", filter=Q(status="FAIL", muted=False)),
|
||
manual_count=Count("id", filter=Q(status="MANUAL", muted=False)),
|
||
pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)),
|
||
fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)),
|
||
manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)),
|
||
muted_count=Count("id", filter=Q(muted=True)),
|
||
nonmuted_count=Count("id", filter=Q(muted=False)),
|
||
new_count=Count("id", filter=Q(delta="new", muted=False)),
|
||
changed_count=Count("id", filter=Q(delta="changed", muted=False)),
|
||
new_fail_count=Count(
|
||
"id", filter=Q(delta="new", status="FAIL", muted=False)
|
||
),
|
||
new_fail_muted_count=Count(
|
||
"id", filter=Q(delta="new", status="FAIL", muted=True)
|
||
),
|
||
new_pass_count=Count(
|
||
"id", filter=Q(delta="new", status="PASS", muted=False)
|
||
),
|
||
new_pass_muted_count=Count(
|
||
"id", filter=Q(delta="new", status="PASS", muted=True)
|
||
),
|
||
new_manual_count=Count(
|
||
"id", filter=Q(delta="new", status="MANUAL", muted=False)
|
||
),
|
||
new_manual_muted_count=Count(
|
||
"id", filter=Q(delta="new", status="MANUAL", muted=True)
|
||
),
|
||
changed_fail_count=Count(
|
||
"id", filter=Q(delta="changed", status="FAIL", muted=False)
|
||
),
|
||
changed_fail_muted_count=Count(
|
||
"id", filter=Q(delta="changed", status="FAIL", muted=True)
|
||
),
|
||
changed_pass_count=Count(
|
||
"id", filter=Q(delta="changed", status="PASS", muted=False)
|
||
),
|
||
changed_pass_muted_count=Count(
|
||
"id", filter=Q(delta="changed", status="PASS", muted=True)
|
||
),
|
||
changed_manual_count=Count(
|
||
"id", filter=Q(delta="changed", status="MANUAL", muted=False)
|
||
),
|
||
changed_manual_muted_count=Count(
|
||
"id", filter=Q(delta="changed", status="MANUAL", muted=True)
|
||
),
|
||
resources_total=Count("resources__id", distinct=True),
|
||
resources_fail=Count(
|
||
"resources__id",
|
||
distinct=True,
|
||
filter=Q(status="FAIL", muted=False),
|
||
),
|
||
# Use prefixed names to avoid conflict with model field names
|
||
agg_first_seen_at=Min(
|
||
"first_seen_at", filter=Q(delta="new", muted=False)
|
||
),
|
||
agg_last_seen_at=Max("inserted_at"),
|
||
agg_failing_since=Min(
|
||
"first_seen_at", filter=Q(status="FAIL", muted=False)
|
||
),
|
||
)
|
||
)
|
||
|
||
# Force evaluate queryset while inside RLS transaction (prevents lazy re-query issues)
|
||
aggregated_list = list(aggregated)
|
||
|
||
# Fetch check metadata for all check_ids in one query
|
||
check_ids = [row["check_id"] for row in aggregated_list]
|
||
check_metadata_map = {}
|
||
if check_ids:
|
||
findings_with_metadata = (
|
||
Finding.objects.filter(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
check_id__in=check_ids,
|
||
)
|
||
.order_by("check_id")
|
||
.distinct("check_id")
|
||
.values("check_id", "check_metadata")
|
||
)
|
||
|
||
for f in findings_with_metadata:
|
||
if f["check_id"] not in check_metadata_map and f["check_metadata"]:
|
||
check_metadata_map[f["check_id"]] = f["check_metadata"]
|
||
|
||
# Upsert summaries in bulk for performance
|
||
created_count = 0
|
||
updated_count = 0
|
||
|
||
with rls_transaction(tenant_id):
|
||
check_ids = [row["check_id"] for row in aggregated_list]
|
||
existing_check_ids = set()
|
||
if check_ids:
|
||
existing_check_ids = set(
|
||
FindingGroupDailySummary.objects.filter(
|
||
tenant_id=tenant_id,
|
||
provider_id=provider_id,
|
||
check_id__in=check_ids,
|
||
inserted_at=summary_timestamp,
|
||
).values_list("check_id", flat=True)
|
||
)
|
||
|
||
created_count = len(check_ids) - len(existing_check_ids)
|
||
updated_count = len(existing_check_ids)
|
||
|
||
summaries_to_upsert = []
|
||
updated_at = django_timezone.now()
|
||
for row in aggregated_list:
|
||
check_id = row["check_id"]
|
||
metadata = check_metadata_map.get(check_id, {})
|
||
|
||
summaries_to_upsert.append(
|
||
FindingGroupDailySummary(
|
||
tenant_id=tenant_id,
|
||
provider_id=provider_id,
|
||
check_id=check_id,
|
||
inserted_at=summary_timestamp,
|
||
updated_at=updated_at,
|
||
check_title=metadata.get("checktitle", ""),
|
||
check_description=metadata.get("description", "")
|
||
or metadata.get("Description", ""),
|
||
severity_order=row["severity_order"] or 1,
|
||
pass_count=row["pass_count"],
|
||
fail_count=row["fail_count"],
|
||
manual_count=row["manual_count"],
|
||
pass_muted_count=row["pass_muted_count"],
|
||
fail_muted_count=row["fail_muted_count"],
|
||
manual_muted_count=row["manual_muted_count"],
|
||
muted_count=row["muted_count"],
|
||
muted=row["nonmuted_count"] == 0,
|
||
new_count=row["new_count"],
|
||
changed_count=row["changed_count"],
|
||
new_fail_count=row["new_fail_count"],
|
||
new_fail_muted_count=row["new_fail_muted_count"],
|
||
new_pass_count=row["new_pass_count"],
|
||
new_pass_muted_count=row["new_pass_muted_count"],
|
||
new_manual_count=row["new_manual_count"],
|
||
new_manual_muted_count=row["new_manual_muted_count"],
|
||
changed_fail_count=row["changed_fail_count"],
|
||
changed_fail_muted_count=row["changed_fail_muted_count"],
|
||
changed_pass_count=row["changed_pass_count"],
|
||
changed_pass_muted_count=row["changed_pass_muted_count"],
|
||
changed_manual_count=row["changed_manual_count"],
|
||
changed_manual_muted_count=row["changed_manual_muted_count"],
|
||
resources_total=row["resources_total"],
|
||
resources_fail=row["resources_fail"],
|
||
first_seen_at=row["agg_first_seen_at"],
|
||
last_seen_at=row["agg_last_seen_at"],
|
||
failing_since=row["agg_failing_since"],
|
||
)
|
||
)
|
||
|
||
if summaries_to_upsert:
|
||
FindingGroupDailySummary.objects.bulk_create(
|
||
summaries_to_upsert,
|
||
update_conflicts=True,
|
||
unique_fields=["tenant_id", "provider", "check_id", "inserted_at"],
|
||
update_fields=[
|
||
"check_title",
|
||
"check_description",
|
||
"severity_order",
|
||
"pass_count",
|
||
"fail_count",
|
||
"manual_count",
|
||
"pass_muted_count",
|
||
"fail_muted_count",
|
||
"manual_muted_count",
|
||
"muted_count",
|
||
"muted",
|
||
"new_count",
|
||
"changed_count",
|
||
"new_fail_count",
|
||
"new_fail_muted_count",
|
||
"new_pass_count",
|
||
"new_pass_muted_count",
|
||
"new_manual_count",
|
||
"new_manual_muted_count",
|
||
"changed_fail_count",
|
||
"changed_fail_muted_count",
|
||
"changed_pass_count",
|
||
"changed_pass_muted_count",
|
||
"changed_manual_count",
|
||
"changed_manual_muted_count",
|
||
"resources_total",
|
||
"resources_fail",
|
||
"first_seen_at",
|
||
"last_seen_at",
|
||
"failing_since",
|
||
"updated_at",
|
||
],
|
||
)
|
||
|
||
logger.info(
|
||
f"Finding group summaries aggregated for scan {scan_id}: "
|
||
f"{created_count} created, {updated_count} updated"
|
||
)
|
||
|
||
return {
|
||
"status": "completed",
|
||
"scan_id": str(scan_id),
|
||
"date": str(summary_timestamp.date()),
|
||
"created": created_count,
|
||
"updated": updated_count,
|
||
}
|
||
|
||
|
||
def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dict:
|
||
"""Zero failed_findings_count for resources missing from a completed full-scope scan.
|
||
|
||
Resources that exist in the database for the scan's provider but were not
|
||
touched by this scan are treated as ephemeral. We keep their historical
|
||
findings, but reset the denormalized counter that drives the Resources page
|
||
sort so they stop ranking at the top.
|
||
|
||
Skipped (no-op) when:
|
||
- The scan is not in COMPLETED state.
|
||
- The scan ran with any scoping filter in scanner_args (partial scope).
|
||
|
||
Query design (must scale to 500k+ resources per provider):
|
||
Phase 1 — collect ephemeral IDs with one anti-join read.
|
||
Outer filter ``(tenant_id, provider_id, failed_findings_count > 0)``
|
||
uses ``resources_tenant_provider_idx``. The correlated
|
||
``NOT EXISTS`` subquery hits the implicit unique index
|
||
``(tenant_id, scan_id, resource_id)`` on ``ResourceScanSummary``.
|
||
``NOT EXISTS`` (vs ``NOT IN``) is null-safe and lets the planner
|
||
choose between hash anti-join and indexed nested-loop anti-join.
|
||
``.iterator(chunk_size=...)`` skips the queryset cache so memory
|
||
stays bounded while streaming UUIDs.
|
||
Phase 2 — UPDATE in fixed-size batches.
|
||
One large UPDATE would hold row-exclusive locks for seconds and
|
||
create a WAL spike. Batched UPDATEs by ``id__in`` (~1k rows each)
|
||
hit the primary key, keep each lock window ~50ms, bound WAL chunks,
|
||
and let other writers proceed between batches.
|
||
``failed_findings_count__gt=0`` in the UPDATE is idempotent under
|
||
concurrent scans and skips no-op rewrites.
|
||
Reads use the primary DB, not the replica: ``ResourceScanSummary`` rows
|
||
were written by the same scan task that triggered this one, so replica
|
||
lag could falsely classify scanned resources as ephemeral.
|
||
|
||
Scope detection (``Scan.is_full_scope()``) derives the set of scoping
|
||
scanner_args from ``prowler.lib.scan.scan.Scan.__init__`` via
|
||
introspection, so the API can never drift from the SDK's filter
|
||
contract. Imported scans are also rejected by trigger — they may only
|
||
cover a partial slice of resources.
|
||
"""
|
||
with rls_transaction(tenant_id):
|
||
scan = Scan.objects.filter(tenant_id=tenant_id, id=scan_id).first()
|
||
|
||
if scan is None:
|
||
logger.warning(f"Scan {scan_id} not found")
|
||
return {"status": "skipped", "reason": "scan not found"}
|
||
|
||
if scan.state != StateChoices.COMPLETED:
|
||
logger.info(f"Scan {scan_id} not completed; skipping ephemeral reset")
|
||
return {"status": "skipped", "reason": "scan not completed"}
|
||
|
||
if not scan.is_full_scope():
|
||
logger.info(
|
||
f"Scan {scan_id} ran with scoping filters; skipping ephemeral reset"
|
||
)
|
||
return {"status": "skipped", "reason": "partial scan scope"}
|
||
|
||
# Race protection: if a newer completed full-scope scan exists for this
|
||
# provider, our ResourceScanSummary set is stale relative to the resources'
|
||
# current failed_findings_count values (which the newer scan already
|
||
# refreshed). Wiping based on the older scan would zero counts the newer
|
||
# scan just set. Skip and let the newer scan's reset task do the work; if
|
||
# this task was delayed in the queue, that's the correct outcome.
|
||
# `completed_at__isnull=False` is required: Postgres orders NULL first in
|
||
# DESC, so a sibling COMPLETED scan with a missing completed_at would sort
|
||
# as "newest" and incorrectly cause us to skip.
|
||
with rls_transaction(tenant_id):
|
||
latest_full_scope_scan_id = (
|
||
Scan.objects.filter(
|
||
tenant_id=tenant_id,
|
||
provider_id=scan.provider_id,
|
||
state=StateChoices.COMPLETED,
|
||
completed_at__isnull=False,
|
||
)
|
||
.order_by("-completed_at", "-inserted_at")
|
||
.values_list("id", flat=True)
|
||
.first()
|
||
)
|
||
if latest_full_scope_scan_id != scan.id:
|
||
logger.info(
|
||
f"Scan {scan_id} is not the latest completed scan for provider "
|
||
f"{scan.provider_id}; skipping ephemeral reset"
|
||
)
|
||
return {"status": "skipped", "reason": "newer scan exists"}
|
||
|
||
# Defensive gate: ResourceScanSummary rows are written by perform_prowler_scan
|
||
# via best-effort bulk_create. If those writes failed silently (or the scan
|
||
# genuinely produced resources but no summaries were persisted), the
|
||
# ~Exists(in_scan) anti-join below would classify EVERY resource for this
|
||
# provider as ephemeral and zero their counts. Bail loudly instead.
|
||
with rls_transaction(tenant_id):
|
||
summaries_present = ResourceScanSummary.objects.filter(
|
||
tenant_id=tenant_id, scan_id=scan_id
|
||
).exists()
|
||
if scan.unique_resource_count > 0 and not summaries_present:
|
||
logger.error(
|
||
f"Scan {scan_id} reports {scan.unique_resource_count} unique "
|
||
f"resources but no ResourceScanSummary rows are persisted; "
|
||
f"skipping ephemeral reset to avoid wiping valid counts"
|
||
)
|
||
return {"status": "skipped", "reason": "summaries missing"}
|
||
|
||
# Stays on the primary DB intentionally. ResourceScanSummary rows are
|
||
# written by perform_prowler_scan in the same chain that triggered this
|
||
# task, so replica lag could return an empty/partial summary set; a stale
|
||
# read here would classify every Resource as ephemeral and wipe valid
|
||
# failed_findings_count values on the primary. Same rationale as
|
||
# update_provider_compliance_scores below in this module.
|
||
# Materializing the ID list (rather than streaming the iterator into
|
||
# batched UPDATEs) is intentional: it lets the UPDATEs run in their own
|
||
# short rls_transactions instead of one long transaction holding row locks
|
||
# on every batch. At 500k UUIDs the peak memory is ~40 MB — acceptable for
|
||
# a Celery worker — and is the better trade-off versus a multi-second
|
||
# write-lock window blocking concurrent scans.
|
||
with rls_transaction(tenant_id):
|
||
in_scan = ResourceScanSummary.objects.filter(
|
||
tenant_id=tenant_id,
|
||
scan_id=scan_id,
|
||
resource_id=OuterRef("pk"),
|
||
)
|
||
ephemeral_ids = list(
|
||
Resource.objects.filter(
|
||
tenant_id=tenant_id,
|
||
provider_id=scan.provider_id,
|
||
failed_findings_count__gt=0,
|
||
)
|
||
.filter(~Exists(in_scan))
|
||
.values_list("id", flat=True)
|
||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||
)
|
||
|
||
if not ephemeral_ids:
|
||
logger.info(f"No ephemeral resources for scan {scan_id}")
|
||
return {
|
||
"status": "completed",
|
||
"scan_id": str(scan_id),
|
||
"provider_id": str(scan.provider_id),
|
||
"reset": 0,
|
||
}
|
||
|
||
total_updated = 0
|
||
for batch, _ in batched(ephemeral_ids, DJANGO_FINDINGS_BATCH_SIZE):
|
||
# batched() always yields a final tuple, which is empty when the input
|
||
# length is an exact multiple of the batch size. Skip it so we don't
|
||
# issue a no-op UPDATE ... WHERE id IN ().
|
||
if not batch:
|
||
continue
|
||
with rls_transaction(tenant_id):
|
||
total_updated += Resource.objects.filter(
|
||
tenant_id=tenant_id,
|
||
id__in=batch,
|
||
failed_findings_count__gt=0,
|
||
).update(failed_findings_count=0)
|
||
|
||
logger.info(
|
||
f"Ephemeral resource reset for scan {scan_id}: "
|
||
f"{total_updated} resources zeroed for provider {scan.provider_id}"
|
||
)
|
||
|
||
return {
|
||
"status": "completed",
|
||
"scan_id": str(scan_id),
|
||
"provider_id": str(scan.provider_id),
|
||
"reset": total_updated,
|
||
}
|