Files
prowler/api/src/backend/tasks/tasks.py
T

1492 lines
54 KiB
Python

import os
from datetime import UTC, datetime, timedelta
from pathlib import Path
from shutil import rmtree
from uuid import uuid4
from api.compliance import (
get_compliance_frameworks,
get_prowler_provider_compliance,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import delete_related_daily_task, rls_transaction
from api.decorators import handle_provider_deletion, set_tenant
from api.exceptions import ProviderDeletedException
from api.models import (
Finding,
Integration,
Provider,
Scan,
ScanSummary,
StateChoices,
Task,
)
from api.utils import initialize_prowler_provider
from api.v1.serializers import ScanTaskSerializer
from celery import chain, group, shared_task, states
from celery.utils.log import get_task_logger
from config.celery import RLSTask
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
from django.db import transaction
from django_celery_beat.models import PeriodicTask
from django_celery_results.models import TaskResult
from prowler.lib.check.compliance_models import Compliance
from prowler.lib.outputs.compliance.compliance import (
process_universal_compliance_frameworks,
)
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.finding import Finding as FindingOutput
from tasks.jobs.attack_paths import (
attack_paths_scan,
can_provider_run_attack_paths_scan,
)
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
from tasks.jobs.backfill import (
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
backfill_compliance_summaries,
backfill_daily_severity_summaries,
backfill_finding_group_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
)
from tasks.jobs.connection import (
check_integration_connection,
check_lighthouse_connection,
check_provider_connection,
)
from tasks.jobs.deletion import delete_provider, delete_tenant
from tasks.jobs.export import (
COMPLIANCE_CLASS_MAP,
OUTPUT_FORMATS_MAPPING,
_compress_output_files,
_generate_output_directory,
_upload_to_s3,
)
from tasks.jobs.integrations import (
send_findings_to_jira,
upload_s3_integration,
upload_security_hub_integration,
)
from tasks.jobs.lighthouse_providers import (
check_lighthouse_provider_connection,
refresh_lighthouse_provider_models,
)
from tasks.jobs.muting import mute_historical_findings
from tasks.jobs.orphan_recovery import reconcile_orphans
from tasks.jobs.report import (
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
_cleanup_stale_tmp_output_directories,
generate_compliance_reports_job,
)
from tasks.jobs.scan import (
aggregate_attack_surface,
aggregate_daily_severity,
aggregate_finding_group_summaries,
aggregate_findings,
create_compliance_requirements,
perform_prowler_scan,
reset_ephemeral_resource_findings_count,
update_provider_compliance_scores,
)
from tasks.utils import (
_get_or_create_scheduled_scan,
batched,
get_next_execution_datetime,
)
logger = get_task_logger(__name__)
QUEUED_SCAN_TASK_STATE = "QUEUED"
DISPATCHED_SCAN_TASK_STATES = (states.PENDING, states.STARTED, "PROGRESS")
def _get_dispatched_provider_scan(tenant_id: str, provider_id: str):
"""Return a scan that has already been dispatched for a provider."""
executing_scan = (
Scan.objects.select_for_update()
.filter(
tenant_id=tenant_id,
provider_id=provider_id,
state=StateChoices.EXECUTING,
)
.order_by("-inserted_at")
.first()
)
if executing_scan:
return executing_scan
return (
Scan.objects.select_for_update(of=("self",))
.select_related("task__task_runner_task")
.filter(
tenant_id=tenant_id,
provider_id=provider_id,
state__in=(StateChoices.AVAILABLE, StateChoices.SCHEDULED),
task__isnull=False,
task__task_runner_task__status__in=DISPATCHED_SCAN_TASK_STATES,
)
.order_by("-inserted_at")
.first()
)
def _get_queued_provider_scan(tenant_id: str, provider_id: str):
"""Return the next DB-queued scan for a provider."""
return (
Scan.objects.select_for_update(of=("self",))
.select_related("task__task_runner_task")
.filter(
tenant_id=tenant_id,
provider_id=provider_id,
state=StateChoices.AVAILABLE,
task__isnull=False,
task__task_runner_task__status=QUEUED_SCAN_TASK_STATE,
)
.order_by("inserted_at", "id")
.first()
)
def get_active_provider_scan(tenant_id: str, provider_id: str):
"""Return a dispatched or DB-queued scan for a provider."""
return _get_dispatched_provider_scan(
tenant_id, provider_id
) or _get_queued_provider_scan(tenant_id, provider_id)
def create_scan_task_record(
tenant_id: str,
task_id: str,
task_name: str = "scan-perform",
task_status: str | None = states.PENDING,
) -> Task:
if task_status is None:
task_status = states.PENDING
task_result, _ = TaskResult.objects.update_or_create(
task_id=str(task_id),
defaults={"status": task_status, "task_name": task_name},
)
prowler_task, _ = Task.objects.update_or_create(
id=str(task_id),
tenant_id=tenant_id,
defaults={"task_runner_task": task_result},
)
return prowler_task
def enqueue_scan_execution_on_commit(
tenant_id: str,
scan: Scan,
task_id: str,
) -> None:
transaction.on_commit(
lambda: perform_scan_task.apply_async(
kwargs={
"tenant_id": str(tenant_id),
"scan_id": str(scan.id),
"provider_id": str(scan.provider_id),
},
task_id=str(task_id),
)
)
def _get_queued_scheduled_scan(tenant_id: str, provider_id: str):
return (
Scan.objects.select_for_update(of=("self",))
.select_related("task__task_runner_task")
.filter(
tenant_id=tenant_id,
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
task__isnull=False,
task__task_runner_task__status=QUEUED_SCAN_TASK_STATE,
)
.order_by("inserted_at", "id")
.first()
)
def _get_or_create_queued_scheduled_scan(
tenant_id: str,
provider_id: str,
periodic_task_instance: PeriodicTask,
scheduled_at: datetime,
) -> Scan:
queued_scan = _get_queued_scheduled_scan(tenant_id, provider_id)
if queued_scan:
return queued_scan
task_id = str(uuid4())
queued_task = create_scan_task_record(
tenant_id=tenant_id,
task_id=task_id,
task_status=QUEUED_SCAN_TASK_STATE,
)
return Scan.objects.create(
tenant_id=tenant_id,
name="Daily scheduled scan",
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduled_at=scheduled_at,
scheduler_task_id=periodic_task_instance.id,
task=queued_task,
)
def _dispatch_next_queued_provider_scan(tenant_id: str, provider_id: str):
with rls_transaction(tenant_id):
if not Provider.objects.select_for_update().filter(pk=provider_id).exists():
return None
if _get_dispatched_provider_scan(tenant_id, provider_id):
return None
queued_scan = _get_queued_provider_scan(tenant_id, provider_id)
if not queued_scan or not queued_scan.task:
return None
task_result = queued_scan.task.task_runner_task
task_result.status = states.PENDING
task_result.task_name = "scan-perform"
task_result.save(update_fields=["status", "task_name"])
enqueue_scan_execution_on_commit(
tenant_id=tenant_id,
scan=queued_scan,
task_id=str(queued_scan.task_id),
)
return queued_scan
def _dispatch_next_queued_provider_scan_best_effort(
tenant_id: str, provider_id: str
) -> None:
try:
_dispatch_next_queued_provider_scan(tenant_id, provider_id)
except Exception:
logger.exception(
"Failed to dispatch next queued scan for provider %s", provider_id
)
def _get_or_create_next_scheduled_scan(
tenant_id: str,
provider_id: str,
periodic_task_instance: PeriodicTask,
next_scan_datetime: datetime,
) -> Scan:
interval = periodic_task_instance.interval
now = datetime.now(UTC)
while next_scan_datetime <= now:
next_scan_datetime += timedelta(**{interval.period: interval.every})
return _get_or_create_scheduled_scan(
tenant_id=tenant_id,
provider_id=provider_id,
scheduler_task_id=periodic_task_instance.id,
scheduled_at=next_scan_datetime,
update_state=True,
)
def _ensure_next_scheduled_scan_best_effort(
tenant_id: str,
provider_id: str,
periodic_task_instance: PeriodicTask,
next_scan_datetime: datetime,
) -> None:
try:
with rls_transaction(tenant_id):
_get_or_create_next_scheduled_scan(
tenant_id=tenant_id,
provider_id=provider_id,
periodic_task_instance=periodic_task_instance,
next_scan_datetime=next_scan_datetime,
)
except Exception:
logger.exception(
"Failed to ensure next scheduled scan for provider %s", provider_id
)
def _cleanup_orphan_scheduled_scans(
tenant_id: str,
provider_id: str,
scheduler_task_id: int,
) -> int:
"""
TEMPORARY WORKAROUND: Clean up orphan AVAILABLE scans.
Detects and removes AVAILABLE scans that were never used due to an
issue during the first scheduled scan setup.
An AVAILABLE scan is considered orphan if there's also a SCHEDULED scan for
the same provider with the same scheduler_task_id. This situation indicates
that the first scan execution didn't find the AVAILABLE scan (because it
wasn't committed yet, probably) and created a new one, leaving the AVAILABLE orphaned.
Args:
tenant_id: The tenant ID.
provider_id: The provider ID.
scheduler_task_id: The PeriodicTask ID that triggers these scans.
Returns:
Number of orphan scans deleted (0 if none found).
"""
orphan_available_scans = Scan.objects.filter(
tenant_id=tenant_id,
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=scheduler_task_id,
task__isnull=True,
)
scheduled_scan_exists = Scan.objects.filter(
tenant_id=tenant_id,
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=scheduler_task_id,
).exists()
if scheduled_scan_exists and orphan_available_scans.exists():
orphan_count = orphan_available_scans.count()
logger.warning(
f"[WORKAROUND] Found {orphan_count} orphan AVAILABLE scan(s) for "
f"provider {provider_id} alongside a SCHEDULED scan. Cleaning up orphans..."
)
orphan_available_scans.delete()
return orphan_count
return 0
def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str):
"""
Helper function to perform tasks after a scan is completed.
Args:
tenant_id (str): The tenant ID under which the scan was performed.
scan_id (str): The ID of the scan that was performed.
provider_id (str): The primary key of the Provider instance that was scanned.
"""
chain(
create_compliance_requirements_task.si(tenant_id=tenant_id, scan_id=scan_id),
update_provider_compliance_scores_task.si(tenant_id=tenant_id, scan_id=scan_id),
).apply_async()
aggregate_attack_surface_task.apply_async(
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
)
chain(
perform_scan_summary_task.si(tenant_id=tenant_id, scan_id=scan_id),
group(
aggregate_daily_severity_task.si(tenant_id=tenant_id, scan_id=scan_id),
aggregate_finding_group_summaries_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
generate_outputs_task.si(
scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id
),
# post-scan task — runs in the parallel group so a
# failure cannot cascade into reports or integrations. Its only
# prerequisite is that perform_prowler_scan has committed
# ResourceScanSummary, which is true by the time this chain fires.
reset_ephemeral_resource_findings_count_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
),
group(
# Use optimized task that generates both reports with shared queries
generate_compliance_reports_task.si(
tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id
),
check_integrations_task.si(
tenant_id=tenant_id,
provider_id=provider_id,
scan_id=scan_id,
),
),
).apply_async()
if can_provider_run_attack_paths_scan(tenant_id, provider_id):
# Row is normally created upstream, so this is a safeguard so we can attach the task id below
attack_paths_scan = attack_paths_db_utils.retrieve_attack_paths_scan(
tenant_id, scan_id
)
if attack_paths_scan is None:
attack_paths_scan = attack_paths_db_utils.create_attack_paths_scan(
tenant_id, scan_id, provider_id
)
# Persist the Celery task id so the periodic cleanup can revoke scans stuck in SCHEDULED
result = perform_attack_paths_scan_task.apply_async(
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
)
if attack_paths_scan and result:
attack_paths_db_utils.set_attack_paths_scan_task_id(
tenant_id, attack_paths_scan.id, result.task_id
)
@shared_task(base=RLSTask, name="provider-connection-check")
@set_tenant
def check_provider_connection_task(provider_id: str):
"""
Task to check the connection status of a provider.
Args:
provider_id (str): The primary key of the Provider instance to check.
Returns:
dict: A dictionary containing:
- 'connected' (bool): Indicates whether the provider is successfully connected.
- 'error' (str or None): The error message if the connection failed, otherwise `None`.
"""
return check_provider_connection(provider_id=provider_id)
@shared_task(base=RLSTask, name="integration-connection-check")
@set_tenant
def check_integration_connection_task(integration_id: str):
"""
Task to check the connection status of an integration.
Args:
integration_id (str): The primary key of the Integration instance to check.
"""
return check_integration_connection(integration_id=integration_id)
@shared_task(
base=RLSTask, name="provider-deletion", queue="deletion", autoretry_for=(Exception,)
)
def delete_provider_task(provider_id: str, tenant_id: str):
"""
Task to delete a specific Provider instance.
It will delete in batches all the related resources first.
Args:
provider_id (str): The primary key of the `Provider` instance to be deleted.
tenant_id (str): Tenant ID the provider belongs to.
Returns:
tuple: A tuple containing:
- The number of instances deleted.
- A dictionary with the count of deleted instances per model,
including related models if cascading deletes were triggered.
"""
return delete_provider(tenant_id=tenant_id, pk=provider_id)
# acks_late=False: a re-run would duplicate findings and the task is not auto-recovered,
# so a crashed scan is dropped rather than redelivered by the broker (as before #11416).
@shared_task(base=RLSTask, name="scan-perform", queue="scans", acks_late=False)
@handle_provider_deletion
def perform_scan_task(
tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None
):
"""
Task to perform a Prowler scan on a given provider.
This task runs a Prowler scan on the provider identified by `provider_id`
under the tenant identified by `tenant_id`. The scan will use the `scan_id`
for tracking purposes.
Args:
tenant_id (str): The tenant ID under which the scan is being performed.
scan_id (str): The ID of the scan to be performed.
provider_id (str): The primary key of the Provider instance to scan.
checks_to_execute (list[str], optional): A list of specific checks to perform during the scan. Defaults to None.
Returns:
dict: The result of the scan execution, typically including the status and results of the performed checks.
"""
with rls_transaction(tenant_id):
if not Provider.objects.filter(pk=provider_id).exists():
logger.warning(
"scan-perform skipped: provider %s no longer exists "
"(tenant=%s, scan=%s)",
provider_id,
tenant_id,
scan_id,
)
return None
try:
result = perform_prowler_scan(
tenant_id=tenant_id,
scan_id=scan_id,
provider_id=provider_id,
checks_to_execute=checks_to_execute,
)
_perform_scan_complete_tasks(tenant_id, scan_id, provider_id)
return result
finally:
_dispatch_next_queued_provider_scan_best_effort(tenant_id, provider_id)
# acks_late=False: like scan-perform; a dropped run is re-fired by Beat on the next tick.
@shared_task(
base=RLSTask,
bind=True,
name="scan-perform-scheduled",
queue="scans",
acks_late=False,
)
@handle_provider_deletion
def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
"""
Task to perform a scheduled Prowler scan on a given provider.
This task creates and executes a Prowler scan for the provider identified by `provider_id`
under the tenant identified by `tenant_id`. It initiates a new scan instance with the task ID
for tracking purposes. This task is intended to be run on a schedule (e.g., daily) to
automatically perform scans without manual intervention.
Args:
self: The task instance (automatically passed when bind=True).
tenant_id (str): The tenant ID under which the scan is being performed.
provider_id (str): The primary key of the Provider instance to scan.
Returns:
dict: The result of the scan execution, typically including the status and results
of the performed checks.
"""
task_id = self.request.id
with rls_transaction(tenant_id):
if not Provider.objects.select_for_update().filter(pk=provider_id).exists():
logger.warning(
"scheduled scan-perform skipped: provider %s no longer exists "
"(tenant=%s)",
provider_id,
tenant_id,
)
delete_related_daily_task(provider_id)
return None
periodic_task_instance = PeriodicTask.objects.get(
name=f"scan-perform-scheduled-{provider_id}"
)
executed_scan = Scan.objects.filter(
tenant_id=tenant_id,
provider_id=provider_id,
task__task_runner_task__task_id=task_id,
).first()
if executed_scan:
# Duplicated task execution due to visibility timeout
logger.warning(f"Duplicated scheduled scan for provider {provider_id}.")
return ScanTaskSerializer(instance=executed_scan).data
interval = periodic_task_instance.interval
next_scan_datetime = get_next_execution_datetime(task_id, provider_id)
current_scan_datetime = next_scan_datetime - timedelta(
**{interval.period: interval.every}
)
# TEMPORARY WORKAROUND: Clean up orphan scans from transaction isolation issue
_cleanup_orphan_scheduled_scans(
tenant_id=tenant_id,
provider_id=provider_id,
scheduler_task_id=periodic_task_instance.id,
)
active_scan = get_active_provider_scan(tenant_id, provider_id)
if active_scan:
logger.warning(
"Scan already queued or executing for provider %s. Queueing scheduled run.",
provider_id,
)
queued_scheduled_scan = _get_or_create_queued_scheduled_scan(
tenant_id=tenant_id,
provider_id=provider_id,
periodic_task_instance=periodic_task_instance,
scheduled_at=current_scan_datetime,
)
_get_or_create_next_scheduled_scan(
tenant_id=tenant_id,
provider_id=provider_id,
periodic_task_instance=periodic_task_instance,
next_scan_datetime=next_scan_datetime,
)
return ScanTaskSerializer(instance=queued_scheduled_scan).data
scan_instance = _get_or_create_scheduled_scan(
tenant_id=tenant_id,
provider_id=provider_id,
scheduler_task_id=periodic_task_instance.id,
scheduled_at=current_scan_datetime,
)
scan_instance.task_id = task_id
scan_instance.save()
try:
result = perform_prowler_scan(
tenant_id=tenant_id,
scan_id=str(scan_instance.id),
provider_id=provider_id,
)
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
return result
finally:
_ensure_next_scheduled_scan_best_effort(
tenant_id=tenant_id,
provider_id=provider_id,
periodic_task_instance=periodic_task_instance,
next_scan_datetime=next_scan_datetime,
)
_dispatch_next_queued_provider_scan_best_effort(tenant_id, provider_id)
@shared_task(name="scan-summary", queue="overview")
@handle_provider_deletion
def perform_scan_summary_task(tenant_id: str, scan_id: str):
return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)
class AttackPathsScanRLSTask(RLSTask):
"""
RLS task that marks the `AttackPathsScan` DB row as `FAILED` when the Celery task fails.
Covers failures that happen outside the job's own try/except (e.g. provider lookup,
SDK initialization, or Neo4j configuration errors during setup).
"""
def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002
del args # Required by Celery's Task.on_failure signature; not used.
tenant_id = kwargs.get("tenant_id")
scan_id = kwargs.get("scan_id")
if tenant_id and scan_id:
if isinstance(exc, ProviderDeletedException):
logger.warning(
f"Attack paths scan task {task_id} stopped because its provider "
f"or tenant was deleted: {exc}"
)
else:
logger.error(f"Attack paths scan task {task_id} failed: {exc}")
attack_paths_db_utils.fail_attack_paths_scan(tenant_id, scan_id, str(exc))
@shared_task(
base=AttackPathsScanRLSTask,
bind=True,
name="attack-paths-scan-perform",
queue="attack-paths-scans",
)
@handle_provider_deletion
def perform_attack_paths_scan_task(self, tenant_id: str, scan_id: str):
"""
Execute an Attack Paths scan for the given provider within the current tenant RLS context.
Args:
self: The task instance (automatically passed when bind=True).
tenant_id (str): The tenant identifier for RLS context.
scan_id (str): The Prowler scan identifier for obtaining the tenant and provider context.
Returns:
Any: The result from `attack_paths_scan`, including any per-scan failure details.
"""
return attack_paths_scan(
tenant_id=tenant_id, scan_id=scan_id, task_id=self.request.id
)
@shared_task(name="attack-paths-cleanup-stale-scans", queue="attack-paths-scans")
def cleanup_stale_attack_paths_scans_task():
return cleanup_stale_attack_paths_scans()
@shared_task(name="reconcile-orphan-tasks", queue="celery")
def reconcile_orphan_tasks_task():
"""Periodic watchdog: recover tasks whose worker is gone (deploys, crashes)."""
return reconcile_orphans()
@shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,))
def delete_tenant_task(tenant_id: str):
return delete_tenant(pk=tenant_id)
def _scan_tmp_output_directory(tenant_id: str, scan_id: str) -> Path:
"""Root tmp output directory for a scan ({tmp}/{tenant_id}/{scan_id})."""
return Path(DJANGO_TMP_OUTPUT_DIRECTORY) / str(tenant_id) / str(scan_id)
class ScanReportRLSTask(RLSTask):
"""
RLS task that removes the scan's tmp output directory when the task fails.
Covers failures both inside and outside the task body (e.g. ENOSPC mid-write,
or setup errors) so partial artifacts do not accumulate on the worker disk.
"""
def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002
del args # Required by Celery's Task.on_failure signature; not used.
tenant_id = kwargs.get("tenant_id")
scan_id = kwargs.get("scan_id")
if tenant_id and scan_id:
logger.error(f"Scan report task {task_id} failed: {exc}")
rmtree(_scan_tmp_output_directory(tenant_id, scan_id), ignore_errors=True)
@shared_task(
base=ScanReportRLSTask,
name="scan-report",
queue="scan-reports",
)
@set_tenant(keep_tenant=True)
@handle_provider_deletion
def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
"""
Process findings in batches and generate output files in multiple formats.
This function retrieves findings associated with a scan, processes them
in batches of 50, and writes each batch to the corresponding output files.
It reuses output writer instances across batches, updates them with each
batch of transformed findings, and uses a flag to indicate when the final
batch is being processed. Finally, the output files are compressed and
uploaded to S3.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
provider_id (str): The provider_id id to be used in generating outputs.
"""
try:
_cleanup_stale_tmp_output_directories(
DJANGO_TMP_OUTPUT_DIRECTORY,
max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS,
exclude_scan=(tenant_id, scan_id),
)
except Exception as error:
logger.warning(
"Skipping stale tmp cleanup before output generation for scan %s: %s",
scan_id,
error,
)
# Check if the scan has findings
if not ScanSummary.objects.filter(scan_id=scan_id).exists():
logger.info(f"No findings found for scan {scan_id}")
return {"upload": False}
provider_obj = Provider.objects.get(id=provider_id)
prowler_provider = initialize_prowler_provider(provider_obj)
provider_uid = provider_obj.uid
provider_type = provider_obj.provider
# Per-framework exporters in `COMPLIANCE_CLASS_MAP` consume the legacy bulk.
frameworks_bulk = Compliance.get_bulk(provider_type)
# Universal-only frameworks (top-level JSONs like `dora_2022_2554.json`) are emitted
# via `process_universal_compliance_frameworks` below.
universal_bulk = get_prowler_provider_compliance(provider_type)
universal_only_names = {
name
for name in universal_bulk
if name not in frameworks_bulk and universal_bulk[name].outputs
}
frameworks_avail = get_compliance_frameworks(provider_type)
# Idempotency: a previous run of this task for the same scan may have left
# output files behind (e.g. broker redelivery after a worker was killed
# mid-run with task_acks_late, or a successful run on a deployment without
# S3 where the tmp dir is not removed). Output writers open files in append
# mode with a deterministic path (derived from scan.started_at), so reusing
# them would append every finding row again and duplicate the CSV/output
# rows. Start from a clean slate before (re)generating.
scan_tmp_dir = _scan_tmp_output_directory(tenant_id, scan_id)
if os.path.exists(scan_tmp_dir):
rmtree(scan_tmp_dir, ignore_errors=True)
# The writers below open output files in append mode with deterministic
# paths (derived from scan.started_at). Any stale file that survives the
# cleanup would get every finding row appended again, which is the exact
# duplication this guards against. Continuing is therefore unsafe: abort
# so `ScanReportRLSTask.on_failure` removes the tmp dir and the retry
# starts from a clean slate instead of publishing duplicated rows.
if os.path.exists(scan_tmp_dir):
raise RuntimeError(
"Could not remove stale output directory for scan "
f"{scan_id} before generating outputs; aborting to avoid "
"duplicated rows in appended outputs."
)
out_dir, comp_dir = _generate_output_directory(
DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
)
# Removed on success here and on failure by ScanReportRLSTask.on_failure,
# so partial artifacts do not accumulate and fill the disk (ENOSPC).
def get_writer(writer_map, name, factory, is_last):
"""
Return existing writer_map[name] or create via factory().
In both cases set `.close_file = is_last`.
"""
initialization = False
if name not in writer_map:
writer_map[name] = factory()
initialization = True
w = writer_map[name]
w.close_file = is_last
return w, initialization
output_writers = {}
compliance_writers = {}
# Shared across batches so universal writers are created once and reused.
universal_compliance_state: dict[str, list] = {"compliance": []}
universal_base_dir = os.path.dirname(out_dir)
universal_output_filename = os.path.basename(out_dir)
scan_summary = FindingOutput._transform_findings_stats(
ScanSummary.objects.filter(scan_id=scan_id)
)
# Check if we need to generate ASFF output for AWS providers with SecurityHub integration
generate_asff = False
if provider_type == "aws":
security_hub_integrations = Integration.objects.filter(
integrationproviderrelationship__provider_id=provider_id,
integration_type=Integration.IntegrationChoices.AWS_SECURITY_HUB,
enabled=True,
)
generate_asff = security_hub_integrations.exists()
qs = (
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
.order_by("uid")
.iterator()
)
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
for batch, is_last in batched(qs, DJANGO_FINDINGS_BATCH_SIZE):
fos = [
FindingOutput.transform_api_finding(f, prowler_provider) for f in batch
]
# Outputs
for mode, cfg in OUTPUT_FORMATS_MAPPING.items():
# Skip ASFF generation if not needed
if mode == "json-asff" and not generate_asff:
continue
cls = cfg["class"]
suffix = cfg["suffix"]
extra = cfg.get("kwargs", {}).copy()
if mode == "html":
extra.update(provider=prowler_provider, stats=scan_summary)
writer, initialization = get_writer(
output_writers,
cls,
lambda cls=cls, fos=fos, suffix=suffix: cls(
findings=fos,
file_path=out_dir,
file_extension=suffix,
from_cli=False,
),
is_last,
)
if not initialization:
writer.transform(fos)
writer.batch_write_data_to_file(**extra)
writer._data.clear()
# Universal-only frameworks (e.g. `dora_2022_2554.json`).
if universal_only_names:
process_universal_compliance_frameworks(
input_compliance_frameworks=universal_only_names,
universal_frameworks=universal_bulk,
finding_outputs=fos,
output_directory=universal_base_dir,
output_filename=universal_output_filename,
provider=provider_type,
generated_outputs=universal_compliance_state,
from_cli=False,
is_last=is_last,
)
# Compliance CSVs (per-framework exporters).
for name in frameworks_avail:
if name in universal_only_names:
continue
if name not in frameworks_bulk:
logger.warning(
"Compliance framework '%s' missing from bulk; skipping CSV export",
name,
)
continue
compliance_obj = frameworks_bulk[name]
klass = GenericCompliance
for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []):
if condition(name):
klass = cls
break
filename = f"{comp_dir}_{name}.csv"
writer, initialization = get_writer(
compliance_writers,
name,
lambda klass=klass, fos=fos: klass(
findings=fos,
compliance=compliance_obj,
file_path=filename,
from_cli=False,
),
is_last,
)
if not initialization:
writer.transform(fos, compliance_obj, name)
writer.batch_write_data_to_file()
writer._data.clear()
compressed = _compress_output_files(out_dir)
upload_uri = _upload_to_s3(
tenant_id,
scan_id,
compressed,
os.path.basename(compressed),
)
compliance_dir_path = Path(comp_dir).parent
if compliance_dir_path.exists():
for artifact_path in sorted(compliance_dir_path.iterdir()):
if artifact_path.is_file():
_upload_to_s3(
tenant_id,
scan_id,
str(artifact_path),
f"compliance/{artifact_path.name}",
)
# S3 integrations (need output_directory)
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
s3_integrations = Integration.objects.filter(
integrationproviderrelationship__provider_id=provider_id,
integration_type=Integration.IntegrationChoices.AMAZON_S3,
enabled=True,
)
if s3_integrations:
# Pass the output directory path to S3 integration task to reconstruct objects from files
s3_integration_task.apply_async(
kwargs={
"tenant_id": tenant_id,
"provider_id": provider_id,
"output_directory": out_dir,
}
).get(
disable_sync_subtasks=False
) # TODO: This synchronous execution is NOT recommended
# We're forced to do this because we need the files to exist before deletion occurs.
# Once we have the periodic file cleanup task implemented, we should:
# 1. Remove this .get() call and make it fully async
# 2. For Cloud deployments, develop a secondary approach where outputs are stored
# directly in S3 and read from there, eliminating local file dependencies
if upload_uri:
# TODO: We need to create a new periodic task to delete the output files
# This task shouldn't be responsible for deleting the output files
try:
rmtree(scan_tmp_dir, ignore_errors=True)
except Exception as e:
logger.error(f"Error deleting output files: {e}")
final_location, did_upload = upload_uri, True
else:
final_location, did_upload = compressed, False
Scan.all_objects.filter(id=scan_id).update(output_location=final_location)
logger.info(f"Scan outputs at {final_location}")
return {
"upload": did_upload,
}
@shared_task(name="backfill-scan-resource-summaries", queue="backfill")
@handle_provider_deletion
def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str):
"""
Tries to backfill the resource scan summaries table for a given scan.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return backfill_resource_scan_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-compliance-summaries", queue="backfill")
@handle_provider_deletion
def backfill_compliance_summaries_task(tenant_id: str, scan_id: str):
"""
Tries to backfill compliance overview summaries for a completed scan.
This task aggregates compliance requirement data across regions
to create pre-computed summary records for fast compliance overview queries.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return backfill_compliance_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-daily-severity-summaries", queue="backfill")
def backfill_daily_severity_summaries_task(tenant_id: str, days: int = None):
"""Backfill DailySeveritySummary from historical scans. Use days param to limit scope."""
return backfill_daily_severity_summaries(tenant_id=tenant_id, days=days)
@shared_task(name="backfill-finding-group-summaries", queue="backfill")
def backfill_finding_group_summaries_task(tenant_id: str, days: int = None):
"""Backfill FindingGroupDailySummary from historical scans. Use days param to limit scope."""
return backfill_finding_group_summaries(tenant_id=tenant_id, days=days)
@shared_task(name="scan-category-summaries", queue="overview")
@handle_provider_deletion
def aggregate_scan_category_summaries_task(tenant_id: str, scan_id: str):
"""
Backfill ScanCategorySummary for a completed scan.
Aggregates unique categories from findings and creates a summary row.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return aggregate_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-resource-group-summaries", queue="overview")
@handle_provider_deletion
def aggregate_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
"""
Backfill ScanGroupSummary for a completed scan.
Aggregates unique resource groups from findings and creates a summary row.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return aggregate_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-provider-compliance-scores", queue="backfill")
def backfill_provider_compliance_scores_task(tenant_id: str):
"""
Backfill ProviderComplianceScore from latest completed scan per provider.
Used to populate the compliance watchlist materialized table for tenants
that had scans before the feature was deployed.
Args:
tenant_id: Target tenant UUID.
"""
return backfill_provider_compliance_scores(tenant_id=tenant_id)
@shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance")
@handle_provider_deletion
def create_compliance_requirements_task(tenant_id: str, scan_id: str):
"""
Creates detailed compliance requirement records for a scan.
This task processes the compliance data collected during a scan and creates
individual records for each compliance requirement in each region. These detailed
records provide a granular view of compliance status.
Args:
tenant_id (str): The tenant ID for which to create records.
scan_id (str): The ID of the scan for which to create records.
"""
return create_compliance_requirements(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-attack-surface-overviews", queue="overview")
@handle_provider_deletion
def aggregate_attack_surface_task(tenant_id: str, scan_id: str):
"""
Creates attack surface overview records for a scan.
This task processes findings and aggregates them into attack surface categories
(internet-exposed, secrets, privilege-escalation, ec2-imdsv1) for quick overview queries.
Args:
tenant_id (str): The tenant ID for which to create records.
scan_id (str): The ID of the scan for which to create records.
"""
return aggregate_attack_surface(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-provider-compliance-scores", queue="compliance")
def update_provider_compliance_scores_task(tenant_id: str, scan_id: str):
"""
Update provider compliance scores from a completed scan.
This task materializes compliance requirement statuses into ProviderComplianceScore
for efficient watchlist queries. Uses atomic upsert with concurrency protection.
Args:
tenant_id (str): The tenant ID for which to update scores.
scan_id (str): The ID of the scan whose data should be materialized.
"""
return update_provider_compliance_scores(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-daily-severity", queue="overview")
@handle_provider_deletion
def aggregate_daily_severity_task(tenant_id: str, scan_id: str):
"""Aggregate scan severity into DailySeveritySummary for findings_severity/timeseries endpoint."""
return aggregate_daily_severity(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-reset-ephemeral-resources", queue="overview")
@handle_provider_deletion
def reset_ephemeral_resource_findings_count_task(tenant_id: str, scan_id: str):
"""Reset failed_findings_count for resources missing from a completed full-scope scan.
Failures are swallowed and returned as a status: this task lives inside the
post-scan group, and Celery propagates group-member exceptions into the next
chain step — meaning a crash here would block compliance reports and
integrations. The reset is purely cosmetic (UI sort optimization), so a
bad run is logged and absorbed rather than allowed to cascade.
"""
try:
return reset_ephemeral_resource_findings_count(
tenant_id=tenant_id, scan_id=scan_id
)
except Exception as exc: # noqa: BLE001 — intentionally broad
logger.exception(
f"reset_ephemeral_resource_findings_count failed for scan {scan_id}: {exc}"
)
return {
"status": "failed",
"scan_id": str(scan_id),
"reason": str(exc),
}
@shared_task(base=RLSTask, name="scan-finding-group-summaries", queue="overview")
@set_tenant(keep_tenant=True)
@handle_provider_deletion
def aggregate_finding_group_summaries_task(tenant_id: str, scan_id: str):
"""Aggregate findings by check_id into FindingGroupDailySummary for finding-groups endpoint."""
return aggregate_finding_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(
base=RLSTask, name="reaggregate-all-finding-group-summaries", queue="overview"
)
@set_tenant(keep_tenant=True)
def reaggregate_all_finding_group_summaries_task(tenant_id: str):
"""Reaggregate every pre-aggregated summary table for this tenant.
Mirrors the unbounded scope of `mute_historical_findings_task`: that task
rewrites every Finding row whose UID matches a mute rule, with no time
limit. To keep the pre-aggregated tables consistent with that update,
this task re-runs the same per-scan aggregation pipeline that scan
completion runs on the latest completed scan of every (provider, day)
pair, rebuilding the tables that power the read endpoints:
- `ScanSummary` and `DailySeveritySummary` -> `/overviews/findings`,
`/overviews/findings-severity`, `/overviews/services`.
- `FindingGroupDailySummary` -> `/finding-groups` and
`/finding-groups/latest`.
- `ScanGroupSummary` -> `/overviews/resource-groups` (resource
inventory).
- `ScanCategorySummary` -> `/overviews/categories`.
- `AttackSurfaceOverview` -> `/overviews/attack-surfaces`.
Per-scan pipelines are dispatched in parallel via a Celery group so
wallclock scales with the worker pool.
"""
completed_scans = list(
Scan.objects.filter(
tenant_id=tenant_id,
state=StateChoices.COMPLETED,
completed_at__isnull=False,
)
.order_by("-completed_at")
.values("id", "completed_at", "provider_id")
)
# Keep the latest scan per (provider, day) pair so the daily summary row
# the aggregator writes is the most recent snapshot of that day for that
# provider. Iterating from most recent to oldest means the first scan we
# see for a given key wins.
latest_scans: dict[tuple, str] = {}
for scan in completed_scans:
key = (scan["provider_id"], scan["completed_at"].date())
if key not in latest_scans:
latest_scans[key] = str(scan["id"])
scan_ids = list(latest_scans.values())
if scan_ids:
logger.info(
"Reaggregating overview/finding summaries for %d scans (provider x day)",
len(scan_ids),
)
# DailySeveritySummary reads from ScanSummary, so ScanSummary must be
# recomputed first; the other aggregators read Finding directly and
# can run in parallel with the severity step.
group(
chain(
perform_scan_summary_task.si(tenant_id=tenant_id, scan_id=scan_id),
group(
aggregate_daily_severity_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
aggregate_finding_group_summaries_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
aggregate_scan_resource_group_summaries_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
aggregate_scan_category_summaries_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
aggregate_attack_surface_task.si(
tenant_id=tenant_id, scan_id=scan_id
),
),
)
for scan_id in scan_ids
).apply_async()
return {"scans_reaggregated": len(scan_ids)}
@shared_task(base=RLSTask, name="lighthouse-connection-check")
@set_tenant
def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = None):
"""
Task to check the connection status of a Lighthouse configuration.
Args:
lighthouse_config_id (str): The primary key of the LighthouseConfiguration instance to check.
tenant_id (str): The tenant ID for the task.
Returns:
dict: A dictionary containing:
- 'connected' (bool): Indicates whether the connection is successful.
- 'error' (str or None): The error message if the connection failed, otherwise `None`.
- 'available_models' (list): List of available models if connection is successful.
"""
return check_lighthouse_connection(lighthouse_config_id=lighthouse_config_id)
@shared_task(base=RLSTask, name="lighthouse-provider-connection-check")
@set_tenant
def check_lighthouse_provider_connection_task(
provider_config_id: str, tenant_id: str | None = None
) -> dict:
"""Task wrapper to validate provider credentials and set is_active."""
return check_lighthouse_provider_connection(provider_config_id=provider_config_id)
@shared_task(base=RLSTask, name="lighthouse-provider-models-refresh")
@set_tenant
def refresh_lighthouse_provider_models_task(
provider_config_id: str, tenant_id: str | None = None
) -> dict:
"""Task wrapper to refresh provider models catalog for the given configuration."""
return refresh_lighthouse_provider_models(provider_config_id=provider_config_id)
@shared_task(name="integration-check")
@handle_provider_deletion
def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = None):
"""
Check and execute all configured integrations for a provider.
Args:
tenant_id (str): The tenant identifier
provider_id (str): The provider identifier
scan_id (str, optional): The scan identifier for integrations that need scan data
"""
logger.info(f"Checking integrations for provider {provider_id}")
try:
integration_tasks = []
with rls_transaction(tenant_id):
integrations = Integration.objects.filter(
integrationproviderrelationship__provider_id=provider_id,
enabled=True,
)
if not integrations.exists():
logger.info(f"No integrations configured for provider {provider_id}")
return {"integrations_processed": 0}
# Security Hub integration
security_hub_integrations = integrations.filter(
integration_type=Integration.IntegrationChoices.AWS_SECURITY_HUB
)
if security_hub_integrations.exists():
integration_tasks.append(
security_hub_integration_task.s(
tenant_id=tenant_id, provider_id=provider_id, scan_id=scan_id
)
)
# TODO: Add other integration types here
# slack_integrations = integrations.filter(
# integration_type=Integration.IntegrationChoices.SLACK
# )
# if slack_integrations.exists():
# integration_tasks.append(
# slack_integration_task.s(
# tenant_id=tenant_id,
# provider_id=provider_id,
# )
# )
except Exception as e:
logger.error(f"Integration check failed for provider {provider_id}: {str(e)}")
return {"integrations_processed": 0, "error": str(e)}
# Execute all integration tasks in parallel if any were found
if integration_tasks:
job = group(integration_tasks)
job.apply_async()
logger.info(f"Launched {len(integration_tasks)} integration task(s)")
return {"integrations_processed": len(integration_tasks)}
@shared_task(
base=RLSTask,
name="integration-s3",
queue="integrations",
)
@handle_provider_deletion
def s3_integration_task(
tenant_id: str,
provider_id: str,
output_directory: str,
):
"""
Process S3 integrations for a provider.
Args:
tenant_id (str): The tenant identifier
provider_id (str): The provider identifier
output_directory (str): Path to the directory containing output files
"""
return upload_s3_integration(tenant_id, provider_id, output_directory)
@shared_task(
base=RLSTask,
name="integration-security-hub",
queue="integrations",
)
def security_hub_integration_task(
tenant_id: str,
provider_id: str,
scan_id: str,
):
"""
Process Security Hub integrations for a provider.
Args:
tenant_id (str): The tenant identifier
provider_id (str): The provider identifier
scan_id (str): The scan identifier
"""
return upload_security_hub_integration(tenant_id, provider_id, scan_id)
# acks_late=False: Jira sends are not deduplicated and the task is not auto-recovered,
# so a crashed send is dropped rather than redelivered (avoids duplicate Jira issues).
@shared_task(
base=RLSTask,
name="integration-jira",
queue="integrations",
acks_late=False,
)
def jira_integration_task(
tenant_id: str,
integration_id: str,
project_key: str,
issue_type: str,
finding_ids: list[str],
):
return send_findings_to_jira(
tenant_id, integration_id, project_key, issue_type, finding_ids
)
@shared_task(
base=RLSTask,
name="scan-compliance-reports",
queue="scan-reports",
)
@handle_provider_deletion
def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: str):
"""
Optimized task to generate ThreatScore, ENS, NIS2, CSA CCM and CIS reports with shared queries.
This task is more efficient than running separate report tasks because it reuses database queries:
- Provider object fetched once (instead of multiple times)
- Requirement statistics aggregated once (instead of multiple times)
- Can reduce database load by up to 50-70%
CIS emits a single PDF per run: the one matching the highest CIS version
available for the scan's provider, picked dynamically from
``Compliance.get_bulk`` (no hard-coded provider → version mapping).
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
provider_id (str): The provider identifier.
Returns:
dict: Results for all reports containing upload status and paths.
"""
return generate_compliance_reports_job(
tenant_id=tenant_id,
scan_id=scan_id,
provider_id=provider_id,
generate_threatscore=True,
generate_ens=True,
generate_nis2=True,
generate_csa=True,
generate_cis=True,
)
@shared_task(name="findings-mute-historical")
def mute_historical_findings_task(tenant_id: str, mute_rule_id: str):
"""
Background task to mute all historical findings matching a mute rule.
This task processes findings in batches to avoid memory issues with large datasets.
It updates the Finding.muted, Finding.muted_at, and Finding.muted_reason fields
for all findings whose UID is in the mute rule's finding_uids list.
Args:
tenant_id (str): The tenant ID for RLS context.
mute_rule_id (str): The primary key of the MuteRule to apply.
Returns:
dict: A dictionary containing:
- 'findings_muted' (int): Total number of findings muted.
- 'rule_id' (str): The mute rule ID.
- 'status' (str): Final status ('completed').
"""
return mute_historical_findings(tenant_id, mute_rule_id)