ben.carrasco/prowler
1
0 Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-10-28 06:14:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: unify ruff tooling and route code quality through the Makefile (#11675)

Browse Source
This commit is contained in:
Rubén De la Torre Vico authored and GitHub committed 2026-06-23 17:15:05 +02:00
1 parent de7da3e960
commit 058a1dc8fe
194 files changed
+1112 -1210
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

api/src/backend/tasks/beat.py
+5 -6
View File
@@ -1,13 +1,12 @@
import json
from datetime import datetime, timedelta, timezone
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from tasks.tasks import perform_scheduled_scan_task
from datetime import UTC, datetime, timedelta
from api.db_utils import rls_transaction
from api.exceptions import ConflictException
from api.models import Provider, Scan, StateChoices
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
from tasks.tasks import perform_scheduled_scan_task
def schedule_provider_scan(provider_instance: Provider):
@@ -37,7 +36,7 @@ def schedule_provider_scan(provider_instance: Provider):
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduled_at=datetime.now(timezone.utc),
scheduled_at=datetime.now(UTC),
)
attack_paths_db_utils.create_attack_paths_scan(
@@ -58,7 +57,7 @@ def schedule_provider_scan(provider_instance: Provider):
}
),
one_off=False,
start_time=datetime.now(timezone.utc) + timedelta(hours=24),
start_time=datetime.now(UTC) + timedelta(hours=24),
)
scheduled_scan.scheduler_task_id = periodic_task_instance.id
scheduled_scan.save()
api/src/backend/tasks/jobs/attack_paths/aws.py
+6 -7
View File
@@ -2,21 +2,20 @@
# (https://github.com/cartography-cncf/cartography), which is licensed under the Apache 2.0 License.
import time
from typing import Any
import aioboto3
import boto3
import neo4j
from api.models import (
AttackPathsScan as ProwlerAPIAttackPathsScan,
)
from api.models import (
Provider as ProwlerAPIProvider,
)
from cartography.config import Config as CartographyConfig
from cartography.intel import aws as cartography_aws
from celery.utils.log import get_task_logger
from api.models import (
AttackPathsScan as ProwlerAPIAttackPathsScan,
Provider as ProwlerAPIProvider,
)
from prowler.providers.common.provider import Provider as ProwlerSDKProvider
from tasks.jobs.attack_paths import db_utils, utils
api/src/backend/tasks/jobs/attack_paths/cleanup.py
+8 -9
View File
@@ -1,5 +1,9 @@
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from api.attack_paths import database as graph_database
from api.db_router import MainRouter
from api.db_utils import rls_transaction
from api.models import AttackPathsScan, StateChoices
from celery import states
from celery.utils.log import get_task_logger
from config.django.base import ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES
@@ -10,11 +14,6 @@ from tasks.jobs.attack_paths.db_utils import (
from tasks.jobs.orphan_recovery import is_worker_alive as _is_worker_alive
from tasks.jobs.orphan_recovery import revoke_task as _revoke_task
from api.attack_paths import database as graph_database
from api.db_router import MainRouter
from api.db_utils import rls_transaction
from api.models import AttackPathsScan, StateChoices
logger = get_task_logger(__name__)
@@ -30,7 +29,7 @@ def cleanup_stale_attack_paths_scans() -> dict:
age plus the parent `Scan` no longer being in flight.
"""
threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
now = datetime.now(tz=timezone.utc)
now = datetime.now(tz=UTC)
cutoff = now - threshold
cleaned_up: list[str] = []
@@ -175,7 +174,7 @@ def _cleanup_scan(scan, task_result, reason: str) -> bool:
# Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock)
if task_result:
task_result.status = states.FAILURE
task_result.date_done = datetime.now(tz=timezone.utc)
task_result.date_done = datetime.now(tz=UTC)
task_result.save(update_fields=["status", "date_done"])
recover_graph_data_ready(fresh_scan)
@@ -201,7 +200,7 @@ def _cleanup_scheduled_scan(scan, task_result, reason: str) -> bool:
if task_result:
task_result.status = states.FAILURE
task_result.date_done = datetime.now(tz=timezone.utc)
task_result.date_done = datetime.now(tz=UTC)
task_result.save(update_fields=["status", "date_done"])
logger.info(f"Cleaned up scheduled scan {scan_id_str}: {reason}")
api/src/backend/tasks/jobs/attack_paths/config.py
+1 -1
View File
@@ -1,5 +1,5 @@
from collections.abc import Callable
from dataclasses import dataclass
from typing import Callable
from uuid import UUID
from config.env import env
api/src/backend/tasks/jobs/attack_paths/db_utils.py
+7 -8
View File
@@ -1,15 +1,14 @@
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
from cartography.config import Config as CartographyConfig
from celery.utils.log import get_task_logger
from tasks.jobs.attack_paths.config import is_provider_available
from api.attack_paths import database as graph_database
from api.db_utils import rls_transaction
from api.models import AttackPathsScan as ProwlerAPIAttackPathsScan
from api.models import Provider as ProwlerAPIProvider
from api.models import StateChoices
from cartography.config import Config as CartographyConfig
from celery.utils.log import get_task_logger
from tasks.jobs.attack_paths.config import is_provider_available
logger = get_task_logger(__name__)
@@ -43,7 +42,7 @@ def create_attack_paths_scan(
provider_id=provider_id,
scan_id=scan_id,
state=StateChoices.SCHEDULED,
started_at=datetime.now(tz=timezone.utc),
started_at=datetime.now(tz=UTC),
graph_data_ready=previous_data_ready,
)
attack_paths_scan.save()
@@ -104,7 +103,7 @@ def starting_attack_paths_scan(
return False
locked.state = StateChoices.EXECUTING
locked.started_at = datetime.now(tz=timezone.utc)
locked.started_at = datetime.now(tz=UTC)
locked.update_tag = cartography_config.update_tag
locked.save(update_fields=["state", "started_at", "update_tag"])
@@ -121,7 +120,7 @@ def _mark_scan_finished(
ingestion_exceptions: dict[str, Any],
) -> None:
"""Set terminal fields on a scan. Caller must be inside a transaction."""
now = datetime.now(tz=timezone.utc)
now = datetime.now(tz=UTC)
duration = (
int((now - attack_paths_scan.started_at).total_seconds())
if attack_paths_scan.started_at
api/src/backend/tasks/jobs/attack_paths/findings.py
+7 -8
View File
@@ -8,13 +8,18 @@ This module handles:
"""
from collections import defaultdict
from typing import Any, Callable, Generator
from collections.abc import Callable, Generator
from typing import Any
from uuid import UUID
import neo4j
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Finding as FindingModel
from api.models import Provider, ResourceFindingMapping
from cartography.config import Config as CartographyConfig
from celery.utils.log import get_task_logger
from prowler.config import config as ProwlerConfig
from tasks.jobs.attack_paths.config import (
BATCH_SIZE,
FINDINGS_BATCH_SIZE,
@@ -29,12 +34,6 @@ from tasks.jobs.attack_paths.queries import (
render_cypher_template,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Finding as FindingModel
from api.models import Provider, ResourceFindingMapping
from prowler.config import config as ProwlerConfig
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/attack_paths/indexes.py
+1 -3
View File
@@ -1,13 +1,11 @@
import neo4j
from cartography.client.core.tx import run_write_query
from celery.utils.log import get_task_logger
from tasks.jobs.attack_paths.config import (
INTERNET_NODE_LABEL,
PROWLER_FINDING_LABEL,
PROVIDER_ELEMENT_ID_PROPERTY,
PROVIDER_RESOURCE_LABEL,
PROWLER_FINDING_LABEL,
)
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/attack_paths/internet.py
+1 -3
View File
@@ -7,11 +7,9 @@ in the temporary scan database before sync.
"""
import neo4j
from api.models import Provider
from cartography.config import Config as CartographyConfig
from celery.utils.log import get_task_logger
from api.models import Provider
from prowler.config import config as ProwlerConfig
from tasks.jobs.attack_paths.config import get_root_node_label
from tasks.jobs.attack_paths.queries import (
api/src/backend/tasks/jobs/attack_paths/queries.py
+1 -1
View File
@@ -1,9 +1,9 @@
# Cypher query templates for Attack Paths operations
from tasks.jobs.attack_paths.config import (
INTERNET_NODE_LABEL,
PROWLER_FINDING_LABEL,
PROVIDER_ELEMENT_ID_PROPERTY,
PROVIDER_RESOURCE_LABEL,
PROWLER_FINDING_LABEL,
)
api/src/backend/tasks/jobs/attack_paths/scan.py
+5 -7
View File
@@ -55,9 +55,13 @@ exception propagates to Celery.
import logging
import time
from typing import Any
from api.attack_paths import database as graph_database
from api.db_utils import rls_transaction
from api.models import Provider as ProwlerAPIProvider
from api.models import StateChoices
from api.utils import initialize_prowler_provider
from cartography.config import Config as CartographyConfig
from cartography.intel import analysis as cartography_analysis
from cartography.intel import create_indexes as cartography_create_indexes
@@ -66,12 +70,6 @@ from celery.utils.log import get_task_logger
from tasks.jobs.attack_paths import db_utils, findings, indexes, internet, sync, utils
from tasks.jobs.attack_paths.config import get_cartography_ingestion_function
from api.attack_paths import database as graph_database
from api.db_utils import rls_transaction
from api.models import Provider as ProwlerAPIProvider
from api.models import StateChoices
from api.utils import initialize_prowler_provider
# Without this Celery goes crazy with Cartography logging
logging.getLogger("cartography").setLevel(logging.ERROR)
logging.getLogger("neo4j").propagate = False
api/src/backend/tasks/jobs/attack_paths/sync.py
+1 -3
View File
@@ -6,11 +6,11 @@ to the tenant database, adding provider isolation labels and properties.
"""
import time
from collections import defaultdict
from typing import Any
import neo4j
from api.attack_paths import database as graph_database
from celery.utils.log import get_task_logger
from tasks.jobs.attack_paths.config import (
PROVIDER_ISOLATION_PROPERTIES,
@@ -27,8 +27,6 @@ from tasks.jobs.attack_paths.queries import (
render_cypher_template,
)
from api.attack_paths import database as graph_database
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/attack_paths/utils.py
+2 -3
View File
@@ -1,7 +1,6 @@
import asyncio
import traceback
from datetime import datetime, timezone
from datetime import UTC, datetime
from celery.utils.log import get_task_logger
@@ -10,7 +9,7 @@ logger = get_task_logger(__name__)
def stringify_exception(exception: Exception, context: str) -> str:
"""Format an exception with timestamp and traceback for logging."""
timestamp = datetime.now(tz=timezone.utc)
timestamp = datetime.now(tz=UTC)
exception_traceback = traceback.TracebackException.from_exception(exception)
traceback_string = "".join(exception_traceback.format())
return f"{timestamp} - {context}\n{traceback_string}"
api/src/backend/tasks/jobs/backfill.py
+12 -13
View File
@@ -1,19 +1,6 @@
from collections import defaultdict
from datetime import timedelta
from celery.utils.log import get_task_logger
from django.db.models import OuterRef, Subquery, Sum
from django.utils import timezone
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
)
from tasks.jobs.scan import (
aggregate_category_counts,
aggregate_finding_group_summaries,
aggregate_resource_group_counts,
)
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import (
POSTGRES_TENANT_VAR,
@@ -36,6 +23,18 @@ from api.models import (
ScanSummary,
StateChoices,
)
from celery.utils.log import get_task_logger
from django.db.models import OuterRef, Subquery, Sum
from django.utils import timezone
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
)
from tasks.jobs.scan import (
aggregate_category_counts,
aggregate_finding_group_summaries,
aggregate_resource_group_counts,
)
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/connection.py
+4 -5
View File
@@ -1,13 +1,12 @@
from datetime import datetime, timezone
from datetime import UTC, datetime
import openai
from celery.utils.log import get_task_logger
from api.models import Integration, LighthouseConfiguration, Provider
from api.utils import (
prowler_integration_connection_test,
prowler_provider_connection_test,
)
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
@@ -38,7 +37,7 @@ def check_provider_connection(provider_id: str):
raise e
provider_instance.connected = connection_result.is_connected
provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc)
provider_instance.connection_last_checked_at = datetime.now(tz=UTC)
provider_instance.save()
connection_error = f"{connection_result.error}" if connection_result.error else None
@@ -111,7 +110,7 @@ def check_integration_connection(integration_id: str):
# Update integration connection status
integration.connected = result.is_connected
integration.connection_last_checked_at = datetime.now(tz=timezone.utc)
integration.connection_last_checked_at = datetime.now(tz=UTC)
integration.save()
return {
api/src/backend/tasks/jobs/deletion.py
+6 -7
View File
@@ -1,10 +1,3 @@
from celery.utils.log import get_task_logger
from django.db import DatabaseError
from tasks.jobs.queries import (
COMPLIANCE_DELETE_EMPTY_TENANT_SUMMARY_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
)
from api.attack_paths import database as graph_database
from api.db_router import MainRouter
from api.db_utils import batch_delete, rls_transaction
@@ -18,6 +11,12 @@ from api.models import (
ScanSummary,
Tenant,
)
from celery.utils.log import get_task_logger
from django.db import DatabaseError
from tasks.jobs.queries import (
COMPLIANCE_DELETE_EMPTY_TENANT_SUMMARY_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
)
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/export.py
+5 -6
View File
@@ -4,12 +4,11 @@ import zipfile
import boto3
import config.django.base as base
from api.db_utils import rls_transaction
from api.models import Scan
from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
from celery.utils.log import get_task_logger
from django.conf import settings
from api.db_utils import rls_transaction
from api.models import Scan
from prowler.config.config import (
csv_file_suffix,
html_file_suffix,
@@ -18,6 +17,9 @@ from prowler.config.config import (
set_output_timestamp,
)
from prowler.lib.outputs.asff.asff import ASFF
from prowler.lib.outputs.compliance.asd_essential_eight.asd_essential_eight_aws import (
ASDEssentialEightAWS,
)
from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected import (
AWSWellArchitected,
)
@@ -42,9 +44,6 @@ from prowler.lib.outputs.compliance.cisa_scuba.cisa_scuba_googleworkspace import
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
from prowler.lib.outputs.compliance.asd_essential_eight.asd_essential_eight_aws import (
ASDEssentialEightAWS,
)
from prowler.lib.outputs.compliance.iso27001.iso27001_aws import AWSISO27001
from prowler.lib.outputs.compliance.iso27001.iso27001_azure import AzureISO27001
from prowler.lib.outputs.compliance.iso27001.iso27001_gcp import GCPISO27001
api/src/backend/tasks/jobs/integrations.py
+4 -5
View File
@@ -2,15 +2,13 @@ import os
import time
from glob import glob
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from django.db import OperationalError
from tasks.utils import batched
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import REPLICA_MAX_ATTEMPTS, REPLICA_RETRY_BASE_DELAY, rls_transaction
from api.models import Finding, Integration, Provider
from api.utils import initialize_prowler_integration, initialize_prowler_provider
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from django.db import OperationalError
from prowler.lib.outputs.asff.asff import ASFF
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.csv.csv import CSV
@@ -24,6 +22,7 @@ from prowler.providers.aws.lib.security_hub.exceptions.exceptions import (
)
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
from prowler.providers.common.models import Connection
from tasks.utils import batched
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/lighthouse_providers.py
+15 -18
View File
@@ -1,14 +1,11 @@
from typing import Dict
import boto3
import openai
from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
from botocore import UNSIGNED
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from celery.utils.log import get_task_logger
from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
logger = get_task_logger(__name__)
# OpenAI model prefixes to exclude from Lighthouse model selection.
@@ -104,7 +101,7 @@ def _extract_openai_api_key(
def _extract_openai_compatible_params(
provider_cfg: LighthouseProviderConfiguration,
) -> Dict[str, str] | None:
) -> dict[str, str] | None:
"""
Extract base_url and api_key for OpenAI-compatible providers.
"""
@@ -122,7 +119,7 @@ def _extract_openai_compatible_params(
def _extract_bedrock_credentials(
provider_cfg: LighthouseProviderConfiguration,
) -> Dict[str, str] | None:
) -> dict[str, str] | None:
"""
Safely extract AWS Bedrock credentials from a provider configuration.
@@ -177,7 +174,7 @@ def _extract_bedrock_credentials(
def _create_bedrock_client(
bedrock_creds: Dict[str, str], service_name: str = "bedrock"
bedrock_creds: dict[str, str], service_name: str = "bedrock"
):
"""
Create a boto3 Bedrock client with the appropriate authentication method.
@@ -221,7 +218,7 @@ def _create_bedrock_client(
)
def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
def check_lighthouse_provider_connection(provider_config_id: str) -> dict:
"""
Validate a Lighthouse provider configuration by calling the provider API and
toggle its active state accordingly.
@@ -314,7 +311,7 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
return {"connected": False, "error": error_message}
def _fetch_openai_models(api_key: str) -> Dict[str, str]:
def _fetch_openai_models(api_key: str) -> dict[str, str]:
"""
Fetch available models from OpenAI API.
@@ -355,7 +352,7 @@ def _fetch_openai_models(api_key: str) -> Dict[str, str]:
return filtered_models
def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]:
def _fetch_openai_compatible_models(base_url: str, api_key: str) -> dict[str, str]:
"""
Fetch available models from an OpenAI-compatible API using the OpenAI SDK.
@@ -367,7 +364,7 @@ def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, st
client = openai.OpenAI(api_key=api_key, base_url=base_url)
models = client.models.list()
available_models: Dict[str, str] = {}
available_models: dict[str, str] = {}
for model in models.data:
model_id = model.id
# Prefer provider-supplied human-friendly name when available
@@ -462,7 +459,7 @@ def _extract_foundation_model_ids(profile_models: list) -> list[str]:
def _build_inference_profile_map(
bedrock_client, region: str
) -> Dict[str, tuple[str, str]]:
) -> dict[str, tuple[str, str]]:
"""
Build map of foundation_model_id -> best inference profile.
@@ -472,7 +469,7 @@ def _build_inference_profile_map(
Prefers region-matched profiles over others
"""
region_prefix = _get_region_prefix(region)
model_to_profile: Dict[str, tuple[str, str]] = {}
model_to_profile: dict[str, tuple[str, str]] = {}
try:
response = bedrock_client.list_inference_profiles()
@@ -533,7 +530,7 @@ def _check_on_demand_availability(bedrock_client, model_id: str) -> bool:
return False
def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
def _fetch_bedrock_models(bedrock_creds: dict[str, str]) -> dict[str, str]:
"""
Fetch available models from AWS Bedrock, preferring inference profiles over ON_DEMAND.
@@ -560,7 +557,7 @@ def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
foundation_response = bedrock_client.list_foundation_models()
model_summaries = foundation_response.get("modelSummaries", [])
models_to_return: Dict[str, str] = {}
models_to_return: dict[str, str] = {}
on_demand_models: set[str] = set()
for model in model_summaries:
@@ -585,7 +582,7 @@ def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
models_to_return[model_id] = model_name
on_demand_models.add(model_id)
available_models: Dict[str, str] = {}
available_models: dict[str, str] = {}
for model_id, model_name in models_to_return.items():
if model_id in on_demand_models:
@@ -597,7 +594,7 @@ def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
return available_models
def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
def refresh_lighthouse_provider_models(provider_config_id: str) -> dict:
"""
Refresh the catalog of models for a Lighthouse provider configuration.
@@ -619,7 +616,7 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID.
"""
provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id)
fetched_models: Dict[str, str] = {}
fetched_models: dict[str, str] = {}
try:
if (
api/src/backend/tasks/jobs/muting.py
+2 -3
View File
@@ -1,10 +1,9 @@
from api.db_utils import rls_transaction
from api.models import Finding, MuteRule
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from tasks.utils import batched
from api.db_utils import rls_transaction
from api.models import Finding, MuteRule
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/orphan_recovery.py
+3 -3
View File
@@ -21,7 +21,7 @@ This is the shared engine behind both the periodic Beat watchdog and the
import ast
import json
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from celery import current_app, states
@@ -213,7 +213,7 @@ def _reconcile_task_results(
) -> dict:
from django_celery_results.models import TaskResult
cutoff = datetime.now(tz=timezone.utc) - timedelta(minutes=grace_minutes)
cutoff = datetime.now(tz=UTC) - timedelta(minutes=grace_minutes)
candidates = list(
TaskResult.objects.filter(status__in=IN_FLIGHT_STATES, date_created__lt=cutoff)
.exclude(worker__isnull=True)
@@ -278,7 +278,7 @@ def _recover_task(task_result, max_attempts: int, window_hours: int) -> str:
name = task_result.task_name
args_repr = task_result.task_args
kwargs_repr = task_result.task_kwargs
now = datetime.now(tz=timezone.utc)
now = datetime.now(tz=UTC)
# Drop any future broker redelivery of the stale message.
revoke_task(task_result, terminate=False)
api/src/backend/tasks/jobs/report.py
+10 -11
View File
@@ -1,3 +1,4 @@
import fcntl
import gc
import os
import re
@@ -7,9 +8,17 @@ from pathlib import Path
from shutil import rmtree
from uuid import UUID
import fcntl
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import rls_transaction
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
from api.utils import initialize_prowler_provider
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_TMP_OUTPUT_DIRECTORY
from prowler.lib.check.compliance_models import (
Compliance,
get_bulk_compliance_frameworks_universal,
)
from prowler.lib.outputs.finding import Finding as FindingOutput
from tasks.jobs.export import _generate_compliance_output_directory, _upload_to_s3
from tasks.jobs.reports import (
FRAMEWORK_REGISTRY,
@@ -25,16 +34,6 @@ from tasks.jobs.threatscore_utils import (
_get_compliance_check_ids,
)
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import rls_transaction
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
from api.utils import initialize_prowler_provider
from prowler.lib.check.compliance_models import (
Compliance,
get_bulk_compliance_frameworks_universal,
)
from prowler.lib.outputs.finding import Finding as FindingOutput
logger = get_task_logger(__name__)
STALE_TMP_OUTPUT_MAX_AGE_HOURS = 48
STALE_TMP_OUTPUT_MAX_DELETIONS_PER_RUN = 50
api/src/backend/tasks/jobs/reports/base.py
+9 -10
View File
@@ -8,7 +8,16 @@ from dataclasses import dataclass, field
from types import SimpleNamespace
from typing import Any
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Provider, StatusChoices
from api.utils import initialize_prowler_provider
from celery.utils.log import get_task_logger
from prowler.lib.check.compliance_models import (
Compliance,
get_bulk_compliance_frameworks_universal,
)
from prowler.lib.outputs.finding import Finding as FindingOutput
from reportlab.lib.enums import TA_CENTER
from reportlab.lib.pagesizes import letter
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
@@ -23,16 +32,6 @@ from tasks.jobs.threatscore_utils import (
_load_findings_for_requirement_checks,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Provider, StatusChoices
from api.utils import initialize_prowler_provider
from prowler.lib.check.compliance_models import (
Compliance,
get_bulk_compliance_frameworks_universal,
)
from prowler.lib.outputs.finding import Finding as FindingOutput
from .components import (
ColumnConfig,
create_data_table,
api/src/backend/tasks/jobs/reports/charts.py
+1 -1
View File
@@ -2,7 +2,7 @@ import gc
import io
import math
import time
from typing import Callable
from collections.abc import Callable
import matplotlib
from celery.utils.log import get_task_logger
api/src/backend/tasks/jobs/reports/cis.py
+1 -2
View File
@@ -3,11 +3,10 @@ import re
from collections import defaultdict
from typing import Any
from api.models import StatusChoices
from reportlab.lib.units import inch
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
from api.models import StatusChoices
from .base import (
BaseComplianceReportGenerator,
ComplianceData,
api/src/backend/tasks/jobs/reports/components.py
+2 -1
View File
@@ -1,5 +1,6 @@
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any, Callable
from typing import Any
from reportlab.lib import colors
from reportlab.lib.styles import ParagraphStyle
api/src/backend/tasks/jobs/reports/csa.py
+1 -2
View File
@@ -1,11 +1,10 @@
from collections import defaultdict
from api.models import StatusChoices
from celery.utils.log import get_task_logger
from reportlab.lib.units import inch
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
from api.models import StatusChoices
from .base import (
BaseComplianceReportGenerator,
ComplianceData,
api/src/backend/tasks/jobs/reports/ens.py
+1 -2
View File
@@ -1,13 +1,12 @@
import os
from collections import defaultdict
from api.models import StatusChoices
from reportlab.lib import colors
from reportlab.lib.styles import ParagraphStyle
from reportlab.lib.units import inch
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
from api.models import StatusChoices
from .base import (
BaseComplianceReportGenerator,
ComplianceData,
api/src/backend/tasks/jobs/reports/nis2.py
+1 -2
View File
@@ -1,11 +1,10 @@
import os
from collections import defaultdict
from api.models import StatusChoices
from reportlab.lib.units import inch
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
from api.models import StatusChoices
from .base import (
BaseComplianceReportGenerator,
ComplianceData,
api/src/backend/tasks/jobs/reports/threatscore.py
+1 -2
View File
@@ -1,12 +1,11 @@
import gc
from api.models import StatusChoices
from reportlab.lib import colors
from reportlab.lib.styles import ParagraphStyle
from reportlab.lib.units import inch
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
from api.models import StatusChoices
from .base import (
BaseComplianceReportGenerator,
ComplianceData,
api/src/backend/tasks/jobs/scan.py
+33 -38
View File
@@ -6,34 +6,10 @@ import time
import uuid
from collections import defaultdict
from collections.abc import Iterable
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
import sentry_sdk
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from config.env import env
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
from django.db import IntegrityError, OperationalError
from django.db.models import (
Case,
Count,
Exists,
IntegerField,
Max,
Min,
OuterRef,
Q,
Sum,
When,
)
from django.utils import timezone as django_timezone
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
)
from tasks.utils import CustomEncoder, batched
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
from api.constants import SEVERITY_ORDER
from api.db_router import READ_REPLICA_ALIAS, MainRouter
@@ -68,9 +44,32 @@ from api.models import (
from api.models import StatusChoices as FindingStatus
from api.utils import initialize_prowler_provider, return_prowler_provider
from api.v1.serializers import ScanTaskSerializer
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from config.env import env
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
from django.db import IntegrityError, OperationalError
from django.db.models import (
Case,
Count,
Exists,
IntegerField,
Max,
Min,
OuterRef,
Q,
Sum,
When,
)
from django.utils import timezone as django_timezone
from prowler.lib.check.models import CheckMetadata
from prowler.lib.outputs.finding import Finding as ProwlerFinding
from prowler.lib.scan.scan import Scan as ProwlerScan
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
)
from tasks.utils import CustomEncoder, batched
logger = get_task_logger(__name__)
@@ -311,7 +310,7 @@ def _copy_compliance_requirement_rows(
csv_buffer = io.StringIO()
writer = csv.writer(csv_buffer)
datetime_now = datetime.now(tz=timezone.utc)
datetime_now = datetime.now(tz=UTC)
for row in rows:
writer.writerow(
[
@@ -783,7 +782,7 @@ def _process_finding_micro_batch(
delta = _create_finding_delta(last_status, status)
if not last_first_seen_at:
last_first_seen_at = datetime.now(tz=timezone.utc)
last_first_seen_at = datetime.now(tz=UTC)
# Determine if finding should be muted and why
# Priority: mutelist processor (highest) > manual mute rules
@@ -814,7 +813,7 @@ def _process_finding_micro_batch(
scan=scan_instance,
first_seen_at=last_first_seen_at,
muted=is_muted,
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
muted_at=datetime.now(tz=UTC) if is_muted else None,
muted_reason=muted_reason,
compliance=finding.compliance,
categories=check_metadata.get("categories", []) or [],
@@ -941,7 +940,7 @@ def _process_finding_micro_batch(
set(dirty_resources.keys()) | resources_with_new_tag_mappings
)
if all_resource_uids_to_touch:
now_utc = datetime.now(tz=timezone.utc)
now_utc = datetime.now(tz=UTC)
resources_to_bulk_update = []
for uid in all_resource_uids_to_touch:
# Use the instance from dirty_resources if present (has mutated
@@ -1035,7 +1034,7 @@ def perform_prowler_scan(
provider_instance = Provider.objects.get(pk=provider_id)
scan_instance = Scan.objects.get(pk=scan_id)
scan_instance.state = StateChoices.EXECUTING
scan_instance.started_at = datetime.now(tz=timezone.utc)
scan_instance.started_at = datetime.now(tz=UTC)
scan_instance.save(update_fields=["state", "started_at", "updated_at"])
# Find the mutelist processor if it exists
@@ -1078,9 +1077,7 @@ def perform_prowler_scan(
f"Provider {provider_instance.provider} is not connected: {e}"
)
finally:
provider_instance.connection_last_checked_at = datetime.now(
tz=timezone.utc
)
provider_instance.connection_last_checked_at = datetime.now(tz=UTC)
provider_instance.save(
update_fields=[
"connected",
@@ -1181,7 +1178,7 @@ def perform_prowler_scan(
finally:
with rls_transaction(tenant_id):
scan_instance.duration = time.time() - start_time
scan_instance.completed_at = datetime.now(tz=timezone.utc)
scan_instance.completed_at = datetime.now(tz=UTC)
scan_instance.unique_resource_count = len(unique_resources)
scan_instance.save(
update_fields=[
@@ -1588,7 +1585,7 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
else:
requirement_stats["failed_checks"] += 1
utc_datetime_now = datetime.now(tz=timezone.utc)
utc_datetime_now = datetime.now(tz=UTC)
tenant_id_str = str(tenant_id)
scan_id_str = str(scan_instance.id)
@@ -2073,9 +2070,7 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
summary_timestamp = scan.completed_at
if django_timezone.is_naive(summary_timestamp):
summary_timestamp = django_timezone.make_aware(
summary_timestamp, timezone.utc
)
summary_timestamp = django_timezone.make_aware(summary_timestamp, UTC)
summary_timestamp = summary_timestamp.replace(
hour=0, minute=0, second=0, microsecond=0
)
api/src/backend/tasks/jobs/threatscore.py
+4 -5
View File
@@ -1,14 +1,13 @@
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Provider, StatusChoices
from celery.utils.log import get_task_logger
from prowler.lib.check.compliance_models import Compliance
from tasks.jobs.threatscore_utils import (
_aggregate_requirement_statistics_from_database,
_calculate_requirements_data_from_statistics,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Provider, StatusChoices
from prowler.lib.check.compliance_models import Compliance
logger = get_task_logger(__name__)
api/src/backend/tasks/jobs/threatscore_utils.py
+4 -5
View File
@@ -1,13 +1,12 @@
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Finding, Scan, StatusChoices
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from django.db.models import Count, F, Q, Window
from django.db.models.functions import RowNumber
from tasks.jobs.reports.config import MAX_FINDINGS_PER_CHECK
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Finding, Scan, StatusChoices
from prowler.lib.outputs.finding import Finding as FindingOutput
from tasks.jobs.reports.config import MAX_FINDINGS_PER_CHECK
logger = get_task_logger(__name__)
api/src/backend/tasks/tasks.py
+20 -22
View File
@@ -1,13 +1,29 @@
import os
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from pathlib import Path
from shutil import rmtree
from api.compliance import (
get_compliance_frameworks,
get_prowler_provider_compliance,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import delete_related_daily_task, rls_transaction
from api.decorators import handle_provider_deletion, set_tenant
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
from api.utils import initialize_prowler_provider
from api.v1.serializers import ScanTaskSerializer
from celery import chain, group, shared_task
from celery.utils.log import get_task_logger
from config.celery import RLSTask
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
from django_celery_beat.models import PeriodicTask
from prowler.lib.check.compliance_models import Compliance
from prowler.lib.outputs.compliance.compliance import (
process_universal_compliance_frameworks,
)
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.finding import Finding as FindingOutput
from tasks.jobs.attack_paths import (
attack_paths_scan,
can_provider_run_attack_paths_scan,
@@ -15,13 +31,13 @@ from tasks.jobs.attack_paths import (
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
from tasks.jobs.backfill import (
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
backfill_compliance_summaries,
backfill_daily_severity_summaries,
backfill_finding_group_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
)
from tasks.jobs.connection import (
check_integration_connection,
@@ -68,24 +84,6 @@ from tasks.utils import (
get_next_execution_datetime,
)
from api.compliance import (
get_compliance_frameworks,
get_prowler_provider_compliance,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import delete_related_daily_task, rls_transaction
from api.decorators import handle_provider_deletion, set_tenant
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
from api.utils import initialize_prowler_provider
from api.v1.serializers import ScanTaskSerializer
from prowler.lib.check.compliance_models import Compliance
from prowler.lib.outputs.compliance.compliance import (
process_universal_compliance_frameworks,
)
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.finding import Finding as FindingOutput
logger = get_task_logger(__name__)
@@ -407,7 +405,7 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
)
finally:
with rls_transaction(tenant_id):
now = datetime.now(timezone.utc)
now = datetime.now(UTC)
if next_scan_datetime <= now:
interval_delta = timedelta(**{interval.period: interval.every})
while next_scan_datetime <= now:
api/src/backend/tasks/tests/test_attack_paths_scan.py
+11 -12
View File
@@ -1,16 +1,9 @@
from contextlib import nullcontext
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from unittest.mock import MagicMock, call, patch
import pytest
from django_celery_results.models import TaskResult
from tasks.jobs.attack_paths import findings as findings_module
from tasks.jobs.attack_paths import indexes as indexes_module
from tasks.jobs.attack_paths import internet as internet_module
from tasks.jobs.attack_paths import sync as sync_module
from tasks.jobs.attack_paths.scan import run as attack_paths_run
from api.models import (
AttackPathsScan,
Finding,
@@ -22,7 +15,13 @@ from api.models import (
StatusChoices,
Task,
)
from django_celery_results.models import TaskResult
from prowler.lib.check.models import Severity
from tasks.jobs.attack_paths import findings as findings_module
from tasks.jobs.attack_paths import indexes as indexes_module
from tasks.jobs.attack_paths import internet as internet_module
from tasks.jobs.attack_paths import sync as sync_module
from tasks.jobs.attack_paths.scan import run as attack_paths_run
@pytest.mark.django_db
@@ -2374,7 +2373,7 @@ class TestCleanupStaleAttackPathsScans:
provider=provider,
scan=scan,
state=StateChoices.EXECUTING,
started_at=started_at or datetime.now(tz=timezone.utc),
started_at=started_at or datetime.now(tz=UTC),
)
task_result = None
@@ -2467,7 +2466,7 @@ class TestCleanupStaleAttackPathsScans:
provider.provider = Provider.ProviderChoices.AWS
provider.save()
old_start = datetime.now(tz=timezone.utc) - timedelta(hours=49)
old_start = datetime.now(tz=UTC) - timedelta(hours=49)
ap_scan, task_result = self._create_executing_scan(
tenant, provider, started_at=old_start, worker="live-worker@host"
)
@@ -2685,7 +2684,7 @@ class TestCleanupStaleAttackPathsScans:
provider.save()
# Old scan with no Task/TaskResult
old_start = datetime.now(tz=timezone.utc) - timedelta(hours=49)
old_start = datetime.now(tz=UTC) - timedelta(hours=49)
ap_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
@@ -2761,7 +2760,7 @@ class TestCleanupStaleAttackPathsScans:
provider=provider,
scan=parent_scan,
state=StateChoices.SCHEDULED,
started_at=datetime.now(tz=timezone.utc) - timedelta(minutes=age_minutes),
started_at=datetime.now(tz=UTC) - timedelta(minutes=age_minutes),
)
task_result = None
api/src/backend/tasks/tests/test_backfill.py
+9 -10
View File
@@ -1,16 +1,8 @@
from datetime import datetime, timezone
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
from tasks.jobs.backfill import (
backfill_compliance_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
)
from api.models import (
ComplianceOverviewSummary,
Finding,
@@ -24,6 +16,13 @@ from api.models import (
)
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.backfill import (
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
backfill_compliance_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
)
@pytest.fixture(scope="function")
@@ -536,7 +535,7 @@ class TestBackfillProviderComplianceScores:
scan2 = scans_fixture[1]
# Set completed_at to make the scan eligible for backfill
scan.completed_at = datetime.now(timezone.utc)
scan.completed_at = datetime.now(UTC)
scan.save()
scan2.state = StateChoices.AVAILABLE
scan2.completed_at = None
api/src/backend/tasks/tests/test_beat.py
+2 -3
View File
@@ -2,11 +2,10 @@ import json
from unittest.mock import patch
import pytest
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from tasks.beat import schedule_provider_scan
from api.exceptions import ConflictException
from api.models import Scan
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from tasks.beat import schedule_provider_scan
@pytest.mark.django_db
api/src/backend/tasks/tests/test_connection.py
+3 -4
View File
@@ -1,16 +1,15 @@
import uuid
from datetime import datetime, timezone
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
import pytest
from api.models import Integration, LighthouseConfiguration, Provider
from tasks.jobs.connection import (
check_integration_connection,
check_lighthouse_connection,
check_provider_connection,
)
from api.models import Integration, LighthouseConfiguration, Provider
@pytest.mark.parametrize(
"provider_data",
@@ -38,7 +37,7 @@ def test_check_provider_connection(
mock_provider_connection_test.assert_called_once()
assert provider.connected is True
assert provider.connection_last_checked_at is not None
assert provider.connection_last_checked_at <= datetime.now(tz=timezone.utc)
assert provider.connection_last_checked_at <= datetime.now(tz=UTC)
@patch("tasks.jobs.connection.Provider.objects.get")
api/src/backend/tasks/tests/test_deletion.py
+2 -3
View File
@@ -1,11 +1,10 @@
from unittest.mock import call, patch
import pytest
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.deletion import delete_provider, delete_tenant
from api.attack_paths import database as graph_database
from api.models import Provider, Tenant, TenantComplianceSummary
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.deletion import delete_provider, delete_tenant
@pytest.mark.django_db
api/src/backend/tasks/tests/test_integrations.py
+7 -10
View File
@@ -1,7 +1,12 @@
from unittest.mock import MagicMock, patch
import pytest
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.models import Integration
from api.utils import prowler_integration_connection_test
from django.db import OperationalError
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.common.models import Connection
from tasks.jobs.integrations import (
get_s3_client_from_integration,
get_security_hub_client_from_integration,
@@ -10,12 +15,6 @@ from tasks.jobs.integrations import (
upload_security_hub_integration,
)
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.models import Integration
from api.utils import prowler_integration_connection_test
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.common.models import Connection
@pytest.mark.django_db
class TestS3IntegrationUploads:
@@ -264,10 +263,9 @@ class TestS3IntegrationUploads:
def test_s3_integration_rejects_invalid_output_directory_characters(self):
"""Test that S3 integration validation rejects invalid characters."""
from rest_framework.exceptions import ValidationError
from api.models import Integration
from api.v1.serializers import BaseWriteIntegrationSerializer
from rest_framework.exceptions import ValidationError
integration_type = Integration.IntegrationChoices.AMAZON_S3
providers = []
@@ -290,10 +288,9 @@ class TestS3IntegrationUploads:
def test_s3_integration_rejects_empty_output_directory(self):
"""Test that S3 integration validation rejects empty directories."""
from rest_framework.exceptions import ValidationError
from api.models import Integration
from api.v1.serializers import BaseWriteIntegrationSerializer
from rest_framework.exceptions import ValidationError
integration_type = Integration.IntegrationChoices.AMAZON_S3
providers = []
api/src/backend/tasks/tests/test_muting.py
+4 -5
View File
@@ -1,13 +1,12 @@
from datetime import datetime, timezone
from datetime import UTC, datetime
from uuid import uuid4
import pytest
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.muting import mute_historical_findings
from api.models import Finding, MuteRule
from django.core.exceptions import ObjectDoesNotExist
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.muting import mute_historical_findings
@pytest.mark.django_db
@@ -162,7 +161,7 @@ class TestMuteHistoricalFindings:
"Description": f"Muted description {i}",
},
muted=True,
muted_at=datetime.now(timezone.utc),
muted_at=datetime.now(UTC),
muted_reason="Already muted",
)
muted_uids.append(finding.uid)
api/src/backend/tasks/tests/test_orphan_recovery.py
+2 -4
View File
@@ -1,4 +1,4 @@
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
from uuid import uuid4
@@ -6,7 +6,6 @@ import pytest
from celery import states
from django.test import override_settings
from django_celery_results.models import TaskResult
from tasks.jobs.orphan_recovery import (
_decode_celery_field,
_reconcile_task_results,
@@ -29,8 +28,7 @@ def _orphan_result(*, name, kwargs, worker, created_minutes_ago, status=states.S
task_args=repr([]),
)
TaskResult.objects.filter(pk=tr.pk).update(
date_created=datetime.now(tz=timezone.utc)
- timedelta(minutes=created_minutes_ago)
date_created=datetime.now(tz=UTC) - timedelta(minutes=created_minutes_ago)
)
tr.refresh_from_db()
return tr
api/src/backend/tasks/tests/test_reports.py
+31 -31
View File
@@ -5,10 +5,20 @@ from unittest.mock import Mock, patch
import matplotlib
import pytest
from api.models import (
Finding,
Resource,
ResourceFindingMapping,
ResourceTag,
ResourceTagMapping,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from reportlab.lib import colors
from tasks.jobs.report import (
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
STALE_TMP_OUTPUT_LOCK_FILE_NAME,
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
_cleanup_stale_tmp_output_directories,
_is_scan_directory_protected,
_pick_latest_cis_variant,
@@ -40,17 +50,6 @@ from tasks.jobs.threatscore_utils import (
_load_findings_for_requirement_checks,
)
from api.models import (
Finding,
Resource,
ResourceFindingMapping,
ResourceTag,
ResourceTagMapping,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
matplotlib.use("Agg") # Use non-interactive backend for tests
@@ -377,8 +376,8 @@ class TestLoadFindingsForChecks:
finding. Without ``prefetch_related`` that's 2N additional queries;
with prefetch it collapses to a small constant per iterator chunk.
"""
from django.test.utils import CaptureQueriesContext
from django.db import connections
from django.test.utils import CaptureQueriesContext
tenant = tenants_fixture[0]
scan = scans_fixture[0]
@@ -539,12 +538,12 @@ class TestLoadFindingsForChecks:
total_counts_out=totals,
)
assert (
len(result[check_id]) == 5
), f"cap=5 should yield exactly 5 loaded findings, got {len(result[check_id])}"
assert (
totals[check_id] == 12
), f"total_counts_out should report the pre-cap total (12), got {totals[check_id]}"
assert len(result[check_id]) == 5, (
f"cap=5 should yield exactly 5 loaded findings, got {len(result[check_id])}"
)
assert totals[check_id] == 12, (
f"total_counts_out should report the pre-cap total (12), got {totals[check_id]}"
)
def test_only_failed_findings_pushes_down_to_sql(
self, tenants_fixture, scans_fixture
@@ -616,13 +615,13 @@ class TestLoadFindingsForChecks:
loaded = result[check_id]
assert len(loaded) == 3, f"expected 3 FAIL findings, got {len(loaded)}"
statuses = {getattr(f, "status", None) for f in loaded}
assert statuses == {
StatusChoices.FAIL
}, f"expected all loaded findings to be FAIL; got statuses {statuses}"
assert statuses == {StatusChoices.FAIL}, (
f"expected all loaded findings to be FAIL; got statuses {statuses}"
)
# total_counts must reflect the FAIL-only total, not the global total.
assert (
totals[check_id] == 3
), f"total_counts should be FAIL-only (3), got {totals[check_id]}"
assert totals[check_id] == 3, (
f"total_counts should be FAIL-only (3), got {totals[check_id]}"
)
def test_max_findings_per_check_disabled(self, tenants_fixture, scans_fixture):
"""``MAX_FINDINGS_PER_CHECK=0`` disables the cap; load all rows."""
@@ -1205,6 +1204,7 @@ class TestGenerateComplianceReportsOptimized:
ThreatScore finishes, before ENS runs.
"""
from types import SimpleNamespace
from tasks.jobs import report as report_mod
mock_scan_summary_filter.return_value.exists.return_value = True
@@ -1259,12 +1259,12 @@ class TestGenerateComplianceReportsOptimized:
# ``tsc_only`` was exclusive to ThreatScore → evicted before ENS ran.
# ``shared`` is still pending for ENS → must remain.
assert (
"tsc_only" not in observed_state["cache_keys_when_ens_runs"]
), "tsc_only should have been evicted before ENS ran"
assert (
"shared" in observed_state["cache_keys_when_ens_runs"]
), "shared must remain in cache because ENS still needs it"
assert "tsc_only" not in observed_state["cache_keys_when_ens_runs"], (
"tsc_only should have been evicted before ENS ran"
)
assert "shared" in observed_state["cache_keys_when_ens_runs"], (
"shared must remain in cache because ENS still needs it"
)
@patch("tasks.jobs.report.initialize_prowler_provider")
@patch("tasks.jobs.report.rmtree")
api/src/backend/tasks/tests/test_reports_cis.py
+1 -2
View File
@@ -1,6 +1,7 @@
from unittest.mock import Mock, patch
import pytest
from api.models import StatusChoices
from reportlab.platypus import Image, LongTable, Paragraph, Table
from tasks.jobs.reports import FRAMEWORK_REGISTRY, ComplianceData, RequirementData
from tasks.jobs.reports.cis import (
@@ -9,8 +10,6 @@ from tasks.jobs.reports.cis import (
_profile_badge_text,
)
from api.models import StatusChoices
# =============================================================================
# Fixtures
# =============================================================================
api/src/backend/tasks/tests/test_reports_threatscore.py
+1 -2
View File
@@ -2,6 +2,7 @@ import io
from unittest.mock import Mock
import pytest
from api.models import StatusChoices
from reportlab.platypus import Image, PageBreak, Paragraph, Table
from tasks.jobs.reports import (
FRAMEWORK_REGISTRY,
@@ -10,8 +11,6 @@ from tasks.jobs.reports import (
ThreatScoreReportGenerator,
)
from api.models import StatusChoices
# =============================================================================
# Fixtures
# =============================================================================
api/src/backend/tasks/tests/test_scan.py
+32 -32
View File
@@ -3,11 +3,26 @@ import json
import re
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from datetime import UTC, datetime
from io import StringIO
from unittest.mock import MagicMock, patch
import pytest
from api.db_router import MainRouter
from api.exceptions import ProviderConnectionError
from api.models import (
Finding,
MuteRule,
Provider,
Resource,
ResourceScanSummary,
Scan,
ScanSummary,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.scan import (
_ATTACK_SURFACE_MAPPING_CACHE,
_aggregate_findings_by_region,
@@ -29,22 +44,6 @@ from tasks.jobs.scan import (
)
from tasks.utils import CustomEncoder
from api.db_router import MainRouter
from api.exceptions import ProviderConnectionError
from api.models import (
Finding,
MuteRule,
Provider,
Resource,
ResourceScanSummary,
Scan,
ScanSummary,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
@contextmanager
def noop_rls_transaction(*args, **kwargs):
@@ -1335,9 +1334,9 @@ class TestPerformScan:
)
# Capture time before and after scan
before_scan = datetime.now(timezone.utc)
before_scan = datetime.now(UTC)
perform_prowler_scan(tenant_id, scan_id, provider_id, [])
after_scan = datetime.now(timezone.utc)
after_scan = datetime.now(UTC)
# Verify muted_at is within the scan time window
finding_db = Finding.objects.get(uid=finding_uid)
@@ -1473,7 +1472,7 @@ class TestProcessFindingMicroBatch:
partition="aws-old",
)
previous_first_seen = datetime(2024, 1, 1, tzinfo=timezone.utc)
previous_first_seen = datetime(2024, 1, 1, tzinfo=UTC)
finding = FakeFinding(
uid="finding-muted",
@@ -2123,7 +2122,7 @@ class TestCreateComplianceRequirements:
tags={},
check_id=existing_finding.check_id,
check_metadata={"CheckId": existing_finding.check_id},
first_seen_at=datetime.now(timezone.utc),
first_seen_at=datetime.now(UTC),
muted=False,
)
resource = existing_finding.resources.first()
@@ -2313,7 +2312,7 @@ class TestComplianceRequirementCopy:
def test_persist_compliance_requirement_rows_fallback(
self, mock_copy, mock_rls_transaction, mock_bulk_create
):
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
row = {
"id": uuid.uuid4(),
"tenant_id": str(uuid.uuid4()),
@@ -2394,7 +2393,7 @@ class TestComplianceRequirementCopy:
tenant_id = str(uuid.uuid4())
scan_id = uuid.uuid4()
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
rows = [
{
@@ -2644,10 +2643,10 @@ class TestComplianceRequirementCopy:
# Note: inserted_at is intentionally missing
}
before_call = datetime.now(timezone.utc)
before_call = datetime.now(UTC)
with patch.object(MainRouter, "admin_db", "admin"):
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
after_call = datetime.now(timezone.utc)
after_call = datetime.now(UTC)
csv_rows = list(csv.reader(StringIO(captured["data"])))
assert len(csv_rows) == 1
@@ -2811,7 +2810,7 @@ class TestComplianceRequirementCopy:
{
"id": uuid.uuid4(),
"tenant_id": tenant_id,
"inserted_at": datetime.now(timezone.utc),
"inserted_at": datetime.now(UTC),
"compliance_id": "test",
"framework": "Test",
"version": "1.0",
@@ -2846,7 +2845,7 @@ class TestComplianceRequirementCopy:
row = {
"id": uuid.uuid4(),
"tenant_id": tenant_id,
"inserted_at": datetime.now(timezone.utc),
"inserted_at": datetime.now(UTC),
"compliance_id": "test",
"framework": "Test",
"version": "1.0",
@@ -2886,7 +2885,7 @@ class TestComplianceRequirementCopy:
"""Test ORM fallback with multiple rows."""
tenant_id = str(uuid.uuid4())
scan_id = uuid.uuid4()
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
rows = [
{
@@ -2968,7 +2967,7 @@ class TestComplianceRequirementCopy:
tenant_id = str(uuid.uuid4())
row_id = uuid.uuid4()
scan_id = uuid.uuid4()
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
row = {
"id": row_id,
@@ -4461,7 +4460,7 @@ class TestUpdateProviderComplianceScores:
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = datetime.now(timezone.utc)
scan.completed_at = datetime.now(UTC)
scan.save()
connection = MagicMock()
@@ -4538,7 +4537,7 @@ class TestUpdateProviderComplianceScores:
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = datetime.now(timezone.utc)
scan.completed_at = datetime.now(UTC)
scan.save()
connection = MagicMock()
@@ -4591,9 +4590,10 @@ class TestScanIsFullScope:
# If the SDK adds a new filter, this test still passes via the
# introspection-driven derivation; if it adds a non-filter kwarg
# (e.g. provider-like), keep the exclusion list in sync in models.py.
from prowler.lib.scan.scan import Scan as ProwlerScan
import inspect
from prowler.lib.scan.scan import Scan as ProwlerScan
expected = tuple(
name
for name in inspect.signature(ProwlerScan.__init__).parameters
api/src/backend/tasks/tests/test_tasks.py
+17 -18
View File
@@ -1,10 +1,18 @@
import uuid
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
import openai
import pytest
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Scan,
StateChoices,
Task,
)
from botocore.exceptions import ClientError
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from django_celery_results.models import TaskResult
@@ -31,15 +39,6 @@ from tasks.tasks import (
security_hub_integration_task,
)
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Scan,
StateChoices,
Task,
)
@pytest.mark.django_db
class TestExtractBedrockCredentials:
@@ -1478,9 +1477,9 @@ class TestCheckIntegrationsTask:
)
# Verify ASFF was NOT created for non-AWS provider
assert (
"asff" not in created_writers
), "ASFF writer should NOT be created for non-AWS providers"
assert "asff" not in created_writers, (
"ASFF writer should NOT be created for non-AWS providers"
)
assert "csv" in created_writers, "CSV writer should be created"
assert "ocsf" in created_writers, "OCSF writer should be created"
@@ -2328,7 +2327,7 @@ class TestPerformScheduledScanTask:
task_id=task_id,
task_name="scan-perform-scheduled",
status="STARTED",
date_created=datetime.now(timezone.utc),
date_created=datetime.now(UTC),
)
Task.objects.create(
id=task_id, task_runner_task=task_result, tenant_id=tenant_id
@@ -2416,7 +2415,7 @@ class TestPerformScheduledScanTask:
state=StateChoices.SCHEDULED,
)
assert scheduled_scans.count() == 1
assert scheduled_scans.first().scheduled_at > datetime.now(timezone.utc)
assert scheduled_scans.first().scheduled_at > datetime.now(UTC)
assert (
Scan.objects.filter(
tenant_id=tenant.id,
@@ -2452,7 +2451,7 @@ class TestPerformScheduledScanTask:
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduled_at=datetime.now(timezone.utc),
scheduled_at=datetime.now(UTC),
scheduler_task_id=periodic_task.id,
)
duplicate_scan = Scan.objects.create(
@@ -2582,7 +2581,7 @@ class TestReaggregateAllFindingGroupSummaries:
scan_id_today_p1 = uuid.uuid4()
scan_id_yesterday_p1 = uuid.uuid4()
scan_id_today_p2 = uuid.uuid4()
today = datetime.now(tz=timezone.utc)
today = datetime.now(tz=UTC)
yesterday = today - timedelta(days=1)
mock_outer_group_result = MagicMock()
@@ -2663,7 +2662,7 @@ class TestReaggregateAllFindingGroupSummaries:
provider_id = uuid.uuid4()
latest_scan_today = uuid.uuid4()
earlier_scan_today = uuid.uuid4()
today_late = datetime.now(tz=timezone.utc)
today_late = datetime.now(tz=UTC)
today_early = today_late - timedelta(hours=4)
mock_outer_group_result = MagicMock()
api/src/backend/tasks/tests/test_utils.py
+2 -2
View File
@@ -1,4 +1,4 @@
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from unittest.mock import patch
import pytest
@@ -29,7 +29,7 @@ class TestGetNextExecutionDatetime:
task_id="abc123",
task_name="scan-perform-scheduled",
status="SUCCESS",
date_created=datetime.now(timezone.utc) - timedelta(hours=1),
date_created=datetime.now(UTC) - timedelta(hours=1),
result="Success",
)
return task_result
api/src/backend/tasks/utils.py
+4 -5
View File
@@ -1,12 +1,11 @@
import json
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from enum import Enum
from api.models import Scan, StateChoices
from django_celery_beat.models import PeriodicTask
from django_celery_results.models import TaskResult
from api.models import Scan, StateChoices
SCHEDULED_SCAN_NAME = "Daily scheduled scan"
@@ -45,9 +44,9 @@ def get_next_execution_datetime(task_id: int, provider_id: str) -> datetime:
interval = periodic_task_instance.interval
current_scheduled_time = datetime.combine(
datetime.now(timezone.utc).date(),
datetime.now(UTC).date(),
task_instance.date_created.time(),
tzinfo=timezone.utc,
tzinfo=UTC,
)
return current_scheduled_time + timedelta(**{interval.period: interval.every})