chore: unify ruff tooling and route code quality through the Makefile (#11675)

This commit is contained in:
Rubén De la Torre Vico authored and GitHub committed 2026-06-23 17:15:05 +02:00
1 parent de7da3e960
commit 058a1dc8fe
194 files changed
+1112 -1210

No files matched your search

@@ -1,16 +1,9 @@
from contextlib import nullcontext
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from unittest.mock import MagicMock, call, patch
import pytest
from django_celery_results.models import TaskResult
from tasks.jobs.attack_paths import findings as findings_module
from tasks.jobs.attack_paths import indexes as indexes_module
from tasks.jobs.attack_paths import internet as internet_module
from tasks.jobs.attack_paths import sync as sync_module
from tasks.jobs.attack_paths.scan import run as attack_paths_run
from api.models import (
AttackPathsScan,
Finding,
@@ -22,7 +15,13 @@ from api.models import (
StatusChoices,
Task,
)
from django_celery_results.models import TaskResult
from prowler.lib.check.models import Severity
from tasks.jobs.attack_paths import findings as findings_module
from tasks.jobs.attack_paths import indexes as indexes_module
from tasks.jobs.attack_paths import internet as internet_module
from tasks.jobs.attack_paths import sync as sync_module
from tasks.jobs.attack_paths.scan import run as attack_paths_run
@pytest.mark.django_db
@@ -2374,7 +2373,7 @@ class TestCleanupStaleAttackPathsScans:
provider=provider,
scan=scan,
state=StateChoices.EXECUTING,
started_at=started_at or datetime.now(tz=timezone.utc),
started_at=started_at or datetime.now(tz=UTC),
)
task_result = None
@@ -2467,7 +2466,7 @@ class TestCleanupStaleAttackPathsScans:
provider.provider = Provider.ProviderChoices.AWS
provider.save()
old_start = datetime.now(tz=timezone.utc) - timedelta(hours=49)
old_start = datetime.now(tz=UTC) - timedelta(hours=49)
ap_scan, task_result = self._create_executing_scan(
tenant, provider, started_at=old_start, worker="live-worker@host"
)
@@ -2685,7 +2684,7 @@ class TestCleanupStaleAttackPathsScans:
provider.save()
# Old scan with no Task/TaskResult
old_start = datetime.now(tz=timezone.utc) - timedelta(hours=49)
old_start = datetime.now(tz=UTC) - timedelta(hours=49)
ap_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
@@ -2761,7 +2760,7 @@ class TestCleanupStaleAttackPathsScans:
provider=provider,
scan=parent_scan,
state=StateChoices.SCHEDULED,
started_at=datetime.now(tz=timezone.utc) - timedelta(minutes=age_minutes),
started_at=datetime.now(tz=UTC) - timedelta(minutes=age_minutes),
)
task_result = None
+9 -10
View File
@@ -1,16 +1,8 @@
from datetime import datetime, timezone
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
from tasks.jobs.backfill import (
backfill_compliance_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
)
from api.models import (
ComplianceOverviewSummary,
Finding,
@@ -24,6 +16,13 @@ from api.models import (
)
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.backfill import (
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
backfill_compliance_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
)
@pytest.fixture(scope="function")
@@ -536,7 +535,7 @@ class TestBackfillProviderComplianceScores:
scan2 = scans_fixture[1]
# Set completed_at to make the scan eligible for backfill
scan.completed_at = datetime.now(timezone.utc)
scan.completed_at = datetime.now(UTC)
scan.save()
scan2.state = StateChoices.AVAILABLE
scan2.completed_at = None
+2 -3
View File
@@ -2,11 +2,10 @@ import json
from unittest.mock import patch
import pytest
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from tasks.beat import schedule_provider_scan
from api.exceptions import ConflictException
from api.models import Scan
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from tasks.beat import schedule_provider_scan
@pytest.mark.django_db
@@ -1,16 +1,15 @@
import uuid
from datetime import datetime, timezone
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
import pytest
from api.models import Integration, LighthouseConfiguration, Provider
from tasks.jobs.connection import (
check_integration_connection,
check_lighthouse_connection,
check_provider_connection,
)
from api.models import Integration, LighthouseConfiguration, Provider
@pytest.mark.parametrize(
"provider_data",
@@ -38,7 +37,7 @@ def test_check_provider_connection(
mock_provider_connection_test.assert_called_once()
assert provider.connected is True
assert provider.connection_last_checked_at is not None
assert provider.connection_last_checked_at <= datetime.now(tz=timezone.utc)
assert provider.connection_last_checked_at <= datetime.now(tz=UTC)
@patch("tasks.jobs.connection.Provider.objects.get")
+2 -3
View File
@@ -1,11 +1,10 @@
from unittest.mock import call, patch
import pytest
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.deletion import delete_provider, delete_tenant
from api.attack_paths import database as graph_database
from api.models import Provider, Tenant, TenantComplianceSummary
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.deletion import delete_provider, delete_tenant
@pytest.mark.django_db
@@ -1,7 +1,12 @@
from unittest.mock import MagicMock, patch
import pytest
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.models import Integration
from api.utils import prowler_integration_connection_test
from django.db import OperationalError
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.common.models import Connection
from tasks.jobs.integrations import (
get_s3_client_from_integration,
get_security_hub_client_from_integration,
@@ -10,12 +15,6 @@ from tasks.jobs.integrations import (
upload_security_hub_integration,
)
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.models import Integration
from api.utils import prowler_integration_connection_test
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.common.models import Connection
@pytest.mark.django_db
class TestS3IntegrationUploads:
@@ -264,10 +263,9 @@ class TestS3IntegrationUploads:
def test_s3_integration_rejects_invalid_output_directory_characters(self):
"""Test that S3 integration validation rejects invalid characters."""
from rest_framework.exceptions import ValidationError
from api.models import Integration
from api.v1.serializers import BaseWriteIntegrationSerializer
from rest_framework.exceptions import ValidationError
integration_type = Integration.IntegrationChoices.AMAZON_S3
providers = []
@@ -290,10 +288,9 @@ class TestS3IntegrationUploads:
def test_s3_integration_rejects_empty_output_directory(self):
"""Test that S3 integration validation rejects empty directories."""
from rest_framework.exceptions import ValidationError
from api.models import Integration
from api.v1.serializers import BaseWriteIntegrationSerializer
from rest_framework.exceptions import ValidationError
integration_type = Integration.IntegrationChoices.AMAZON_S3
providers = []
+4 -5
View File
@@ -1,13 +1,12 @@
from datetime import datetime, timezone
from datetime import UTC, datetime
from uuid import uuid4
import pytest
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.muting import mute_historical_findings
from api.models import Finding, MuteRule
from django.core.exceptions import ObjectDoesNotExist
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.muting import mute_historical_findings
@pytest.mark.django_db
@@ -162,7 +161,7 @@ class TestMuteHistoricalFindings:
"Description": f"Muted description {i}",
},
muted=True,
muted_at=datetime.now(timezone.utc),
muted_at=datetime.now(UTC),
muted_reason="Already muted",
)
muted_uids.append(finding.uid)
@@ -1,4 +1,4 @@
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
from uuid import uuid4
@@ -6,7 +6,6 @@ import pytest
from celery import states
from django.test import override_settings
from django_celery_results.models import TaskResult
from tasks.jobs.orphan_recovery import (
_decode_celery_field,
_reconcile_task_results,
@@ -29,8 +28,7 @@ def _orphan_result(*, name, kwargs, worker, created_minutes_ago, status=states.S
task_args=repr([]),
)
TaskResult.objects.filter(pk=tr.pk).update(
date_created=datetime.now(tz=timezone.utc)
- timedelta(minutes=created_minutes_ago)
date_created=datetime.now(tz=UTC) - timedelta(minutes=created_minutes_ago)
)
tr.refresh_from_db()
return tr
+31 -31
View File
@@ -5,10 +5,20 @@ from unittest.mock import Mock, patch
import matplotlib
import pytest
from api.models import (
Finding,
Resource,
ResourceFindingMapping,
ResourceTag,
ResourceTagMapping,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from reportlab.lib import colors
from tasks.jobs.report import (
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
STALE_TMP_OUTPUT_LOCK_FILE_NAME,
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
_cleanup_stale_tmp_output_directories,
_is_scan_directory_protected,
_pick_latest_cis_variant,
@@ -40,17 +50,6 @@ from tasks.jobs.threatscore_utils import (
_load_findings_for_requirement_checks,
)
from api.models import (
Finding,
Resource,
ResourceFindingMapping,
ResourceTag,
ResourceTagMapping,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
matplotlib.use("Agg") # Use non-interactive backend for tests
@@ -377,8 +376,8 @@ class TestLoadFindingsForChecks:
finding. Without ``prefetch_related`` that's 2N additional queries;
with prefetch it collapses to a small constant per iterator chunk.
"""
from django.test.utils import CaptureQueriesContext
from django.db import connections
from django.test.utils import CaptureQueriesContext
tenant = tenants_fixture[0]
scan = scans_fixture[0]
@@ -539,12 +538,12 @@ class TestLoadFindingsForChecks:
total_counts_out=totals,
)
assert (
len(result[check_id]) == 5
), f"cap=5 should yield exactly 5 loaded findings, got {len(result[check_id])}"
assert (
totals[check_id] == 12
), f"total_counts_out should report the pre-cap total (12), got {totals[check_id]}"
assert len(result[check_id]) == 5, (
f"cap=5 should yield exactly 5 loaded findings, got {len(result[check_id])}"
)
assert totals[check_id] == 12, (
f"total_counts_out should report the pre-cap total (12), got {totals[check_id]}"
)
def test_only_failed_findings_pushes_down_to_sql(
self, tenants_fixture, scans_fixture
@@ -616,13 +615,13 @@ class TestLoadFindingsForChecks:
loaded = result[check_id]
assert len(loaded) == 3, f"expected 3 FAIL findings, got {len(loaded)}"
statuses = {getattr(f, "status", None) for f in loaded}
assert statuses == {
StatusChoices.FAIL
}, f"expected all loaded findings to be FAIL; got statuses {statuses}"
assert statuses == {StatusChoices.FAIL}, (
f"expected all loaded findings to be FAIL; got statuses {statuses}"
)
# total_counts must reflect the FAIL-only total, not the global total.
assert (
totals[check_id] == 3
), f"total_counts should be FAIL-only (3), got {totals[check_id]}"
assert totals[check_id] == 3, (
f"total_counts should be FAIL-only (3), got {totals[check_id]}"
)
def test_max_findings_per_check_disabled(self, tenants_fixture, scans_fixture):
"""``MAX_FINDINGS_PER_CHECK=0`` disables the cap; load all rows."""
@@ -1205,6 +1204,7 @@ class TestGenerateComplianceReportsOptimized:
ThreatScore finishes, before ENS runs.
"""
from types import SimpleNamespace
from tasks.jobs import report as report_mod
mock_scan_summary_filter.return_value.exists.return_value = True
@@ -1259,12 +1259,12 @@ class TestGenerateComplianceReportsOptimized:
# ``tsc_only`` was exclusive to ThreatScore → evicted before ENS ran.
# ``shared`` is still pending for ENS → must remain.
assert (
"tsc_only" not in observed_state["cache_keys_when_ens_runs"]
), "tsc_only should have been evicted before ENS ran"
assert (
"shared" in observed_state["cache_keys_when_ens_runs"]
), "shared must remain in cache because ENS still needs it"
assert "tsc_only" not in observed_state["cache_keys_when_ens_runs"], (
"tsc_only should have been evicted before ENS ran"
)
assert "shared" in observed_state["cache_keys_when_ens_runs"], (
"shared must remain in cache because ENS still needs it"
)
@patch("tasks.jobs.report.initialize_prowler_provider")
@patch("tasks.jobs.report.rmtree")
@@ -1,6 +1,7 @@
from unittest.mock import Mock, patch
import pytest
from api.models import StatusChoices
from reportlab.platypus import Image, LongTable, Paragraph, Table
from tasks.jobs.reports import FRAMEWORK_REGISTRY, ComplianceData, RequirementData
from tasks.jobs.reports.cis import (
@@ -9,8 +10,6 @@ from tasks.jobs.reports.cis import (
_profile_badge_text,
)
from api.models import StatusChoices
# =============================================================================
# Fixtures
# =============================================================================
@@ -2,6 +2,7 @@ import io
from unittest.mock import Mock
import pytest
from api.models import StatusChoices
from reportlab.platypus import Image, PageBreak, Paragraph, Table
from tasks.jobs.reports import (
FRAMEWORK_REGISTRY,
@@ -10,8 +11,6 @@ from tasks.jobs.reports import (
ThreatScoreReportGenerator,
)
from api.models import StatusChoices
# =============================================================================
# Fixtures
# =============================================================================
+32 -32
View File
@@ -3,11 +3,26 @@ import json
import re
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from datetime import UTC, datetime
from io import StringIO
from unittest.mock import MagicMock, patch
import pytest
from api.db_router import MainRouter
from api.exceptions import ProviderConnectionError
from api.models import (
Finding,
MuteRule,
Provider,
Resource,
ResourceScanSummary,
Scan,
ScanSummary,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.scan import (
_ATTACK_SURFACE_MAPPING_CACHE,
_aggregate_findings_by_region,
@@ -29,22 +44,6 @@ from tasks.jobs.scan import (
)
from tasks.utils import CustomEncoder
from api.db_router import MainRouter
from api.exceptions import ProviderConnectionError
from api.models import (
Finding,
MuteRule,
Provider,
Resource,
ResourceScanSummary,
Scan,
ScanSummary,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
@contextmanager
def noop_rls_transaction(*args, **kwargs):
@@ -1335,9 +1334,9 @@ class TestPerformScan:
)
# Capture time before and after scan
before_scan = datetime.now(timezone.utc)
before_scan = datetime.now(UTC)
perform_prowler_scan(tenant_id, scan_id, provider_id, [])
after_scan = datetime.now(timezone.utc)
after_scan = datetime.now(UTC)
# Verify muted_at is within the scan time window
finding_db = Finding.objects.get(uid=finding_uid)
@@ -1473,7 +1472,7 @@ class TestProcessFindingMicroBatch:
partition="aws-old",
)
previous_first_seen = datetime(2024, 1, 1, tzinfo=timezone.utc)
previous_first_seen = datetime(2024, 1, 1, tzinfo=UTC)
finding = FakeFinding(
uid="finding-muted",
@@ -2123,7 +2122,7 @@ class TestCreateComplianceRequirements:
tags={},
check_id=existing_finding.check_id,
check_metadata={"CheckId": existing_finding.check_id},
first_seen_at=datetime.now(timezone.utc),
first_seen_at=datetime.now(UTC),
muted=False,
)
resource = existing_finding.resources.first()
@@ -2313,7 +2312,7 @@ class TestComplianceRequirementCopy:
def test_persist_compliance_requirement_rows_fallback(
self, mock_copy, mock_rls_transaction, mock_bulk_create
):
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
row = {
"id": uuid.uuid4(),
"tenant_id": str(uuid.uuid4()),
@@ -2394,7 +2393,7 @@ class TestComplianceRequirementCopy:
tenant_id = str(uuid.uuid4())
scan_id = uuid.uuid4()
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
rows = [
{
@@ -2644,10 +2643,10 @@ class TestComplianceRequirementCopy:
# Note: inserted_at is intentionally missing
}
before_call = datetime.now(timezone.utc)
before_call = datetime.now(UTC)
with patch.object(MainRouter, "admin_db", "admin"):
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
after_call = datetime.now(timezone.utc)
after_call = datetime.now(UTC)
csv_rows = list(csv.reader(StringIO(captured["data"])))
assert len(csv_rows) == 1
@@ -2811,7 +2810,7 @@ class TestComplianceRequirementCopy:
{
"id": uuid.uuid4(),
"tenant_id": tenant_id,
"inserted_at": datetime.now(timezone.utc),
"inserted_at": datetime.now(UTC),
"compliance_id": "test",
"framework": "Test",
"version": "1.0",
@@ -2846,7 +2845,7 @@ class TestComplianceRequirementCopy:
row = {
"id": uuid.uuid4(),
"tenant_id": tenant_id,
"inserted_at": datetime.now(timezone.utc),
"inserted_at": datetime.now(UTC),
"compliance_id": "test",
"framework": "Test",
"version": "1.0",
@@ -2886,7 +2885,7 @@ class TestComplianceRequirementCopy:
"""Test ORM fallback with multiple rows."""
tenant_id = str(uuid.uuid4())
scan_id = uuid.uuid4()
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
rows = [
{
@@ -2968,7 +2967,7 @@ class TestComplianceRequirementCopy:
tenant_id = str(uuid.uuid4())
row_id = uuid.uuid4()
scan_id = uuid.uuid4()
inserted_at = datetime.now(timezone.utc)
inserted_at = datetime.now(UTC)
row = {
"id": row_id,
@@ -4461,7 +4460,7 @@ class TestUpdateProviderComplianceScores:
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = datetime.now(timezone.utc)
scan.completed_at = datetime.now(UTC)
scan.save()
connection = MagicMock()
@@ -4538,7 +4537,7 @@ class TestUpdateProviderComplianceScores:
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = datetime.now(timezone.utc)
scan.completed_at = datetime.now(UTC)
scan.save()
connection = MagicMock()
@@ -4591,9 +4590,10 @@ class TestScanIsFullScope:
# If the SDK adds a new filter, this test still passes via the
# introspection-driven derivation; if it adds a non-filter kwarg
# (e.g. provider-like), keep the exclusion list in sync in models.py.
from prowler.lib.scan.scan import Scan as ProwlerScan
import inspect
from prowler.lib.scan.scan import Scan as ProwlerScan
expected = tuple(
name
for name in inspect.signature(ProwlerScan.__init__).parameters
+17 -18
View File
@@ -1,10 +1,18 @@
import uuid
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
import openai
import pytest
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Scan,
StateChoices,
Task,
)
from botocore.exceptions import ClientError
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from django_celery_results.models import TaskResult
@@ -31,15 +39,6 @@ from tasks.tasks import (
security_hub_integration_task,
)
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Scan,
StateChoices,
Task,
)
@pytest.mark.django_db
class TestExtractBedrockCredentials:
@@ -1478,9 +1477,9 @@ class TestCheckIntegrationsTask:
)
# Verify ASFF was NOT created for non-AWS provider
assert (
"asff" not in created_writers
), "ASFF writer should NOT be created for non-AWS providers"
assert "asff" not in created_writers, (
"ASFF writer should NOT be created for non-AWS providers"
)
assert "csv" in created_writers, "CSV writer should be created"
assert "ocsf" in created_writers, "OCSF writer should be created"
@@ -2328,7 +2327,7 @@ class TestPerformScheduledScanTask:
task_id=task_id,
task_name="scan-perform-scheduled",
status="STARTED",
date_created=datetime.now(timezone.utc),
date_created=datetime.now(UTC),
)
Task.objects.create(
id=task_id, task_runner_task=task_result, tenant_id=tenant_id
@@ -2416,7 +2415,7 @@ class TestPerformScheduledScanTask:
state=StateChoices.SCHEDULED,
)
assert scheduled_scans.count() == 1
assert scheduled_scans.first().scheduled_at > datetime.now(timezone.utc)
assert scheduled_scans.first().scheduled_at > datetime.now(UTC)
assert (
Scan.objects.filter(
tenant_id=tenant.id,
@@ -2452,7 +2451,7 @@ class TestPerformScheduledScanTask:
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduled_at=datetime.now(timezone.utc),
scheduled_at=datetime.now(UTC),
scheduler_task_id=periodic_task.id,
)
duplicate_scan = Scan.objects.create(
@@ -2582,7 +2581,7 @@ class TestReaggregateAllFindingGroupSummaries:
scan_id_today_p1 = uuid.uuid4()
scan_id_yesterday_p1 = uuid.uuid4()
scan_id_today_p2 = uuid.uuid4()
today = datetime.now(tz=timezone.utc)
today = datetime.now(tz=UTC)
yesterday = today - timedelta(days=1)
mock_outer_group_result = MagicMock()
@@ -2663,7 +2662,7 @@ class TestReaggregateAllFindingGroupSummaries:
provider_id = uuid.uuid4()
latest_scan_today = uuid.uuid4()
earlier_scan_today = uuid.uuid4()
today_late = datetime.now(tz=timezone.utc)
today_late = datetime.now(tz=UTC)
today_early = today_late - timedelta(hours=4)
mock_outer_group_result = MagicMock()
+2 -2
View File
@@ -1,4 +1,4 @@
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from unittest.mock import patch
import pytest
@@ -29,7 +29,7 @@ class TestGetNextExecutionDatetime:
task_id="abc123",
task_name="scan-perform-scheduled",
status="SUCCESS",
date_created=datetime.now(timezone.utc) - timedelta(hours=1),
date_created=datetime.now(UTC) - timedelta(hours=1),
result="Success",
)
return task_result