Merge remote-tracking branch 'origin/master' into mintlify/55c1f1a5

This commit is contained in:
mintlify[bot]
2026-09-28 07:07:36 +00:00
committed by GitHub
249 changed files with 11500 additions and 1497 deletions
+10 -1
View File
@@ -110,11 +110,20 @@ DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY=""
DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN=""
# The AWS region where your S3 bucket is located (e.g., "us-east-1")
# Required if the bucket uses SSE-KMS: download URLs are then signed with SigV4, which
# is scoped to this region, so it must match the bucket's
DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION=""
# The name of the S3 bucket where scan output should be stored
DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET=""
# The storage endpoint the API and Celery workers use to upload and list scan output
# (e.g. "http://minio:9000"). Leave empty on AWS S3. Set it when scan output is stored on
# S3-compatible object storage such as MinIO instead of real S3.
# If set without DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL below, report download URLs are
# signed against this internal host, and a browser outside the container network cannot open them.
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL=""
# The storage address the browser can reach, used only to sign report download URLs
# (e.g. "https://storage.example.com"). Leave empty on AWS S3. Set it when storage is
# only reachable inside the container network, such as MinIO on "http://minio:9000".
@@ -165,7 +174,7 @@ SENTRY_RELEASE=local
# REO_DEV_CLIENT_ID=
#### Prowler release version ####
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.43.0
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.44.0
# Social login credentials
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
+3 -2
View File
@@ -11,6 +11,7 @@ on:
- "v5.*"
paths:
- ".github/workflows/ui-e2e-tests-v2.yml"
- ".github/workflows/test-impact-analysis.yml"
- ".github/test-impact.yml"
- "ui/**"
- "api/**" # API changes can affect UI E2E
@@ -237,8 +238,8 @@ jobs:
- name: Add AWS credentials for testing
run: |
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
echo "AWS_ACCESS_KEY_ID=${E2E_AWS_PROVIDER_ACCESS_KEY}" >> .env
echo "AWS_SECRET_ACCESS_KEY=${E2E_AWS_PROVIDER_SECRET_KEY}" >> .env
- name: Build API image from current code
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
+11
View File
@@ -115,3 +115,14 @@ ignore:
- vulnerability: CVE-2026-9669
package:
name: python
# CVE-2026-82049 (tarfile data/tar filter bypass via a hard link to a symlink) has no
# fixed CPython release on any branch: the fix is merged on main and 3.13 only, and the
# 3.12 backport is still open. Grype records 3.14.0b1 as the fix, so only-fixed does not
# drop it, yet python:3.12.14-slim-trixie reports it too. Prowler never extracts tar
# archives to disk: the ECR image inspection reads members in memory with extractfile().
# Remove once the base image ships a 3.12 release that includes the backport.
# https://github.com/python/cpython/issues/157190
# https://github.com/python/cpython/pull/157454
- vulnerability: CVE-2026-82049
package:
name: python
+16
View File
@@ -4,6 +4,22 @@ All notable changes to the **Prowler API** are documented in this file.
<!-- changelog: release notes start -->
## [1.44.0] (Prowler v5.43.0)
### 🐞 Fixed
- Report download URLs can be signed against a browser-reachable storage host via `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL`, so downloads complete on deployments where storage is only reachable inside the container network [(#12552)](https://github.com/prowler-cloud/prowler/pull/12552)
- A scan report download no longer fails with a server error when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is unset, which is common on storage with no meaningful region [(#12552)](https://github.com/prowler-cloud/prowler/pull/12552)
- Lapsed pending invitations are reported as expired and no longer block a new invitation for the same email [(#12831)](https://github.com/prowler-cloud/prowler/pull/12831)
### 🔐 Security
- `libsqlite3-0`, `gzip`, `perl-base` and `libpcre2-8-0` upgraded in the API container image, patching high Debian CVEs [(#12804)](https://github.com/prowler-cloud/prowler/pull/12804)
- PowerShell from 7.5.9 to 7.5.11 in the API container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901 [(#12811)](https://github.com/prowler-cloud/prowler/pull/12811)
- Bumped `anyio` to 4.14.2 to resolve CVE-2026-63374 [(#12848)](https://github.com/prowler-cloud/prowler/pull/12848)
---
## [1.43.0] (Prowler v5.42.0)
### 🔄 Changed
@@ -1 +0,0 @@
`libsqlite3-0`, `gzip`, `perl-base` and `libpcre2-8-0` upgraded in the API container image, patching high Debian CVEs
@@ -1 +0,0 @@
PowerShell from 7.5.9 to 7.5.11 in the API container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901
@@ -0,0 +1 @@
Adds a periodic sweep that drops orphaned Attack Paths temp Neo4j scan databases left behind when a worker or Neo4j crashes mid-scan, before they accumulate unbounded
@@ -0,0 +1 @@
Resources no longer keep a stale failed findings count forever when a scoped or imported scan for the same provider completes after a full scan, which used to make the full scan skip its own cleanup
@@ -1 +0,0 @@
Lapsed pending invitations are reported as expired and no longer block a new invitation for the same email
@@ -0,0 +1 @@
Providers whose most recent completed scan has no `completed_at` timestamp are no longer missing from every endpoint that reports a provider's latest scan, which now falls back to scan creation order instead of skipping the provider
@@ -0,0 +1 @@
Unify how every endpoint resolves a provider latest completed scan, so overlapping scans no longer make findings, compliance and mute rules read from different scans
@@ -1 +0,0 @@
Report download URLs can be signed against a browser-reachable storage host via `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL`, so downloads complete on deployments where storage is only reachable inside the container network
@@ -1 +0,0 @@
A scan report download no longer fails with a server error when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is unset, which is common on storage with no meaningful region
@@ -0,0 +1 @@
Scan output uploads and downloads can now target S3-compatible object storage such as MinIO directly via `DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL`, instead of relying on process-wide AWS environment variables that also hijacked unrelated AWS API calls
@@ -0,0 +1 @@
Scan report downloads from an S3 bucket with default SSE-KMS encryption no longer fail with an `InvalidArgument` error: when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is set, presigned download URLs are signed with AWS Signature Version 4 for that region
@@ -0,0 +1 @@
`POST /api/v1/scans` again returns the new scan id in the response `task_args`, which had been empty since the scan broker publish moved to transaction commit
@@ -0,0 +1 @@
Celery loggers are now declared explicitly in `custom_logging.py` so fatal worker errors are no longer silenced by `disable_existing_loggers=True`. All long-running services in `docker-compose.yml` now have `restart: unless-stopped` so containers recover automatically after unexpected crashes.
+2 -2
View File
@@ -71,7 +71,7 @@ name = "prowler-api"
package-mode = false
# Needed for the SDK compatibility
requires-python = ">=3.11,<3.13"
version = "1.44.0"
version = "1.45.0"
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
# target-version tracks this project's lowest supported Python.
@@ -137,7 +137,7 @@ constraint-dependencies = [
"aliyun-log-fastpb==0.2.0",
"amqp==5.3.1",
"annotated-types==0.7.0",
"anyio==4.12.1",
"anyio==4.14.2",
"applicationinsights==0.11.10",
"apscheduler==3.11.2",
"argcomplete==3.5.3",
@@ -207,6 +207,11 @@ def drop_database(database: str) -> None:
sink_module.get_backend().drop_database(database)
def list_databases() -> list[str]:
"""List database names on the ingest cluster. Temp scan DBs always live here."""
return ingest.list_databases()
def drop_subgraph(database: str, provider_id: str) -> int:
return sink_module.get_backend().drop_subgraph(database, provider_id)
@@ -13,6 +13,7 @@ from api.attack_paths.ingest.driver import (
get_session,
get_uri,
init_driver,
list_databases,
run_cypher,
)
@@ -25,5 +26,6 @@ __all__ = [
"get_session",
"get_uri",
"init_driver",
"list_databases",
"run_cypher",
]
@@ -165,6 +165,14 @@ def drop_database(database: str) -> None:
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
def list_databases() -> list[str]:
"""List every database name on the Neo4j temp-database cluster."""
# A cluster returns one row per hosting server, so dedupe on name
with get_session() as session:
result = session.run("SHOW DATABASES YIELD name RETURN DISTINCT name")
return [record["name"] for record in result]
def clear_cache(database: str) -> None:
"""Best-effort cache clear for a Neo4j database."""
from api.attack_paths.database import GraphDatabaseQueryException
+1 -1
View File
@@ -409,7 +409,7 @@ def batch_delete(tenant_id, queryset, batch_size=settings.DJANGO_DELETION_BATCH_
Args:
tenant_id (str): Tenant ID the queryset belongs to.
queryset (QuerySet): The queryset of objects to delete.
queryset: The queryset of objects to delete.
batch_size (int): The number of objects to delete in each batch.
Returns:
@@ -0,0 +1,48 @@
from django.db import migrations
TASK_NAME = "attack-paths-reap-orphaned-tmp-databases"
INTERVAL_HOURS = 6
def create_periodic_task(apps, schema_editor):
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
schedule, _ = IntervalSchedule.objects.get_or_create(
every=INTERVAL_HOURS,
period="hours",
)
PeriodicTask.objects.update_or_create(
name=TASK_NAME,
defaults={
"task": TASK_NAME,
"interval": schedule,
"enabled": True,
},
)
def delete_periodic_task(apps, schema_editor):
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
PeriodicTask.objects.filter(name=TASK_NAME).delete()
# Clean up the schedule if no other task references it
IntervalSchedule.objects.filter(
every=INTERVAL_HOURS,
period="hours",
periodictask__isnull=True,
).delete()
class Migration(migrations.Migration):
dependencies = [
("api", "0099_delete_tenant_onboarding_profile"),
("django_celery_beat", "0019_alter_periodictasks_options"),
]
operations = [
migrations.RunPython(create_periodic_task, delete_periodic_task),
]
+65 -2
View File
@@ -617,9 +617,66 @@ class Task(RowLevelSecurityProtectedModel):
resource_name = "tasks"
class ScanQuerySet(models.QuerySet):
"""Shared selectors for "the latest scan of a provider".
The queryset must already be scoped by the caller: manager, tenant, RBAC,
providers and database alias.
"""
# How "which completed scan is the provider's current one" is ordered.
LATEST_ORDER_BY = (
models.F("completed_at").desc(nulls_last=True),
models.F("inserted_at").desc(),
models.F("id").desc(),
)
def _eligible_for_latest(self) -> "ScanQuerySet":
"""Restrict to the scans that may be a provider's latest.
Returns:
ScanQuerySet: The completed scans.
"""
return self.filter(state=StateChoices.COMPLETED)
def latest_per_provider(self) -> "ScanQuerySet":
"""Pick each provider's latest scan with `DISTINCT ON (provider_id)`.
Returns:
ScanQuerySet: One scan per provider, the latest one.
"""
return (
self._eligible_for_latest()
.order_by("provider_id", *self.LATEST_ORDER_BY)
.distinct("provider_id")
)
def latest_ids_per_provider(self) -> list[UUID]:
"""Evaluate `latest_per_provider` and return the scan ids.
The ids are materialised so callers can pass them as a literal `IN`
list; as a subquery Postgres misestimates the row count and picks a
slow nested loop.
Returns:
list[UUID]: The id of each provider's latest scan.
"""
return list(self.latest_per_provider().values_list("id", flat=True))
def latest_first(self) -> "ScanQuerySet":
"""Order eligible scans newest first, without deduplicating per provider.
Expects the queryset to be already filtered to a single provider.
Returns:
ScanQuerySet: The eligible scans, latest first.
"""
return self._eligible_for_latest().order_by(*self.LATEST_ORDER_BY)
class Scan(RowLevelSecurityProtectedModel):
objects = ActiveProviderManager()
all_objects = models.Manager()
objects = ActiveProviderManager.from_queryset(ScanQuerySet)()
all_objects = ScanQuerySet.as_manager()
_SCOPING_SCANNER_ARG_KEYS_CACHE: tuple[str, ...] | None = None
@@ -726,6 +783,12 @@ class Scan(RowLevelSecurityProtectedModel):
name="scans_prov_state_ins_desc_idx",
),
# TODO This might replace `scans_prov_state_ins_desc_idx` completely. Review usage
# Since `ScanQuerySet`, no code path reads a provider's
# completed scans by `-inserted_at`. The only query left that
# matches this index (and `scans_prov_state_ins_desc_idx` above)
# is `GET /scans?filter[provider]=…&filter[state]=completed` with
# the default sort. Both are candidates to drop in a follow-up
# once production `pg_stat_user_indexes.idx_scan` confirms it.
models.Index(
fields=["tenant_id", "provider_id", "-inserted_at"],
condition=Q(state=StateChoices.COMPLETED),
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: 3.0.3
info:
title: Prowler API
version: 1.44.0
version: 1.45.0
description: |-
Prowler API specification.
@@ -187,6 +187,27 @@ class TestRoutingByDatabasePrefix:
sink_backend_stub.drop_database.assert_called_once_with("db-tenant-abc")
mock_ingest.drop_database.assert_not_called()
def test_list_databases_always_routes_to_ingest(self, sink_backend_stub):
with patch("api.attack_paths.database.ingest") as mock_ingest:
mock_ingest.list_databases.return_value = ["db-tmp-scan-uuid-1"]
assert db_module.list_databases() == ["db-tmp-scan-uuid-1"]
mock_ingest.list_databases.assert_called_once_with()
def test_ingest_list_databases_dedupes_cluster_rows(self):
from api.attack_paths.ingest import driver as ingest_driver
with patch.object(ingest_driver, "get_session") as mock_get_session:
session = mock_get_session.return_value.__enter__.return_value
session.run.return_value = [{"name": "db-tmp-scan-uuid-1"}]
assert ingest_driver.list_databases() == ["db-tmp-scan-uuid-1"]
session.run.assert_called_once_with(
"SHOW DATABASES YIELD name RETURN DISTINCT name"
)
def test_clear_cache_routes_temp_to_ingest(self, sink_backend_stub):
with patch("api.attack_paths.database.ingest") as mock_ingest:
db_module.clear_cache("db-tmp-scan-uuid-1")
+226 -1
View File
@@ -1,14 +1,16 @@
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
import pytest
from allauth.socialaccount.models import SocialApp
from api.db_router import MainRouter
from api.models import (
Provider,
ProviderComplianceScore,
Resource,
ResourceTag,
SAMLConfiguration,
SAMLDomainIndex,
Scan,
StateChoices,
StatusChoices,
TenantComplianceSummary,
@@ -524,3 +526,226 @@ class TestTenantComplianceSummaryModel:
assert summary1.id != summary2.id
assert summary1.requirements_passed != summary2.requirements_passed
def _latest_scan_fixture(tenant, provider, *, completed_at, inserted_at=None, **kwargs):
scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
trigger=kwargs.pop("trigger", Scan.TriggerChoices.MANUAL),
state=kwargs.pop("state", StateChoices.COMPLETED),
completed_at=completed_at,
**kwargs,
)
if inserted_at is not None:
# `inserted_at` is auto_now_add, so it has to be forced after the fact.
Scan.all_objects.filter(pk=scan.pk).update(inserted_at=inserted_at)
scan.refresh_from_db()
return scan
@pytest.mark.django_db
class TestScanQuerySetOrdering:
def test_completed_later_wins_over_inserted_later(
self, tenants_fixture, aws_provider
):
"""The scan that FINISHED last is current, not the one that started last."""
tenant, *_ = tenants_fixture
now = datetime.now(UTC)
finished_last = _latest_scan_fixture(
tenant,
aws_provider,
inserted_at=now - timedelta(hours=3),
completed_at=now,
)
_latest_scan_fixture(
tenant,
aws_provider,
inserted_at=now - timedelta(hours=1),
completed_at=now - timedelta(hours=1),
)
assert Scan.all_objects.filter(
tenant_id=tenant.id
).latest_ids_per_provider() == [finished_last.id]
def test_null_completed_at_provider_is_still_returned(
self, tenants_fixture, aws_provider
):
"""NULLS LAST, not `completed_at__isnull=False`.
Excluding NULL `completed_at` would drop the provider from every
"latest" endpoint instead of falling back to `inserted_at`.
"""
tenant, *_ = tenants_fixture
only_scan = _latest_scan_fixture(tenant, aws_provider, completed_at=None)
assert Scan.all_objects.filter(
tenant_id=tenant.id
).latest_ids_per_provider() == [only_scan.id]
def test_null_completed_at_never_outranks_a_finished_scan(
self, tenants_fixture, aws_provider
):
"""Postgres sorts NULLs first under DESC; NULLS LAST is what fixes it."""
tenant, *_ = tenants_fixture
now = datetime.now(UTC)
finished = _latest_scan_fixture(
tenant,
aws_provider,
inserted_at=now - timedelta(hours=2),
completed_at=now - timedelta(hours=2),
)
_latest_scan_fixture(
tenant,
aws_provider,
inserted_at=now,
completed_at=None,
)
assert Scan.all_objects.filter(
tenant_id=tenant.id
).latest_ids_per_provider() == [finished.id]
def test_id_breaks_an_exact_timestamp_tie_deterministically(
self, tenants_fixture, aws_provider
):
tenant, *_ = tenants_fixture
now = datetime.now(UTC)
scans = [
_latest_scan_fixture(
tenant, aws_provider, inserted_at=now, completed_at=now
)
for _ in range(3)
]
expected = max(scan.id for scan in scans)
picks = {
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider()[0]
for _ in range(5)
}
assert picks == {expected}
@pytest.mark.django_db
class TestScanQuerySetEligibility:
def test_unfinished_scans_are_excluded(self, tenants_fixture, aws_provider):
tenant, *_ = tenants_fixture
_latest_scan_fixture(
tenant,
aws_provider,
completed_at=None,
state=StateChoices.EXECUTING,
)
assert (
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
)
@pytest.mark.django_db
class TestScanQuerySetManagerChoice:
def test_active_manager_hides_soft_deleted_providers(
self, tenants_fixture, aws_provider
):
"""`Scan.objects` drops soft-deleted providers, `all_objects` keeps them.
The queryset must not decide this for the caller.
"""
tenant, *_ = tenants_fixture
scan = _latest_scan_fixture(
tenant, aws_provider, completed_at=datetime.now(UTC)
)
Provider.all_objects.filter(pk=aws_provider.pk).update(is_deleted=True)
assert Scan.all_objects.filter(
tenant_id=tenant.id
).latest_ids_per_provider() == [scan.id]
assert Scan.objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
@pytest.mark.django_db
class TestScanQuerySetPerProviderScoping:
def test_one_scan_per_provider(self, tenants_fixture, aws_provider_pair):
tenant, *_ = tenants_fixture
provider_one, provider_two = aws_provider_pair
now = datetime.now(UTC)
newest_one = _latest_scan_fixture(tenant, provider_one, completed_at=now)
_latest_scan_fixture(tenant, provider_one, completed_at=now - timedelta(days=1))
newest_two = _latest_scan_fixture(tenant, provider_two, completed_at=now)
assert set(
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider()
) == {newest_one.id, newest_two.id}
def test_caller_filters_are_preserved(self, tenants_fixture, aws_provider_pair):
tenant, *_ = tenants_fixture
provider_one, provider_two = aws_provider_pair
now = datetime.now(UTC)
scan_one = _latest_scan_fixture(tenant, provider_one, completed_at=now)
_latest_scan_fixture(tenant, provider_two, completed_at=now)
assert Scan.all_objects.filter(
tenant_id=tenant.id, provider__in=[provider_one]
).latest_ids_per_provider() == [scan_one.id]
def test_latest_first_is_ordered_not_deduplicated(
self, tenants_fixture, aws_provider
):
tenant, *_ = tenants_fixture
now = datetime.now(UTC)
newest = _latest_scan_fixture(tenant, aws_provider, completed_at=now)
older = _latest_scan_fixture(
tenant, aws_provider, completed_at=now - timedelta(days=1)
)
ordered = list(
Scan.all_objects.filter(
tenant_id=tenant.id, provider_id=aws_provider.id
).latest_first()
)
assert [scan.id for scan in ordered] == [newest.id, older.id]
def test_tenant_isolation(self, tenants_fixture, aws_provider):
tenant, other_tenant, *_ = tenants_fixture
_latest_scan_fixture(tenant, aws_provider, completed_at=datetime.now(UTC))
assert (
Scan.all_objects.filter(tenant_id=other_tenant.id).latest_ids_per_provider()
== []
)
def test_empty_queryset_returns_empty_list(self, tenants_fixture):
tenant, *_ = tenants_fixture
assert (
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
)
@pytest.mark.django_db
class TestScanQuerySetPerProviderQuerysetShape:
def test_returns_a_queryset_not_a_list(self, tenants_fixture, aws_provider):
tenant, *_ = tenants_fixture
_latest_scan_fixture(tenant, aws_provider, completed_at=datetime.now(UTC))
qs = Scan.all_objects.filter(tenant_id=tenant.id).latest_per_provider()
# Callers chain .values(...) / .values_list(...) onto this.
assert qs.values_list("provider_id", flat=True).count() == 1
@pytest.mark.django_db
class TestScanQuerySetRelatedManager:
def test_reverse_relation_exposes_the_methods(self, tenants_fixture, aws_provider):
tenant, *_ = tenants_fixture
now = datetime.now(UTC)
newest = _latest_scan_fixture(tenant, aws_provider, completed_at=now)
_latest_scan_fixture(tenant, aws_provider, completed_at=now - timedelta(days=1))
assert aws_provider.scans.latest_first().first().id == newest.id
+37
View File
@@ -3951,6 +3951,43 @@ class TestScanViewSet:
mock_enqueue_scan_execution.assert_called_once()
# assert scan.scanner_args == expected_scanner_args
@patch("api.v1.views.enqueue_scan_execution_on_commit")
def test_scans_create_returns_the_scan_id_in_task_args(
self,
mock_enqueue_scan_execution,
authenticated_client,
okta_provider,
):
"""The 202 is a task, so `task_args` is the only place the scan id is.
It is serialized before the on_commit publish that would otherwise fill
the kwargs, so the record has to carry them from the start.
"""
payload = {
"data": {
"type": "scans",
"attributes": {"name": "New Scan"},
"relationships": {
"provider": {
"data": {"type": "providers", "id": str(okta_provider.id)}
}
},
}
}
response = authenticated_client.post(
reverse("scan-list"),
data=payload,
content_type=API_JSON_CONTENT_TYPE,
)
assert response.status_code == status.HTTP_202_ACCEPTED
scan = Scan.objects.get()
assert response.json()["data"]["attributes"]["task_args"] == {
"scan_id": str(scan.id),
"provider_id": str(okta_provider.id),
}
@patch("tasks.tasks.perform_scan_task.apply_async")
def test_scans_create_queues_scan_when_provider_has_active_scan(
self,
+44 -73
View File
@@ -2822,6 +2822,15 @@ class ScanViewSet(ProviderVisibilityMixin, BaseRLSViewSet):
tenant_id=self.request.tenant_id,
task_id=pre_task_id,
task_status=(QUEUED_SCAN_TASK_STATE if active_scan else None),
# This response is serialized before the on_commit publish,
# so without these the caller gets a task id and no scan id.
# Kept in step with what `enqueue_scan_execution_on_commit`
# publishes below.
task_kwargs={
"tenant_id": str(self.request.tenant_id),
"scan_id": str(scan.id),
"provider_id": str(scan.provider_id),
},
)
if not active_scan:
@@ -3478,12 +3487,8 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
filtered_queryset = self.filter_queryset(self.get_queryset())
latest_scans = (
Scan.all_objects.filter(
tenant_id=tenant_id,
state=StateChoices.COMPLETED,
)
.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
Scan.all_objects.filter(tenant_id=tenant_id)
.latest_per_provider()
.values("provider_id")
)
@@ -3615,11 +3620,9 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
tenant_id = request.tenant_id
query_params = request.query_params
latest_scans_queryset = (
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
)
latest_scans_queryset = Scan.all_objects.filter(
tenant_id=tenant_id
).latest_per_provider()
queryset = ResourceScanSummary.objects.filter(
tenant_id=tenant_id,
@@ -4206,12 +4209,9 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
tenant_id = request.tenant_id
filtered_queryset = self.filter_queryset(self.get_queryset())
latest_scan_ids = list(
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
latest_scan_ids = Scan.all_objects.filter(
tenant_id=tenant_id
).latest_ids_per_provider()
filtered_queryset = filtered_queryset.filter(
tenant_id=tenant_id, scan_id__in=latest_scan_ids
)
@@ -4234,11 +4234,9 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
tenant_id = request.tenant_id
query_params = request.query_params
latest_scans_queryset = (
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
)
latest_scans_queryset = Scan.all_objects.filter(
tenant_id=tenant_id
).latest_per_provider()
raw_latest_scans_ids = list(
latest_scans_queryset.values_list("id", "unique_resource_count")
)
@@ -4979,11 +4977,7 @@ class ComplianceOverviewViewSet(
if provider_filters:
scans = scans.filter(**provider_filters)
return list(
scans.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
return scans.latest_ids_per_provider()
def _filtered_queryset_for_latest_provider_scans(self, latest_scan_ids=None):
if latest_scan_ids is None:
@@ -5725,14 +5719,9 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
else {}
)
latest_scan_ids = (
Scan.all_objects.filter(
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
)
.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
latest_scan_ids = Scan.all_objects.filter(
tenant_id=tenant_id, **provider_filter
).latest_ids_per_provider()
return filtered_queryset.filter(
tenant_id=tenant_id, scan_id__in=latest_scan_ids
@@ -5759,16 +5748,10 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None):
provider_filter = self._get_provider_filter()
queryset = Scan.all_objects.filter(
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
)
queryset = Scan.all_objects.filter(tenant_id=tenant_id, **provider_filter)
if provider_filters:
queryset = queryset.filter(**provider_filters)
return (
queryset.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
return queryset.latest_ids_per_provider()
@action(detail=False, methods=["get"], url_name="providers")
def providers(self, request):
@@ -5780,14 +5763,9 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
else {}
)
latest_scan_ids = (
Scan.all_objects.filter(
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
)
.order_by("provider_id", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
latest_scan_ids = Scan.all_objects.filter(
tenant_id=tenant_id, **provider_filter
).latest_ids_per_provider()
findings_aggregated = (
queryset.filter(scan_id__in=latest_scan_ids)
@@ -7934,18 +7912,13 @@ class FindingGroupViewSet(JsonApiFilterMixin, BaseRLSViewSet):
def _get_latest_findings_per_provider(self, filtered_queryset):
"""Keep only findings from each provider's most recent completed scan."""
# Materialize to a literal IN list. Left as a subquery, Postgres can't
# estimate the match count and picks a serial nested loop on
# resource_finding_mappings when one scan dominates findings
latest_scan_ids = list(
Scan.objects.filter(
tenant_id=self.request.tenant_id,
state=StateChoices.COMPLETED,
)
.order_by("provider_id", "-completed_at", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
# `latest_ids_per_provider` materializes to a literal IN list.
# Left as a subquery, Postgres can't estimate the match count and picks
# a serial nested loop on resource_finding_mappings when one scan
# dominates findings
latest_scan_ids = Scan.objects.filter(
tenant_id=self.request.tenant_id
).latest_ids_per_provider()
return filtered_queryset.filter(scan_id__in=latest_scan_ids)
def _post_process_aggregation(self, aggregated_data):
@@ -8891,16 +8864,14 @@ class FindingGroupViewSet(JsonApiFilterMixin, BaseRLSViewSet):
tenant_id = request.tenant_id
queryset = self._get_finding_queryset()
# Order by -completed_at (matching the /latest summary path and the
# daily summary upsert keyed on midnight(completed_at)) so that
# overlapping scans do not make /resources and /latest read from
# different scans and report diverging counts.
latest_scan_ids = (
Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
.order_by("provider_id", "-completed_at", "-inserted_at")
.distinct("provider_id")
.values_list("id", flat=True)
)
# The shared selector orders by -completed_at (matching the /latest
# summary path and the daily summary upsert keyed on
# midnight(completed_at)) so that overlapping scans do not make
# /resources and /latest read from different scans and report
# diverging counts.
latest_scan_ids = Scan.objects.filter(
tenant_id=tenant_id
).latest_ids_per_provider()
normalized_params = self._normalize_jsonapi_params(request.query_params)
# Remove date filters since we're using latest
+56
View File
@@ -231,6 +231,62 @@ LOGGING = {
"level": LEVEL,
"propagate": False,
},
# Celery loggers must be declared explicitly because
# disable_existing_loggers=True silences any logger that exists at
# dictConfig time but is not named here. Without these, fatal worker
# errors (e.g. celery.worker CRITICAL) produce no output.
# "celery" must keep propagating: get_task_logger() parents task
# loggers under celery.task, so blocking here hides them from root.
"celery": {
"level": LEVEL,
"propagate": True,
},
"celery.worker": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
"celery.worker.consumer": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
"celery.worker.consumer.consumer": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
"kombu": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
"kombu.transport.redis": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
"billiard": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
"amqp": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
# WARNING keeps task failures but skips one "succeeded" line per task.
"celery.app.trace": {
"handlers": ["tasks_console"],
"level": "WARNING",
"propagate": False,
},
"celery.beat": {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
},
# Gunicorn required configuration
"root": {
+15
View File
@@ -7,6 +7,7 @@ from config.settings.eventstream import * # noqa
from config.settings.partitions import * # noqa
from config.settings.sentry import * # noqa
from config.settings.social_login import * # noqa
from django.core.exceptions import ImproperlyConfigured
SECRET_KEY = env("SECRET_KEY", default="secret")
DEBUG = env.bool("DJANGO_DEBUG", default=False)
@@ -295,6 +296,9 @@ DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY = env.str(
)
DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN = env.str("DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN", "")
DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION = env.str("DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION", "")
# Storage endpoint the API and Celery workers use to talk to S3-compatible object storage
# such as MinIO. Empty means the real AWS S3 endpoint, which is unaffected.
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL = env.str("DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL", "")
# Browser-reachable storage host used to sign download URLs. Empty means sign against the
# same endpoint the API talks to, which is what Prowler Cloud on S3 does.
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL = env.str(
@@ -325,6 +329,17 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 960
) # 16h
# Minimum age (of the scan row, or of the scan id itself when the row is gone) before
# the periodic reaper will drop an orphaned temp Neo4j database. Keeps a scan that is
# still legitimately in flight from ever losing its staging database mid-run.
ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS = env.int(
"ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS", 6
)
if ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS <= 0:
raise ImproperlyConfigured(
"ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS must be a positive number of hours"
)
# Selects where the persistent attack-paths graph is stored. The scan
# temporary database is always Neo4j; only the sink is configurable.
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
@@ -0,0 +1,107 @@
"""Periodic reaper for orphaned temp Neo4j scan databases.
`scan.py` creates a throw-away `db-tmp-scan-<attack_paths_scan_id>` database per
scan and drops it once the scan finishes, success or failure. When the worker
or Neo4j itself dies mid-scan, that drop never runs and nothing else ever
revisits the database - it sits there forever. This sweep lists every temp
database on the ingest cluster and drops the ones whose scan is gone or has
been finished for longer than the configured safety margin.
"""
from datetime import UTC, datetime, timedelta
from api.attack_paths import database as graph_database
from api.db_router import MainRouter
from api.models import AttackPathsScan, StateChoices
from api.uuid_utils import datetime_from_uuid7
from celery.utils.log import get_task_logger
from config.django.base import ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS
from uuid6 import UUID as UUID7
logger = get_task_logger(__name__)
TERMINAL_STATES = (
StateChoices.COMPLETED,
StateChoices.FAILED,
StateChoices.CANCELLED,
)
def reap_orphaned_tmp_databases() -> dict:
"""Drop temp Neo4j scan databases whose scan is gone or long finished.
A failure listing databases aborts the whole sweep (nothing to iterate).
A failure reaping one database is logged and skipped so the rest of the
sweep still runs.
"""
now = datetime.now(tz=UTC)
safety_margin = timedelta(hours=ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS)
try:
databases = graph_database.list_databases()
except Exception:
logger.exception("Failed to list ingest Neo4j databases for temp-db reap")
return {"dropped_count": 0, "databases": []}
tmp_databases = [
name for name in databases if name.startswith(graph_database.TEMP_DB_PREFIX)
]
dropped: list[str] = []
for database in tmp_databases:
try:
if _is_orphaned(database, now, safety_margin):
graph_database.drop_database(database)
dropped.append(database)
logger.info(f"Dropped orphaned temp Neo4j database `{database}`")
except Exception:
logger.exception(f"Failed to reap temp Neo4j database `{database}`")
logger.info(f"Temp Neo4j database reap: {len(dropped)} dropped")
return {"dropped_count": len(dropped), "databases": dropped}
def _is_orphaned(database: str, now: datetime, safety_margin: timedelta) -> bool:
"""Decide whether a temp database is safe to drop.
No scan row: the row was hard-deleted (tenant/provider cleanup) or was
never created. Falls back to the scan id's own UUIDv7 timestamp so a
database created moments ago is never touched even without a row to check.
Scan row present: only reapable once it reached a terminal state and has
been finished for longer than the safety margin, so a scan still
legitimately executing is never touched.
"""
scan_id = database[len(graph_database.TEMP_DB_PREFIX) :]
try:
scan_uuid = UUID7(scan_id)
except ValueError:
logger.warning(
f"Temp database `{database}` has an unparseable scan id, skipping"
)
return False
# Global sweep with no tenant context: admin_db bypasses RLS on purpose, the same
# way cleanup_stale_attack_paths_scans finds stale scans across every tenant.
scan = (
AttackPathsScan.all_objects.using(MainRouter.admin_db)
.filter(id=scan_uuid)
.first()
)
if scan is None:
if scan_uuid.version != 7:
logger.warning(
f"Temp database `{database}` has no scan row and a non-UUIDv7 id, "
"skipping"
)
return False
return now - datetime_from_uuid7(scan_uuid) >= safety_margin
if scan.state not in TERMINAL_STATES:
return False
# `mark_scan_finished` does not touch `updated_at`, so prefer `completed_at`
finished_at = scan.completed_at or scan.updated_at
return now - finished_at >= safety_margin
+6 -3
View File
@@ -499,10 +499,13 @@ def backfill_provider_compliance_scores(tenant_id: str) -> dict:
provider_id__in=existing_providers
)
# `completed_scans` keeps its own `completed_at__isnull=False`: this
# task writes a *dated* ProviderComplianceScore row, so unlike the read
# paths it genuinely cannot use a scan without a `completed_at`.
scan_info = list(
completed_scans.order_by("provider_id", "-completed_at")
.distinct("provider_id")
.values("id", "provider_id", "completed_at")
completed_scans.latest_per_provider().values(
"id", "provider_id", "completed_at"
)
)
if not scan_info:
+28 -10
View File
@@ -207,15 +207,21 @@ def get_s3_client():
This function attempts to initialize an S3 client by reading the AWS access key, secret key,
session token, and region from environment variables. It then validates the client by listing
available S3 buckets. If an error occurs during this process (for example, due to missing or
invalid credentials), it falls back to creating an S3 client without explicitly provided credentials,
which may rely on other configuration sources (e.g., IAM roles).
invalid credentials), it falls back to creating an S3 client without explicitly provided
credentials, which may rely on other configuration sources (e.g., IAM roles).
That fallback is only safe when no explicit endpoint is configured: with an endpoint set, the
explicit client already targets the intended S3-compatible storage, and the fallback client
would go to the AWS default provider chain instead, an unrelated real-AWS account reachable
from the host. So when an endpoint is configured, the original error propagates instead.
Returns:
boto3.client: A configured S3 client instance.
Raises:
ClientError, NoCredentialsError, or ParamValidationError if both attempts to create a client fail.
ClientError, NoCredentialsError, or ParamValidationError if the client cannot be created.
"""
endpoint = settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL
s3_client = None
try:
s3_client = boto3.client(
@@ -226,9 +232,12 @@ def get_s3_client():
# Storage that has no meaningful region, MinIO among it, is usually configured
# without one, and botocore rejects an empty region before any request is made.
region_name=settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION or "us-east-1",
endpoint_url=endpoint or None,
)
s3_client.list_buckets()
except (ClientError, NoCredentialsError, ParamValidationError, ValueError):
if endpoint:
raise
s3_client = boto3.client("s3")
s3_client.list_buckets()
@@ -236,13 +245,21 @@ def get_s3_client():
def get_s3_presign_client():
"""Return a client that signs URLs against the public storage host.
"""Return a client that signs download URLs with SigV4.
None means no public host is configured and the caller should presign with its own
client, which leaves deployments on real S3 with the URL they get today.
It is used when a public or internal storage host is configured, or when the bucket's
region is: boto3 otherwise presigns S3 URLs with SigV2, which S3 rejects for SSE-KMS
objects. None means none of those is set and the caller should presign with its own
client, which leaves those deployments with the URL they get today.
The public endpoint wins when both are set: the internal endpoint may only be reachable
from inside the cluster, and a URL signed against it would not open in a browser.
"""
public_endpoint = settings.DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL
if not public_endpoint:
endpoint = (
settings.DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL
or settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL
)
if not endpoint and not settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION:
return None
# Blank keys are signed as-is (empty credential scope) instead of deferring to the
@@ -266,9 +283,10 @@ def get_s3_presign_client():
# SigV4 puts the region in the credential scope, and MinIO answers to us-east-1
# unless it was told otherwise, so an empty region would sign an unusable URL.
region_name=settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION or "us-east-1",
endpoint_url=public_endpoint,
endpoint_url=endpoint or None,
# The signature covers the host, so the addressing style has to be pinned rather
# than guessed from the endpoint: MinIO serves path-style.
# than guessed from the endpoint: MinIO serves path-style, and on AWS it keeps the
# regional host instead of the global one, which redirects for new buckets.
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
+4 -12
View File
@@ -1,7 +1,7 @@
from collections.abc import Iterable
from api.db_utils import rls_transaction
from api.models import Finding, MuteRule, Scan, StateChoices
from api.models import Finding, MuteRule, Scan
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
@@ -39,17 +39,9 @@ def mute_findings_in_latest_scans(
with rls_transaction(tenant_id):
mute_rule = MuteRule.objects.get(id=mute_rule_id, tenant_id=tenant_id)
latest_scans = list(
Scan.objects.filter(
tenant_id=tenant_id,
provider_id__in=provider_ids,
state=StateChoices.COMPLETED,
completed_at__isnull=False,
)
.order_by("provider_id", "-completed_at", "-inserted_at", "-id")
.distinct("provider_id")
.values_list("id", flat=True)
)
latest_scans = Scan.objects.filter(
tenant_id=tenant_id, provider_id__in=provider_ids
).latest_ids_per_provider()
changed_scan_ids = []
findings_muted = 0
@@ -84,6 +84,7 @@ _SKIP_RECOVERY = {
"scan-perform-scheduled",
"attack-paths-scan-perform",
"attack-paths-cleanup-stale-scans",
"attack-paths-reap-orphaned-tmp-databases",
"reconcile-orphan-tasks",
}
+29 -12
View File
@@ -2700,20 +2700,37 @@ def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dic
# refreshed). Wiping based on the older scan would zero counts the newer
# scan just set. Skip and let the newer scan's reset task do the work; if
# this task was delayed in the queue, that's the correct outcome.
# `completed_at__isnull=False` is required: Postgres orders NULL first in
# DESC, so a sibling COMPLETED scan with a missing completed_at would sort
# as "newest" and incorrectly cause us to skip.
#
# The comparison must be against the newest *full-scope* scan, which is
# what this variable has always been named after but did not use to be:
# the query filtered nothing about scope, so any newer scan that is not
# full-scope (an imported one, for instance) made the full-scope scan
# skip its own cleanup and leave ephemeral resources with a stale
# failed_findings_count permanently.
#
# `is_full_scope()` reads `trigger` plus the scoping keys inside
# `scanner_args`, which is not expressible as a WHERE clause, so the
# candidates are walked newest-first in Python until the first full-scope
# one. The walk needs no cap: `scan` is itself a full-scope candidate, so
# it stops at `scan` at the latest, after reading only the scans newer than
# it. A fixed window would return None once more newer scoped scans had
# landed than it inspected, and skip the cleanup exactly like the bug above.
#
# NULL `completed_at` no longer needs an explicit filter here: the shared
# ordering in `ScanQuerySet.LATEST_ORDER_BY` sorts NULLs last
# rather than excluding them, which also fixes the case where a provider
# whose completed scans all have a NULL `completed_at` resolved to None and
# therefore never ran the reset at all.
with rls_transaction(tenant_id):
latest_full_scope_scan_id = (
Scan.objects.filter(
tenant_id=tenant_id,
provider_id=scan.provider_id,
state=StateChoices.COMPLETED,
completed_at__isnull=False,
candidates = (
Scan.objects.filter(tenant_id=tenant_id, provider_id=scan.provider_id)
.latest_first()
.only("id", "trigger", "scanner_args")
.iterator(chunk_size=100)
)
.order_by("-completed_at", "-inserted_at")
.values_list("id", flat=True)
.first()
latest_full_scope_scan_id = next(
(candidate.id for candidate in candidates if candidate.is_full_scope()),
None,
)
if latest_full_scope_scan_id != scan.id:
logger.info(
+25 -1
View File
@@ -1,3 +1,4 @@
import json
import os
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -42,6 +43,7 @@ from tasks.jobs.attack_paths import (
)
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
from tasks.jobs.backfill import (
aggregate_scan_category_summaries,
aggregate_scan_resource_group_summaries,
@@ -163,13 +165,28 @@ def create_scan_task_record(
task_id: str,
task_name: str = "scan-perform",
task_status: str | None = states.PENDING,
task_kwargs: dict | None = None,
) -> Task:
"""Pre-create the TaskResult + Task rows for a pre-generated task id.
Pass ``task_kwargs`` when the response built from this record is serialized
before the broker publish. ``task_kwargs`` is otherwise only written by the
``before_task_publish`` signal (``api/signals.py``), and the scan publish is
deferred to ``on_commit``, so the 202 would carry an empty ``task_args`` and
the caller would have no way to learn the scan id it was just handed a task
for. The publish later overwrites the field with the same kwargs as a Python
repr; both forms decode to the same dict (``decode_celery_field``).
"""
if task_status is None:
task_status = states.PENDING
defaults = {"status": task_status, "task_name": task_name}
if task_kwargs is not None:
defaults["task_kwargs"] = json.dumps(task_kwargs)
task_result, _ = TaskResult.objects.update_or_create(
task_id=str(task_id),
defaults={"status": task_status, "task_name": task_name},
defaults=defaults,
)
prowler_task, _ = Task.objects.update_or_create(
id=str(task_id),
@@ -711,6 +728,13 @@ def cleanup_stale_attack_paths_scans_task():
return cleanup_stale_attack_paths_scans()
@shared_task(
name="attack-paths-reap-orphaned-tmp-databases", queue="attack-paths-scans"
)
def reap_orphaned_attack_paths_tmp_databases_task():
return reap_orphaned_tmp_databases()
@shared_task(name="reconcile-orphan-tasks", queue="celery")
def reconcile_orphan_tasks_task():
"""Periodic watchdog: recover tasks whose worker is gone (deploys, crashes)."""
@@ -0,0 +1,286 @@
from datetime import UTC, datetime, timedelta
from unittest.mock import patch
from uuid import uuid4
import pytest
from api.attack_paths.database import TEMP_DB_PREFIX
from api.models import AttackPathsScan, StateChoices
from api.uuid_utils import datetime_to_uuid7
from config.django.base import ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS
MARGIN = timedelta(hours=ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS)
def _tmp_db_name(scan_uuid) -> str:
return f"{TEMP_DB_PREFIX}{scan_uuid}"
@pytest.mark.django_db
class TestReapOrphanedTmpDatabases:
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_ignores_databases_without_the_temp_prefix(self, mock_list, mock_drop):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
mock_list.return_value = ["db-tenant-abc123", "system", "neo4j"]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_drops_temp_db_with_no_scan_row_past_safety_margin(
self, mock_list, mock_drop
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
old_scan_id = datetime_to_uuid7(
datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
)
database = _tmp_db_name(old_scan_id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 1, "databases": [database]}
mock_drop.assert_called_once_with(database)
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_preserves_temp_db_with_no_scan_row_inside_safety_margin(
self, mock_list, mock_drop
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
recent_scan_id = datetime_to_uuid7(datetime.now(tz=UTC) - timedelta(minutes=5))
database = _tmp_db_name(recent_scan_id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_preserves_temp_db_with_unparseable_scan_id(self, mock_list, mock_drop):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
database = f"{TEMP_DB_PREFIX}not-a-uuid"
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_drops_terminal_scan_past_safety_margin(
self, mock_list, mock_drop, tenants_fixture, aws_provider
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
tenant = tenants_fixture[0]
old_updated_at = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=aws_provider,
state=StateChoices.COMPLETED,
)
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=old_updated_at)
database = _tmp_db_name(scan.id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 1, "databases": [database]}
mock_drop.assert_called_once_with(database)
@pytest.mark.parametrize(
"state",
[StateChoices.FAILED, StateChoices.CANCELLED],
)
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_drops_other_terminal_states_past_safety_margin(
self, mock_list, mock_drop, tenants_fixture, aws_provider, state
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
tenant = tenants_fixture[0]
old_updated_at = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=aws_provider,
state=state,
)
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=old_updated_at)
database = _tmp_db_name(scan.id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result["dropped_count"] == 1
mock_drop.assert_called_once_with(database)
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_preserves_terminal_scan_inside_safety_margin(
self, mock_list, mock_drop, tenants_fixture, aws_provider
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
tenant = tenants_fixture[0]
scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=aws_provider,
state=StateChoices.COMPLETED,
)
database = _tmp_db_name(scan.id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_margin_counts_from_completed_at_not_updated_at(
self, mock_list, mock_drop, tenants_fixture, aws_provider
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
tenant = tenants_fixture[0]
old = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=aws_provider,
state=StateChoices.COMPLETED,
)
AttackPathsScan.objects.filter(id=scan.id).update(
updated_at=old, completed_at=datetime.now(tz=UTC)
)
database = _tmp_db_name(scan.id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_drops_scan_completed_past_safety_margin(
self, mock_list, mock_drop, tenants_fixture, aws_provider
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
tenant = tenants_fixture[0]
old = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=aws_provider,
state=StateChoices.FAILED,
)
AttackPathsScan.objects.filter(id=scan.id).update(
updated_at=old, completed_at=old
)
database = _tmp_db_name(scan.id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 1, "databases": [database]}
mock_drop.assert_called_once_with(database)
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_never_drops_an_executing_scan_regardless_of_age(
self, mock_list, mock_drop, tenants_fixture, aws_provider
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
tenant = tenants_fixture[0]
very_old = datetime.now(tz=UTC) - timedelta(days=30)
scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=aws_provider,
state=StateChoices.EXECUTING,
)
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=very_old)
database = _tmp_db_name(scan.id)
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_one_failed_drop_does_not_stop_the_rest_of_the_sweep(
self, mock_list, mock_drop
):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
old_time = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
failing_scan_id = datetime_to_uuid7(old_time)
succeeding_scan_id = datetime_to_uuid7(old_time)
failing_db = _tmp_db_name(failing_scan_id)
succeeding_db = _tmp_db_name(succeeding_scan_id)
mock_list.return_value = [failing_db, succeeding_db]
mock_drop.side_effect = [Exception("boom"), None]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 1, "databases": [succeeding_db]}
assert mock_drop.call_count == 2
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_returns_empty_result_when_listing_databases_fails(self, mock_list):
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
mock_list.side_effect = Exception("neo4j unreachable")
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
def test_preserves_temp_db_with_random_uuid_and_no_row(self, mock_list, mock_drop):
"""A non-UUIDv7 id with no matching row has no reliable timestamp, so it
must be left alone rather than guessed at."""
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
database = _tmp_db_name(uuid4())
mock_list.return_value = [database]
result = reap_orphaned_tmp_databases()
assert result == {"dropped_count": 0, "databases": []}
mock_drop.assert_not_called()
class TestReapOrphanedTmpDatabasesTask:
@patch(
"tasks.tasks.reap_orphaned_tmp_databases",
return_value={"dropped_count": 2, "databases": ["db-tmp-scan-a"]},
)
def test_task_invokes_the_reaper(self, mock_reap):
from tasks.tasks import reap_orphaned_attack_paths_tmp_databases_task
result = reap_orphaned_attack_paths_tmp_databases_task.run()
assert result == {"dropped_count": 2, "databases": ["db-tmp-scan-a"]}
mock_reap.assert_called_once_with()
+95 -3
View File
@@ -3,9 +3,10 @@ import uuid
import zipfile
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock, call, patch
from urllib.parse import parse_qs, urlparse
import boto3
import pytest
from botocore.exceptions import ClientError
from django.test import override_settings
@@ -63,15 +64,46 @@ class TestOutputs:
assert mock_boto_client.call_args.kwargs["region_name"] == "us-east-1"
@patch("tasks.jobs.export.boto3.client")
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000")
def test_get_s3_client_passes_the_endpoint_when_set(self, mock_boto_client):
get_s3_client()
assert mock_boto_client.call_args.kwargs["endpoint_url"] == "http://minio:9000"
@patch("tasks.jobs.export.boto3.client")
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="")
def test_get_s3_client_endpoint_empty_by_default(self, mock_boto_client):
"""Empty keeps today's behavior: no endpoint override, real S3 is used."""
get_s3_client()
assert mock_boto_client.call_args.kwargs["endpoint_url"] is None
@patch("tasks.jobs.export.boto3.client")
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000")
def test_get_s3_client_does_not_fall_back_when_endpoint_set(self, mock_boto_client):
"""A configured endpoint means the explicit client failed talking to it. The fallback
goes to the default provider chain (e.g. an EC2 instance role) against real AWS, so it
must not be used: the original error propagates instead."""
error = ClientError({"Error": {"Code": "403"}}, "ListBuckets")
mock_boto_client.side_effect = error
with pytest.raises(ClientError):
get_s3_client()
mock_boto_client.assert_called_once()
@patch("tasks.jobs.export.boto3.client")
@patch("tasks.jobs.export.settings")
def test_get_s3_client_fallback(self, mock_settings, mock_boto_client):
mock_settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL = ""
mock_boto_client.side_effect = [
ClientError({"Error": {"Code": "403"}}, "ListBuckets"),
MagicMock(),
]
client = get_s3_client()
assert client is not None
assert mock_boto_client.call_args_list[1] == call("s3")
@patch("tasks.jobs.export.get_s3_client")
@patch("tasks.jobs.export.base")
@@ -278,10 +310,70 @@ def _presign(client):
class TestS3PresignClient:
@override_settings(**PRESIGN_SETTINGS, DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="")
def test_no_public_endpoint_returns_none(self):
@override_settings(
**{**PRESIGN_SETTINGS, "DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION": ""},
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
)
def test_no_public_endpoint_and_no_region_returns_none(self):
# Without a region, SigV4 would have to guess one and break other regions.
assert get_s3_presign_client() is None
@override_settings(**PRESIGN_SETTINGS, DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="")
def test_region_without_public_endpoint_signs_sigv4_on_the_regional_host(self):
# SSE-KMS objects reject the SigV2 URLs boto3 presigns by default, and the
# global host redirects for new buckets, which breaks a SigV4 signature.
url = urlparse(_presign(get_s3_presign_client()))
query = parse_qs(url.query)
assert url.netloc == "s3.eu-west-1.amazonaws.com"
assert url.path == "/output-bucket/tenant/scan/report.zip"
assert query["X-Amz-Algorithm"] == ["AWS4-HMAC-SHA256"]
assert "/eu-west-1/s3/aws4_request" in query["X-Amz-Credential"][0]
@override_settings(
**{
**PRESIGN_SETTINGS,
"DJANGO_OUTPUT_S3_AWS_ACCESS_KEY_ID": "",
"DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY": "",
},
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
)
def test_region_without_static_keys_signs_with_the_default_chain(self, monkeypatch):
# An ECS task role reaches boto3 through the default chain, like the env here.
# A fresh default session keeps these keys from being cached for later tests.
monkeypatch.setattr(boto3, "DEFAULT_SESSION", None)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "role-access-key")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "role-secret-key")
monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
query = parse_qs(urlparse(_presign(get_s3_presign_client())).query)
assert query["X-Amz-Credential"][0].startswith("role-access-key/")
assert "/eu-west-1/s3/aws4_request" in query["X-Amz-Credential"][0]
@override_settings(
**{**PRESIGN_SETTINGS, "DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION": ""},
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000",
)
def test_internal_endpoint_without_public_endpoint_signs_against_it(self):
# No browser-reachable host was configured, so the internal one is the best
# available target instead of falling through to the real AWS host.
url = urlparse(_presign(get_s3_presign_client()))
assert url.netloc == "minio:9000"
assert url.path == "/output-bucket/tenant/scan/report.zip"
@override_settings(
**PRESIGN_SETTINGS,
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="https://storage.example.com",
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000",
)
def test_public_endpoint_wins_over_the_internal_endpoint(self):
url = urlparse(_presign(get_s3_presign_client()))
assert url.netloc == "storage.example.com"
@override_settings(
**PRESIGN_SETTINGS,
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="https://storage.example.com",
+106
View File
@@ -5967,6 +5967,112 @@ class TestResetEphemeralResourceFindingsCount:
resource2.refresh_from_db()
assert resource2.failed_findings_count == 5
def test_runs_when_newer_scan_is_not_full_scope(
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
):
"""A newer scoped scan must not block the full-scope scan's cleanup.
The race guard used to pick the newest COMPLETED scan of any scope
despite being named after full-scope ones, so a single scoped scan
landing after a complete one made the complete scan skip its cleanup
and leave ephemeral resources with a stale count permanently.
"""
from datetime import timedelta
tenant, *_ = tenants_fixture
scan1, *_ = scans_fixture
resource1, resource2, _ = resources_fixture
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
self._make_scan_summary(tenant.id, scan1.id, resource1)
newer_completed_at = scan1.completed_at + timedelta(minutes=5)
Scan.objects.create(
name="Newer scoped scan",
provider=aws_provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant_id=tenant.id,
started_at=newer_completed_at,
completed_at=newer_completed_at,
scanner_args={"checks": ["check1"]},
)
result = reset_ephemeral_resource_findings_count(
tenant_id=str(tenant.id), scan_id=str(scan1.id)
)
assert result["status"] == "completed"
resource2.refresh_from_db()
assert resource2.failed_findings_count == 0
def test_runs_when_many_newer_scans_are_not_full_scope(
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
):
"""The walk must not give up before it reaches the full-scope scan.
A fixed look-back window returned None once more newer scoped scans
had landed than it inspected, and skipped the cleanup exactly like the
original bug did with one.
"""
from datetime import timedelta
tenant, *_ = tenants_fixture
scan1, *_ = scans_fixture
resource1, resource2, _ = resources_fixture
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
self._make_scan_summary(tenant.id, scan1.id, resource1)
for minutes in range(1, 41):
newer_completed_at = scan1.completed_at + timedelta(minutes=minutes)
Scan.objects.create(
name=f"Newer scoped scan {minutes}",
provider=aws_provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant_id=tenant.id,
started_at=newer_completed_at,
completed_at=newer_completed_at,
scanner_args={"checks": ["check1"]},
)
result = reset_ephemeral_resource_findings_count(
tenant_id=str(tenant.id), scan_id=str(scan1.id)
)
assert result["status"] == "completed"
resource2.refresh_from_db()
assert resource2.failed_findings_count == 0
def test_runs_when_completed_at_is_null(
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
):
"""NULL `completed_at` used to make the reset never run at all.
The old guard filtered `completed_at__isnull=False`, so a provider
whose completed scans all had a NULL `completed_at` resolved the
"latest" scan to None, which never equals `scan.id`.
"""
tenant, *_ = tenants_fixture
scan1, *_ = scans_fixture
resource1, resource2, _ = resources_fixture
Scan.all_objects.filter(id=scan1.id).update(completed_at=None)
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
self._make_scan_summary(tenant.id, scan1.id, resource1)
result = reset_ephemeral_resource_findings_count(
tenant_id=str(tenant.id), scan_id=str(scan1.id)
)
assert result["status"] == "completed"
resource2.refresh_from_db()
assert resource2.failed_findings_count == 0
def test_does_not_touch_other_providers_resources(
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
):
+79
View File
@@ -1,3 +1,4 @@
import json
import uuid
from contextlib import contextmanager
from datetime import UTC, datetime
@@ -14,6 +15,7 @@ from api.models import (
StateChoices,
Task,
)
from api.v1.serializers import TaskSerializer
from botocore.exceptions import ClientError
from celery import states
from django_celery_beat.models import IntervalSchedule, PeriodicTask
@@ -32,6 +34,7 @@ from tasks.tasks import (
_scan_tmp_output_directory,
check_integrations_task,
check_lighthouse_provider_connection_task,
create_scan_task_record,
generate_outputs_task,
mute_findings_in_latest_scans_task,
perform_attack_paths_scan_task,
@@ -3384,3 +3387,79 @@ class TestTaskTimeLimits:
"lighthouse-provider-connection-check",
):
assert celery_app.tasks[name].time_limit < default
@pytest.mark.django_db
class TestCreateScanTaskRecord:
"""`task_kwargs` is what a response built before the publish can report."""
def _scan(self, tenant, provider):
"""A manual scan, like the one `POST /api/v1/scans` creates."""
return Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Manual scan",
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
)
def _publish_kwargs(self, tenant, scan):
"""What `enqueue_scan_execution_on_commit` publishes for this scan."""
return {
"tenant_id": str(tenant.id),
"scan_id": str(scan.id),
"provider_id": str(scan.provider_id),
}
def _task_args(self, task):
"""Read the record back the way `TaskSerializer` does."""
return TaskSerializer(task).data["task_args"]
def test_the_stored_kwargs_are_the_ones_the_publish_would_send(
self, tenants_fixture, aws_provider
):
"""The 202 reports what is stored here, so it has to be the dispatch kwargs."""
tenant = tenants_fixture[0]
scan = self._scan(tenant, aws_provider)
task = create_scan_task_record(
tenant_id=str(tenant.id),
task_id=str(uuid.uuid4()),
task_kwargs=self._publish_kwargs(tenant, scan),
)
assert self._task_args(task) == {
"scan_id": str(scan.id),
"provider_id": str(aws_provider.id),
}
def test_a_record_created_without_kwargs_reports_none(self, tenants_fixture):
"""The argument is optional, so the other callers keep their behaviour."""
task = create_scan_task_record(
tenant_id=str(tenants_fixture[0].id),
task_id=str(uuid.uuid4()),
)
assert self._task_args(task) == {}
def test_the_publish_can_overwrite_the_stored_kwargs(
self, tenants_fixture, aws_provider
):
"""django-celery-results stores a Python repr; both must decode alike."""
tenant = tenants_fixture[0]
scan = self._scan(tenant, aws_provider)
task_id = str(uuid.uuid4())
kwargs = self._publish_kwargs(tenant, scan)
task = create_scan_task_record(
tenant_id=str(tenant.id), task_id=task_id, task_kwargs=kwargs
)
before = self._task_args(task)
# What `before_task_publish` writes once the task reaches the broker.
task_result = TaskResult.objects.get(task_id=task_id)
task_result.task_kwargs = json.dumps(repr(kwargs))
task_result.save(update_fields=["task_kwargs"])
task.refresh_from_db()
assert self._task_args(task) == before
Generated
+5 -5
View File
@@ -53,7 +53,7 @@ constraints = [
{ name = "aliyun-log-fastpb", specifier = "==0.2.0" },
{ name = "amqp", specifier = "==5.3.1" },
{ name = "annotated-types", specifier = "==0.7.0" },
{ name = "anyio", specifier = "==4.12.1" },
{ name = "anyio", specifier = "==4.14.2" },
{ name = "applicationinsights", specifier = "==0.11.10" },
{ name = "apscheduler", specifier = "==3.11.2" },
{ name = "argcomplete", specifier = "==3.5.3" },
@@ -969,15 +969,15 @@ wheels = [
[[package]]
name = "anyio"
version = "4.12.1"
version = "4.14.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "idna" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" }
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" },
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
]
[[package]]
@@ -4938,7 +4938,7 @@ dependencies = [
[[package]]
name = "prowler-api"
version = "1.44.0"
version = "1.45.0"
source = { virtual = "." }
dependencies = [
{ name = "cartography" },
+8
View File
@@ -15,6 +15,7 @@ services:
api:
hostname: "prowler-api"
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
restart: unless-stopped
env_file:
- path: .env
required: false
@@ -44,6 +45,7 @@ services:
ui:
image: prowlercloud/prowler-ui:${PROWLER_UI_VERSION:-stable}
restart: unless-stopped
env_file:
- path: .env
required: false
@@ -61,6 +63,7 @@ services:
postgres:
image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
restart: unless-stopped
hostname: "postgres-db"
volumes:
- ./_data/postgres:/var/lib/postgresql/data
@@ -81,6 +84,7 @@ services:
valkey:
image: valkey/valkey:8-alpine@sha256:a038175878d66b9d274fbf8be73c0305e93798b83917647f167e18cef3c71eec
restart: unless-stopped
hostname: "valkey"
volumes:
- ./_data/valkey:/data
@@ -97,6 +101,7 @@ services:
neo4j:
image: graphstack/dozerdb:5.26.27.0@sha256:9b54d6b3a98a76c00bd23e8e78d8c82081ff168162aebd47b25c234e092cb0a0
restart: unless-stopped
hostname: "neo4j"
volumes:
- ./_data/neo4j:/data
@@ -129,6 +134,7 @@ services:
worker:
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
restart: unless-stopped
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
stop_grace_period: 120s
env_file:
@@ -149,6 +155,7 @@ services:
worker-beat:
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
restart: unless-stopped
env_file:
- path: ./.env
required: false
@@ -165,6 +172,7 @@ services:
mcp-server:
image: prowlercloud/prowler-mcp:${PROWLER_MCP_VERSION:-stable}
restart: unless-stopped
environment:
- PROWLER_MCP_TRANSPORT_MODE=http
env_file:
+67
View File
@@ -4,6 +4,73 @@ description: "New features and improvements in each Prowler release"
rss: true
---
<Update label="v5.43.0" description="September 21, 2026">
### 🏛️ Compliance — FedRAMP 20x Consolidated Rules 2026
The FedRAMP 20x Phase One pilot frameworks (`fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp`) are replaced by two universal frameworks built from the FedRAMP Consolidated Rules for 2026, each covering AWS, Azure, GCP, Kubernetes and Microsoft 365 from a single definition:
- **FedRAMP 20x KSI** (`fedramp_20x_ksi_2026`): the 46 Key Security Indicators across 10 themes. There is one indicator catalog for every class instead of a separate Low fork; each indicator carries its class applicability and NIST SP 800-53 controls.
- **FedRAMP 20x Class C FRR** (`fedramp_20x_frr_class_c_2026`): the 158 FedRAMP Rules of the Class C ruleset that bind cloud service providers. Most are program obligations (reports, notifications, certification package) and stay manual; checks are mapped only where they evidence part of the rule text. Configurable checks carry configuration requirements, so a relaxed `audit_config` cannot turn a requirement green.
Automation or stored results that reference the pilot framework IDs need to move to `fedramp_20x_ksi_2026`. In Prowler App, both frameworks show the per-provider breakdown in the cross-provider compliance view and can be downloaded as OCSF.
Read more in the [Compliance documentation](https://docs.prowler.com/user-guide/compliance/tutorials/compliance).
### 🔎 AWS — Inspector Coverage, CISA KEV and FIPS Checks
Seven new AWS checks back the vulnerability detection and cryptography rules of FedRAMP 20x Class C:
- `inspector2_coverage_scan_status_active` and `inspector2_coverage_recently_scanned` report resources Amazon Inspector is not scanning, or last scanned more than `inspector2_max_days_since_last_scan` days ago (default 3).
- `inspector2_active_findings_no_known_exploited_vulnerabilities` and `inspector2_active_findings_kev_within_due_date` report active findings whose CVE is in the CISA Known Exploited Vulnerabilities catalog, and those still open past the CISA due date. The KEV data comes from Inspector itself through `inspector2:BatchGetFindingDetails`, so no external feed is needed.
- `inspector2_active_findings_within_max_age` reports active findings first observed more than `inspector2_active_finding_max_age_days` days ago (default 192).
- `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` report HTTPS/TLS load balancer listeners and Transfer Family servers without a FIPS security policy.
`inspector2:BatchGetFindingDetails` is not part of `SecurityAudit`, so it is now included in the Prowler additions policy and the CloudFormation scan role. Without it, the KEV checks report `MANUAL` naming the missing permission instead of a false `FAIL`.
Explore all AWS checks at [Prowler Hub](https://hub.prowler.com/check?provider=aws).
### ☁️ AWS — Partition Bootstrap Failover
When `PROWLER_AWS_PARTITION` is set, the bootstrap STS calls (validating credentials, assuming a role and getting an MFA session token) now try up to two more regions of the partition if the first one cannot be reached. A GovCloud host whose configured region belongs to another partition was still sent to `us-gov-east-1`, and on a network that routes only to `us-gov-west-1` the connection check and the scan failed on perfectly valid credentials. Only connection errors and timeouts move on to the next region; credential errors are reported from the first one as before. Later STS calls reuse the region that answered, and nothing changes when `PROWLER_AWS_PARTITION` is unset.
Read more in the [AWS Regions and Partitions documentation](https://docs.prowler.com/user-guide/providers/aws/regions-and-partitions).
### 🌐 Azure — Sovereign Cloud Endpoints for Defender and Key Vault
Defender security contacts and Key Vault key rotation policies now use the endpoints of the cloud selected with `--azure-region` instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud`. Key Vault clients are built from the vault URI that Azure returns for each vault.
Read more in the [Azure non-default cloud documentation](https://docs.prowler.com/user-guide/providers/azure/use-non-default-cloud).
### ✉️ Invitations — Expired Invitations No Longer Block Re-Invites
A pending invitation past its expiry date is now reported as expired, and inviting the same email again marks it as expired and creates the new invitation instead of returning a generic error. The Invitations table disables Edit and Revoke on expired and revoked invitations, and `filter[state__in]` on the invitations endpoint no longer returns a server error.
In Prowler Cloud, new organizations are offered an **Invite your team** step once the first provider is connected, and Prowler Private Cloud deployments can set `UI_SELF_REGISTRATION_ENABLED=false` to make sign-up invitation-only.
Read more in the [Invitations documentation](https://docs.prowler.com/user-guide/tutorials/prowler-app-rbac#invitations).
### 📄 Reports — Downloads on Self-Hosted Storage
Report downloads no longer depend on the storage host being the same inside and outside the container network. `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL` signs the download URL against a browser-reachable host, so a deployment whose object storage answers only on an internal address serves the file instead of a link the browser cannot open. Leaving `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` unset, common on S3-compatible storage with no meaningful region, no longer makes the download fail with a server error.
### 🔍 Checks
- **Huawei Cloud:** new `smn_topic_subscriptions` check reports SMN topics without any subscription.
- **Cloudflare:** the API token links in the provider wizard request the SSL and Certificates, Bot Management and Zone WAF read permissions the checks need, so a token created from the wizard no longer produces failures on permissions it was never granted.
- **Microsoft 365:** five Defender malware, anti-phishing and inbound anti-spam checks no longer fail with `KeyError` on tenants that use the Standard or Strict preset security policies, which dropped every finding of those checks. Preset policies are covered by `defender_strict_preset_security_policy_enabled`.
- **Google Workspace:** `security_2sv_enforced` reports domain-wide 2-Step Verification failures as `FAIL` even when every failing setting is overridden for a group or organizational unit.
Explore all checks at [Prowler Hub](https://hub.prowler.com/check).
### 🔐 Security Updates
- `libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK and API container images, patching high-severity Debian CVEs.
- PowerShell upgraded to 7.5.11 in the SDK and API container images, bundling .NET runtime 9.0.20 and patching CVE-2026-62901.
- `anyio` upgraded to 4.14.2 in the SDK, the API and the MCP Server, patching CVE-2026-63374.
See the [full release notes on GitHub](https://github.com/prowler-cloud/prowler/releases/tag/5.43.0) for the complete list of changes.
</Update>
<Update label="v5.42.0" description="September 11, 2026">
### ☁️ AWS — ISO Partitions
@@ -128,8 +128,8 @@ To update the environment file:
Edit the `.env` file and change version values:
```env
PROWLER_UI_VERSION="5.42.0"
PROWLER_API_VERSION="5.42.0"
PROWLER_UI_VERSION="5.43.0"
PROWLER_API_VERSION="5.43.0"
```
<Note>
+1 -1
View File
@@ -71,7 +71,7 @@ Prowler supports a wide range of providers organized by category:
| [LLM](/user-guide/providers/llm/getting-started-llm) | Official | Models | CLI |
| [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | Tenants | UI, API, CLI |
| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | Organizations | UI, API, CLI |
| [Okta](/user-guide/providers/okta/getting-started-okta) | Official | Organizations | CLI |
| [Okta](/user-guide/providers/okta/getting-started-okta) | Official | Organizations | UI, API, CLI |
| [Vercel](/user-guide/providers/vercel/getting-started-vercel) | Official | Teams / Projects | UI, API, CLI |
### Kubernetes
@@ -29,6 +29,22 @@ Boto3 defaults both timeouts to 60 seconds. In networks with restricted egress (
</Note>
## Retries Configuration
<VersionBadge version="5.44.0" />
The number of retries is set with `--aws-retries-max-attempts`, where `0` disables retries. It can also be set through an environment variable, which is the way to tune it in Prowler Cloud and other deployments without a CLI:
```console
export PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS=0
```
The CLI flag takes precedence over the environment variable. The value must be a non-negative integer; when neither is set, Prowler uses 3 retries.
<Warning>
The environment variable is process-wide: it applies to every AWS provider built in the process where it is set, not only to a connection check. A scan started in that same process picks it up too. Boto3's Standard retry mode, which Prowler uses, also retries service-side throttling responses (see the errors listed below), so `0` disables retries for those as well. On a large account a scan can hit throttling under normal load, and with retries disabled that throttling becomes a hard failure instead of a retried call. Set the variable only on the processes that run connection checks; leave scan workers on the default, or raise their retry count instead of lowering it.
</Warning>
## Retry Behavior Overview
Boto3's Standard retry mode includes the following mechanisms:
@@ -89,6 +89,10 @@ After adding your cloud account credentials, click the `Check connection` button
<img src="/images/test-connection-button.png" alt="Test Connection" width="700" />
<Note>
For a single AWS account, Prowler tests the connection as part of the `Connect account` step, so the wizard moves straight to launching the scan.
</Note>
## Step 6: Scan Started
After the connection check succeeds, save the provider and start your first scan with the `Launch Scan` button. The `Scans` section shows the scan in progress:
+8
View File
@@ -4,6 +4,14 @@ All notable changes to the **Prowler MCP Server** are documented in this file.
<!-- changelog: release notes start -->
## [0.12.2] (Prowler v5.43.0)
### 🔐 Security
- Bumped `anyio` to 4.14.2 to resolve CVE-2026-63374 [(#12848)](https://github.com/prowler-cloud/prowler/pull/12848)
---
## [0.12.1] (Prowler v5.42.0)
### 🔐 Security
+3 -3
View File
@@ -39,15 +39,15 @@ wheels = [
[[package]]
name = "anyio"
version = "4.13.0"
version = "4.14.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "idna" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" }
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" },
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
]
[[package]]
+28
View File
@@ -4,6 +4,34 @@ All notable changes to the **Prowler SDK** are documented in this file.
<!-- changelog: release notes start -->
## [5.43.0] (Prowler v5.43.0)
### 🚀 Added
- `FedRAMP-20x-KSI` universal compliance framework (`fedramp_20x_ksi_2026`) with the 46 Key Security Indicators from the FedRAMP Consolidated Rules 2026 mapped for AWS, Azure, GCP, Kubernetes and M365 [(#11701)](https://github.com/prowler-cloud/prowler/pull/11701)
- `smn_topic_subscriptions` check for Huawei Cloud provider: SMN topics have at least one subscription configured [(#12186)](https://github.com/prowler-cloud/prowler/pull/12186)
- `inspector2_coverage_scan_status_active`, `inspector2_coverage_recently_scanned`, `inspector2_active_findings_no_known_exploited_vulnerabilities`, `inspector2_active_findings_kev_within_due_date`, `inspector2_active_findings_within_max_age`, `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` checks for AWS provider, covering FedRAMP 20x Class C vulnerability detection, CISA KEV remediation and FIPS cryptography rules; the KEV checks require `inspector2:BatchGetFindingDetails`, now in the Prowler additions policy [(#12808)](https://github.com/prowler-cloud/prowler/pull/12808)
- `FedRAMP-20x-FRR-Class-C` universal compliance framework (`fedramp_20x_frr_class_c_2026`) with the 158 provider rules of the FedRAMP 20x Class C ruleset from the FedRAMP Consolidated Rules 2026 for AWS, Azure, GCP, Kubernetes and M365 [(#12808)](https://github.com/prowler-cloud/prowler/pull/12808)
### 🔄 Changed
- FedRAMP 20x Phase One pilot frameworks `fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp` replaced by `fedramp_20x_ksi_2026` [(#12855)](https://github.com/prowler-cloud/prowler/pull/12855)
### 🐞 Fixed
- `security_2sv_enforced` reports domain-wide 2-Step Verification failures as FAIL even when every failing setting is overridden for a group or organizational unit [(#12700)](https://github.com/prowler-cloud/prowler/pull/12700)
- Bootstrap STS calls now try up to two more regions of the partition declared in `PROWLER_AWS_PARTITION` when the first one cannot be reached, so a deployment that routes to only one region of its partition no longer fails on an endpoint it has no path to. This covers validating credentials, assuming a role and getting an MFA session token [(#12799)](https://github.com/prowler-cloud/prowler/pull/12799)
- `KeyError` in M365 Defender malware, anti-phishing and inbound anti-spam checks when the tenant has Standard or Strict preset security policies [(#12809)](https://github.com/prowler-cloud/prowler/pull/12809)
- Azure Defender security contacts and Key Vault key rotation policies now use the endpoints of the selected cloud (`--azure-region`) instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud` [(#12813)](https://github.com/prowler-cloud/prowler/pull/12813)
### 🔐 Security
- `libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK container image, patching nine high Debian CVEs [(#12804)](https://github.com/prowler-cloud/prowler/pull/12804)
- PowerShell from 7.5.9 to 7.5.11 in the SDK container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901 [(#12811)](https://github.com/prowler-cloud/prowler/pull/12811)
- Bumped `anyio` to 4.14.2 to resolve CVE-2026-63374 [(#12848)](https://github.com/prowler-cloud/prowler/pull/12848)
---
## [5.42.0] (Prowler v5.42.0)
### 🚀 Added
@@ -1 +0,0 @@
`inspector2_coverage_scan_status_active`, `inspector2_coverage_recently_scanned`, `inspector2_active_findings_no_known_exploited_vulnerabilities`, `inspector2_active_findings_kev_within_due_date`, `inspector2_active_findings_within_max_age`, `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` checks for AWS provider, covering FedRAMP 20x Class C vulnerability detection, CISA KEV remediation and FIPS cryptography rules; the KEV checks require `inspector2:BatchGetFindingDetails`, now in the Prowler additions policy
@@ -1 +0,0 @@
Bootstrap STS calls now try up to two more regions of the partition declared in `PROWLER_AWS_PARTITION` when the first one cannot be reached, so a deployment that routes to only one region of its partition no longer fails on an endpoint it has no path to. This covers validating credentials, assuming a role and getting an MFA session token
@@ -0,0 +1 @@
`PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS` environment variable to set the Boto3 retries for deployments without CLI flags
@@ -0,0 +1 @@
STS calls after role assumption use the answering region, avoiding a second wait for an unreachable partition region
@@ -1 +0,0 @@
Azure Defender security contacts and Key Vault key rotation policies now use the endpoints of the selected cloud (`--azure-region`) instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud`
@@ -1 +0,0 @@
`FedRAMP-20x-FRR-Class-C` universal compliance framework (`fedramp_20x_frr_class_c_2026`) with the 158 provider rules of the FedRAMP 20x Class C ruleset from the FedRAMP Consolidated Rules 2026 for AWS, Azure, GCP, Kubernetes and M365
@@ -1 +0,0 @@
`FedRAMP-20x-KSI` universal compliance framework (`fedramp_20x_ksi_2026`) with the 46 Key Security Indicators from the FedRAMP Consolidated Rules 2026 mapped for AWS, Azure, GCP, Kubernetes and M365
@@ -1 +0,0 @@
`fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp` FedRAMP 20x Phase One pilot frameworks, superseded by `fedramp_20x_ksi_2026`
@@ -1 +0,0 @@
`security_2sv_enforced` reports domain-wide 2-Step Verification failures as FAIL even when every failing setting is overridden for a group or organizational unit
@@ -1 +0,0 @@
`KeyError` in M365 Defender malware, anti-phishing and inbound anti-spam checks when the tenant has Standard or Strict preset security policies
@@ -1 +0,0 @@
`libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK container image, patching nine high Debian CVEs
@@ -1 +0,0 @@
PowerShell from 7.5.9 to 7.5.11 in the SDK container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901
@@ -1 +0,0 @@
`smn_topic_subscriptions` check for Huawei Cloud provider: SMN topics have at least one subscription configured
@@ -0,0 +1 @@
Pass the E2E AWS credentials to the UI E2E workflow through environment variables instead of template expansion
+1 -1
View File
@@ -52,7 +52,7 @@ class _MutableTimestamp:
timestamp = _MutableTimestamp(datetime.today())
timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc))
prowler_version = "5.43.0"
prowler_version = "5.44.0"
html_logo_url = "https://github.com/prowler-cloud/prowler/"
square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png"
aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png"
+10 -3
View File
@@ -112,7 +112,7 @@ class AwsProvider(Provider):
def __init__(
self,
retries_max_attempts: int = 3,
retries_max_attempts: Optional[int] = None,
role_arn: str = None,
session_duration: int = 3600,
external_id: str = None,
@@ -141,6 +141,7 @@ class AwsProvider(Provider):
Args:
- retries_max_attempts: The maximum number of retries for the AWS client.
Defaults to the PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS environment variable or, if unset, to 3.
- role_arn: The ARN of the IAM role to assume.
- session_duration: The duration of the session in seconds, between 900 and 43200.
- external_id: The external ID to use when assuming the IAM role.
@@ -1230,7 +1231,8 @@ class AwsProvider(Provider):
Args:
- session: The AWS session object
- assumed_role_info: The AWSAssumeRoleInfo object
- assumed_role_info: The AWSAssumeRoleInfo object. Its sts_region is
updated to the region that answered, so later calls go straight there
Returns:
- AWSCredentials: The AWS credentials for the assumed role
@@ -1256,11 +1258,14 @@ class AwsProvider(Provider):
mfa_info = AwsProvider.input_role_mfa_token_and_code()
assume_role_arguments["SerialNumber"] = mfa_info.arn
assume_role_arguments["TokenCode"] = mfa_info.totp
_, assumed_credentials = AwsProvider.sts_call_with_partition_failover(
sts_region, assumed_credentials = (
AwsProvider.sts_call_with_partition_failover(
session,
assumed_role_info.sts_region,
lambda sts_client: sts_client.assume_role(**assume_role_arguments),
)
)
assumed_role_info.sts_region = sts_region
# Convert the UTC datetime object to your local timezone
credentials_expiration_local_time = (
assumed_credentials["Credentials"]["Expiration"]
@@ -1558,6 +1563,8 @@ class AwsProvider(Provider):
session,
assumed_role_information,
)
# Validate where the role was assumed, not where it timed out
aws_region = assumed_role_information.sts_region
session = Session(
aws_access_key_id=assumed_role_credentials.aws_access_key_id,
aws_secret_access_key=assumed_role_credentials.aws_secret_access_key,
+157 -54
View File
@@ -122,6 +122,21 @@
"aws-us-gov": []
}
},
"account-access": {
"regions": {
"aws": [
"ap-southeast-1",
"us-west-1"
],
"aws-cn": [],
"aws-eusc": [],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
}
},
"acm": {
"regions": {
"aws": [
@@ -242,6 +257,24 @@
]
}
},
"agent-registry": {
"regions": {
"aws": [
"ap-northeast-1",
"ap-southeast-2",
"eu-west-1",
"us-east-1",
"us-west-2"
],
"aws-cn": [],
"aws-eusc": [],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
}
},
"ahl": {
"regions": {
"aws": [
@@ -266,14 +299,20 @@
"aiops": {
"regions": {
"aws": [
"ap-east-1",
"ap-northeast-1",
"ap-south-1",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-5",
"ap-southeast-7",
"eu-central-1",
"eu-north-1",
"eu-south-2"
"eu-south-2",
"eu-west-1",
"us-east-1",
"us-east-2",
"us-west-2"
],
"aws-cn": [],
"aws-eusc": [],
@@ -973,12 +1012,15 @@
"aws": [
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-5",
"ca-central-1",
"ca-west-1",
"eu-central-1",
"eu-central-2",
"eu-south-1",
"eu-south-2",
"eu-west-1",
@@ -1545,37 +1587,11 @@
]
}
},
"awstransform": {
"regions": {
"aws": [
"ap-northeast-1",
"ap-northeast-2",
"ap-south-1",
"ap-southeast-2",
"ca-central-1",
"eu-central-1",
"eu-west-2",
"sa-east-1",
"us-east-1"
],
"aws-cn": [],
"aws-eusc": [],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
}
},
"b2bi": {
"regions": {
"aws": [
"ap-south-2",
"ap-southeast-2",
"ca-central-1",
"eu-central-1",
"eu-west-1",
"eu-west-3",
"us-east-1",
"us-east-2",
"us-west-2"
@@ -1889,6 +1905,7 @@
"ap-northeast-1",
"ap-northeast-2",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-5",
@@ -1904,6 +1921,7 @@
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-1",
"us-west-2"
],
"aws-cn": [],
@@ -4202,16 +4220,33 @@
"dax": {
"regions": {
"aws": [
"af-south-1",
"ap-east-1",
"ap-east-2",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-3",
"ap-southeast-4",
"ap-southeast-5",
"ap-southeast-6",
"ap-southeast-7",
"ca-central-1",
"ca-west-1",
"eu-central-1",
"eu-central-2",
"eu-north-1",
"eu-south-1",
"eu-south-2",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"il-central-1",
"mx-central-1",
"sa-east-1",
"us-east-1",
"us-east-2",
@@ -6006,8 +6041,10 @@
"evs": {
"regions": {
"aws": [
"ap-east-2",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
@@ -6019,9 +6056,11 @@
"eu-central-2",
"eu-north-1",
"eu-south-1",
"eu-south-2",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"il-central-1",
"mx-central-1",
"sa-east-1",
"us-east-1",
@@ -9218,9 +9257,12 @@
"aws": [
"af-south-1",
"ap-east-1",
"ap-east-2",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-5",
@@ -9228,12 +9270,17 @@
"ca-central-1",
"ca-west-1",
"eu-central-1",
"eu-central-2",
"eu-north-1",
"eu-south-1",
"eu-south-2",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"il-central-1",
"me-central-1",
"me-south-1",
"mx-central-1",
"sa-east-1",
"us-east-1",
"us-east-2",
@@ -9965,7 +10012,9 @@
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
"aws-us-gov": [
"us-gov-west-1"
]
}
},
"neptune": {
@@ -10427,7 +10476,9 @@
"us-west-2"
],
"aws-cn": [],
"aws-eusc": [],
"aws-eusc": [
"eusc-de-east-1"
],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
@@ -10902,7 +10953,10 @@
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
"aws-us-gov": [
"us-gov-east-1",
"us-gov-west-1"
]
}
},
"pca-connector-scep": {
@@ -10964,6 +11018,8 @@
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-3",
"ap-southeast-4",
"ap-southeast-6",
"eu-central-1",
"eu-north-1",
"eu-south-1",
@@ -13414,6 +13470,21 @@
"aws-us-gov": []
}
},
"securityagent": {
"regions": {
"aws": [
"us-east-1",
"us-west-2"
],
"aws-cn": [],
"aws-eusc": [],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
}
},
"securityhub": {
"regions": {
"aws": [
@@ -13877,6 +13948,7 @@
"ap-southeast-3",
"ap-southeast-4",
"ap-southeast-5",
"ap-southeast-6",
"ap-southeast-7",
"ca-central-1",
"ca-west-1",
@@ -14839,6 +14911,18 @@
"aws-us-gov": []
}
},
"supportauthz": {
"regions": {
"aws": [],
"aws-cn": [],
"aws-eusc": [],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
}
},
"sustainability": {
"regions": {
"aws": [
@@ -15062,8 +15146,10 @@
"aws": [
"ap-northeast-1",
"ap-south-1",
"ap-southeast-1",
"ap-southeast-2",
"eu-central-1",
"eu-north-1",
"eu-west-1",
"us-east-1",
"us-east-2",
@@ -15083,20 +15169,28 @@
"timestream-influxdb": {
"regions": {
"aws": [
"af-south-1",
"ap-east-1",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-3",
"ap-southeast-4",
"ap-southeast-7",
"ca-central-1",
"eu-central-1",
"eu-central-2",
"eu-north-1",
"eu-south-1",
"eu-south-2",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"il-central-1",
"me-central-1",
"mx-central-1",
"sa-east-1",
@@ -15162,29 +15256,6 @@
]
}
},
"tnb": {
"regions": {
"aws": [
"ap-northeast-2",
"ap-southeast-2",
"ca-central-1",
"eu-central-1",
"eu-north-1",
"eu-south-2",
"eu-west-3",
"sa-east-1",
"us-east-1",
"us-west-2"
],
"aws-cn": [],
"aws-eusc": [],
"aws-iso": [],
"aws-iso-b": [],
"aws-iso-e": [],
"aws-iso-f": [],
"aws-us-gov": []
}
},
"transcribe": {
"regions": {
"aws": [
@@ -15437,7 +15508,38 @@
"uxc": {
"regions": {
"aws": [
"us-east-1"
"af-south-1",
"ap-east-1",
"ap-east-2",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-3",
"ap-southeast-4",
"ap-southeast-5",
"ap-southeast-6",
"ap-southeast-7",
"ca-central-1",
"ca-west-1",
"eu-central-1",
"eu-central-2",
"eu-north-1",
"eu-south-1",
"eu-south-2",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"il-central-1",
"mx-central-1",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-1",
"us-west-2"
],
"aws-cn": [],
"aws-eusc": [],
@@ -16123,6 +16225,7 @@
"aws": [
"ap-east-1",
"ap-northeast-2",
"ap-southeast-1",
"ap-southeast-5",
"eu-south-2",
"me-central-1",
+23 -2
View File
@@ -2,7 +2,10 @@ import os
from botocore.config import Config
from prowler.providers.aws.exceptions.exceptions import AWSInvalidBoto3TimeoutError
from prowler.providers.aws.exceptions.exceptions import (
AWSInvalidBoto3RetriesError,
AWSInvalidBoto3TimeoutError,
)
AWS_STS_GLOBAL_ENDPOINT_REGION = "us-east-1"
AWS_REGION_US_EAST_1 = "us-east-1"
@@ -27,10 +30,28 @@ def get_boto3_timeout_from_env(name: str, default: int) -> int:
return int(raw)
def get_boto3_retries_from_env(name: str, default: int) -> int:
"""Non-negative integer retries read from the environment, or default when unset."""
raw = os.getenv(name, "").strip()
if not raw:
return default
if not raw.isdecimal():
raise AWSInvalidBoto3RetriesError(
file=os.path.basename(__file__),
message=f"{name} must be a non-negative integer number of retries, got {raw!r}",
)
return int(raw)
def get_default_session_config() -> Config:
return Config(
user_agent_extra=BOTO3_USER_AGENT_EXTRA,
retries={"max_attempts": BOTO3_RETRIES_MAX_ATTEMPTS, "mode": "standard"},
retries={
"max_attempts": get_boto3_retries_from_env(
"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", BOTO3_RETRIES_MAX_ATTEMPTS
),
"mode": "standard",
},
connect_timeout=get_boto3_timeout_from_env(
"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", BOTO3_CONNECT_TIMEOUT
),
@@ -82,6 +82,10 @@ class AWSBaseException(ProwlerException):
"message": "The Boto3 timeout configured through the environment is invalid",
"remediation": "Set PROWLER_AWS_BOTO3_CONNECT_TIMEOUT and PROWLER_AWS_BOTO3_READ_TIMEOUT to a positive integer number of seconds.",
},
(1919, "AWSInvalidBoto3RetriesError"): {
"message": "The Boto3 retries configured through the environment are invalid",
"remediation": "Set PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS to a non-negative integer, 0 disables retries.",
},
}
def __init__(self, code, file=None, original_exception=None, message=None):
@@ -244,3 +248,12 @@ class AWSInvalidBoto3TimeoutError(AWSBaseException):
super().__init__(
1918, file=file, original_exception=original_exception, message=message
)
class AWSInvalidBoto3RetriesError(AWSBaseException):
"""Boto3 retries configured through the environment are not a non-negative integer."""
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
1919, file=file, original_exception=original_exception, message=message
)
+1 -1
View File
@@ -85,7 +85,7 @@ class S3:
aws_access_key_id: str = None,
aws_secret_access_key: str = None,
aws_session_token: Optional[str] = None,
retries_max_attempts: int = 3,
retries_max_attempts: Optional[int] = None,
regions: set = set(),
) -> None:
"""
@@ -106,7 +106,7 @@ class SecurityHub:
aws_access_key_id: str = None,
aws_secret_access_key: str = None,
aws_session_token: Optional[str] = None,
retries_max_attempts: int = 3,
retries_max_attempts: Optional[int] = None,
regions: set = set(),
) -> "SecurityHub":
"""
@@ -40,7 +40,7 @@ class AwsSetUpSession:
aws_access_key_id: str = None,
aws_secret_access_key: str = None,
aws_session_token: Optional[str] = None,
retries_max_attempts: int = 3,
retries_max_attempts: Optional[int] = None,
regions: set = set(),
connect_timeout: Optional[int] = None,
read_timeout: Optional[int] = None,
@@ -106,6 +106,8 @@ class AwsSetUpSession:
session=self._session.current_session,
aws_region=sts_region,
)
# Later STS calls go where validation got an answer, not where it timed out
sts_region = caller_identity.region
logger.info("Credentials validated")
########
+2 -2
View File
@@ -144,7 +144,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"}
name = "prowler"
readme = "README.md"
requires-python = ">=3.10,<3.14"
version = "5.43.0"
version = "5.44.0"
[project.scripts]
prowler = "prowler.__main__:prowler"
@@ -214,7 +214,7 @@ constraint-dependencies = [
"aliyun-log-fastpb==0.3.0",
"annotated-types==0.7.0",
"antlr4-python3-runtime==4.13.2",
"anyio==4.13.0",
"anyio==4.14.2",
"apscheduler==3.11.2",
"astroid==3.3.11",
"async-timeout==5.0.1",
+258
View File
@@ -28,15 +28,19 @@ from prowler.providers.aws.config import (
AWS_STS_GLOBAL_ENDPOINT_REGION,
BOTO3_CONNECT_TIMEOUT,
BOTO3_READ_TIMEOUT,
BOTO3_RETRIES_MAX_ATTEMPTS,
BOTO3_USER_AGENT_EXTRA,
ROLE_SESSION_NAME,
get_boto3_retries_from_env,
get_boto3_timeout_from_env,
get_default_session_config,
)
from prowler.providers.aws.exceptions.exceptions import (
AWSAccessKeyIDInvalidError,
AWSArgumentTypeValidationError,
AWSAssumeRoleError,
AWSIAMRoleARNInvalidResourceTypeError,
AWSInvalidBoto3RetriesError,
AWSInvalidBoto3TimeoutError,
AWSInvalidPartitionError,
AWSInvalidProviderIdError,
@@ -1845,6 +1849,143 @@ aws:
]
assert isinstance(credentials, AWSCredentials)
assert credentials.aws_access_key_id == "AKIAIOSFODNN7EXAMPLE"
# Refreshing the credentials later goes straight to the region that answered
assert assumed_role_info.sts_region == AWS_REGION_GOV_CLOUD_US_WEST_1
def test_assume_role_does_not_retry_a_credential_error(self, monkeypatch):
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
current_session = session.Session(region_name=AWS_REGION_US_EAST_1)
attempted_regions = []
def create_sts_session(session, aws_region):
attempted_regions.append(aws_region)
sts_client = mock.MagicMock()
sts_client.assume_role.side_effect = botocore.exceptions.ClientError(
{"Error": {"Code": "AccessDenied", "Message": "denied"}},
"AssumeRole",
)
return sts_client
assumed_role_info = AWSAssumeRoleInfo(
role_arn=ARN(
arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role"
),
session_duration=3600,
external_id=None,
mfa_enabled=False,
role_session_name=ROLE_SESSION_NAME,
sts_region=AWS_REGION_GOV_CLOUD_US_EAST_1,
)
with patch(
"prowler.providers.aws.aws_provider.AwsProvider.create_sts_session",
new=create_sts_session,
):
with raises(AWSAssumeRoleError):
AwsProvider.assume_role(current_session, assumed_role_info)
assert attempted_regions == [AWS_REGION_GOV_CLOUD_US_EAST_1]
assert assumed_role_info.sts_region == AWS_REGION_GOV_CLOUD_US_EAST_1
def test_test_connection_role_validates_where_the_role_was_assumed(
self, monkeypatch
):
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
attempted_calls = []
def create_sts_session(session, aws_region):
if aws_region == AWS_REGION_GOV_CLOUD_US_EAST_1:
attempted_calls.append(aws_region)
raise botocore.exceptions.ConnectTimeoutError(
endpoint_url=f"https://sts.{aws_region}.amazonaws.com"
)
sts_client = mock.MagicMock()
def assume_role(**_):
attempted_calls.append(("AssumeRole", aws_region))
return {
"Credentials": {
"AccessKeyId": "AKIAIOSFODNN7EXAMPLE",
"SecretAccessKey": "secret",
"SessionToken": "token",
"Expiration": datetime.now() + timedelta(seconds=3600),
}
}
def get_caller_identity():
attempted_calls.append(("GetCallerIdentity", aws_region))
return {
"UserId": "test-user-id",
"Account": AWS_ACCOUNT_NUMBER,
"Arn": AWS_GOV_CLOUD_ACCOUNT_ARN,
}
sts_client.assume_role.side_effect = assume_role
sts_client.get_caller_identity.side_effect = get_caller_identity
return sts_client
with patch(
"prowler.providers.aws.aws_provider.AwsProvider.create_sts_session",
new=create_sts_session,
):
connection = AwsProvider.test_connection(
role_arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role",
aws_access_key_id="test-access-key",
aws_secret_access_key="test-secret-key",
raise_on_exception=False,
)
assert connection.is_connected
# The unreachable region is paid for once, not again for the validation
assert attempted_calls == [
AWS_REGION_GOV_CLOUD_US_EAST_1,
("AssumeRole", AWS_REGION_GOV_CLOUD_US_WEST_1),
("GetCallerIdentity", AWS_REGION_GOV_CLOUD_US_WEST_1),
]
@mock_aws
def test_aws_set_up_session_assumes_the_role_where_validation_got_an_answer(
self, monkeypatch
):
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
monkeypatch.setenv("AWS_DEFAULT_REGION", AWS_REGION_US_EAST_1)
answered = AWSCallerIdentity(
user_id="test-user-id",
account=AWS_ACCOUNT_NUMBER,
arn=ARN(AWS_GOV_CLOUD_ACCOUNT_ARN),
region=AWS_REGION_GOV_CLOUD_US_WEST_1,
)
sts_regions = []
class RoleAssumed(Exception):
pass
def assume_role(session, assumed_role_info):
sts_regions.append(assumed_role_info.sts_region)
raise RoleAssumed
with (
patch(
"prowler.providers.aws.aws_provider.AwsProvider.validate_credentials",
return_value=answered,
),
patch(
"prowler.providers.aws.aws_provider.AwsProvider.assume_role",
side_effect=assume_role,
),
):
with raises(RoleAssumed):
AwsSetUpSession(
role_arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role",
session_duration=900,
external_id="test-external-id",
role_session_name=ROLE_SESSION_NAME,
aws_access_key_id="testing",
aws_secret_access_key="testing",
)
assert sts_regions == [AWS_REGION_GOV_CLOUD_US_WEST_1]
def test_setup_session_mfa_falls_back_to_the_next_partition_region(
self, monkeypatch
@@ -3352,6 +3493,123 @@ aws:
):
get_boto3_timeout_from_env("PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", 10)
def test_get_default_session_config_retries_from_env(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
):
config = get_default_session_config()
assert config.retries == {"max_attempts": 1, "mode": "standard"}
def test_get_default_session_config_retries_from_env_0_disables_retries(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
):
config = get_default_session_config()
assert config.retries == {"max_attempts": 0, "mode": "standard"}
def test_set_session_config_argument_overrides_env_retries(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
):
config = AwsProvider.set_session_config(5)
assert config.retries == {"max_attempts": 5, "mode": "standard"}
@mock_aws
def test_aws_provider_without_retries_argument_uses_env_retries(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
):
aws_provider = AwsProvider()
client = aws_provider.session.current_session.client(
"ec2", region_name=AWS_REGION_US_EAST_1
)
# botocore rewrites max_attempts into total_max_attempts (retries + 1)
assert client.meta.config.retries["total_max_attempts"] == 1
@mock_aws
def test_aws_provider_retries_argument_overrides_env_retries(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
):
aws_provider = AwsProvider(retries_max_attempts=7)
client = aws_provider.session.current_session.client(
"ec2", region_name=AWS_REGION_US_EAST_1
)
assert client.meta.config.retries["total_max_attempts"] == 8
@mock_aws
def test_aws_set_up_session_without_retries_argument_uses_env_retries(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
):
aws_session = AwsSetUpSession(
aws_access_key_id="testing",
aws_secret_access_key="testing",
)
client = aws_session._session.current_session.client(
"ec2", region_name=AWS_REGION_US_EAST_1
)
assert client.meta.config.retries["total_max_attempts"] == 2
def test_test_connection_session_uses_env_retries(self):
with (
mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
),
mock.patch.object(
AwsProvider,
"validate_credentials",
return_value=AWSCallerIdentity(
user_id="test-user-id",
account=AWS_ACCOUNT_NUMBER,
arn=ARN(AWS_ACCOUNT_ARN),
region=AWS_REGION_US_EAST_1,
),
) as mock_validate_credentials,
):
connection = AwsProvider.test_connection(
aws_access_key_id="test-access-key",
aws_secret_access_key="test-secret-key",
raise_on_exception=False,
)
assert connection.is_connected
validated_session = mock_validate_credentials.call_args.args[0]
assert validated_session._session.get_default_client_config().retries == {
"max_attempts": 0,
"mode": "standard",
}
@pytest.mark.parametrize("raw", ["-1", "three", "1.5"])
def test_get_boto3_retries_from_env_rejects_anything_but_non_negative_integers(
self, raw
):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": raw}
):
with raises(
AWSInvalidBoto3RetriesError,
match="PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS",
):
get_boto3_retries_from_env("PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", 3)
def test_get_boto3_retries_from_env_blank_falls_back_to_default(self):
with mock.patch.dict(
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": " "}
):
assert (
get_boto3_retries_from_env(
"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", BOTO3_RETRIES_MAX_ATTEMPTS
)
== BOTO3_RETRIES_MAX_ATTEMPTS
)
def test_get_boto3_timeout_from_env_blank_falls_back_to_default(self):
with mock.patch.dict(os.environ, {"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": " "}):
assert (
+18
View File
@@ -4,6 +4,24 @@ All notable changes to the **Prowler UI** are documented in this file.
<!-- changelog: release notes start -->
## [1.43.0] (Prowler v5.43.0)
### 🚀 Added
- Registry marketplace and external provider onboarding for Private Cloud, with permission-based access independent of billing, confirmed artifact installation, schema-driven credentials, connection checks, and scan launch [(#12494)](https://github.com/prowler-cloud/prowler/pull/12494)
- AWS Marketplace button variant with outlined styling for light and dark themes [(#12803)](https://github.com/prowler-cloud/prowler/pull/12803)
- `UI_SELF_REGISTRATION_ENABLED` flag for Prowler Private Cloud deployments; when `"false"`, `/sign-up` only opens with an invitation, the sign-in page drops the "Sign up" link and the profile hides **Create organization** [(#12815)](https://github.com/prowler-cloud/prowler/pull/12815)
- "Invite your team" step offered once after the first provider is connected, before the onboarding checkpoint, reusing the invitation form tagged with `source=onboarding` [(#12819)](https://github.com/prowler-cloud/prowler/pull/12819)
### 🐞 Fixed
- Automatic onboarding stays hidden on billing pages and remains available after leaving billing [(#12803)](https://github.com/prowler-cloud/prowler/pull/12803)
- Per-provider breakdown and OCSF download for FedRAMP 20x KSI and Class C FRR in the cross-provider compliance view [(#12810)](https://github.com/prowler-cloud/prowler/pull/12810)
- Edit and Revoke actions are disabled for expired and revoked invitations [(#12831)](https://github.com/prowler-cloud/prowler/pull/12831)
- Cloudflare API token links in the provider wizard request the SSL and Certificates, Bot Management and Zone WAF read permissions the scan needs [(#12842)](https://github.com/prowler-cloud/prowler/pull/12842)
---
## [1.42.0] (Prowler v5.42.0)
### 🚀 Added
@@ -302,6 +302,7 @@ export const handlersForOrganizations = (
organizations.map((o) => o.secretId).filter((id): id is string => !!id),
);
let orgSeq = 0;
let providerSeq = 0;
let secretSeq = 0;
/** Reads per connection task, so `executingPolls` can hold one task running. */
const connectionTaskReads = new Map<string, number>();
@@ -565,6 +566,37 @@ export const handlersForOrganizations = (
HttpResponse.json({ data: [], meta: collectionMeta(0) }),
),
// --- single-account connect (AWS one-step form) -----------------------
http.post(`${API}/providers`, async ({ request }) => {
const body = (await request.json()) as {
data: { attributes: { provider: string; uid: string; alias?: string } };
};
providerSeq += 1;
return HttpResponse.json(
{
data: {
id: `provider-created-${providerSeq}`,
type: "providers",
attributes: {
...body.data.attributes,
connection: { connected: false, last_checked_at: null },
},
},
},
{ status: 201 },
);
}),
http.post(`${API}/providers/secrets`, () => {
secretSeq += 1;
return HttpResponse.json(
{
data: { id: `secret-created-${secretSeq}`, type: "provider-secrets" },
},
{ status: 201 },
);
}),
// --- providers (uid resolution) + connection testing -----------------
http.get<{ id: string }>(`${API}/providers/:id`, ({ params }) => {
const provider = fx.providers.find((p) => p.id === params.id);
@@ -215,3 +215,72 @@ describe("adaptFindingsByResourceResponse — malformed input", () => {
);
});
});
describe("adaptFindingsByResourceResponse — provider id", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("should carry the provider id resolved through the scan include", () => {
// Given — scan.provider include path, as the drawer requests it
createDictMock.mockImplementation((type: string) => {
if (type === "scans") {
return {
"scan-1": {
id: "scan-1",
attributes: {},
relationships: { provider: { data: { id: "provider-1" } } },
},
};
}
if (type === "providers") {
return {
"provider-1": {
id: "provider-1",
attributes: { provider: "aws", alias: "prod", uid: "123" },
},
};
}
return {};
});
const input = {
data: {
id: "finding-1",
attributes: {
uid: "uid-1",
check_id: "s3_check",
status: "FAIL",
severity: "high",
check_metadata: {},
},
relationships: {
resources: { data: [] },
scan: { data: { id: "scan-1" } },
},
},
included: [],
};
// When
const [finding] = adaptFindingsByResourceResponse(input);
// Then — the partial-scan request needs the id, not only the uid
expect(finding.providerId).toBe("provider-1");
expect(finding.providerUid).toBe("123");
});
it("should leave the provider id empty when the scan is not included", () => {
createDictMock.mockReturnValue({});
const [finding] = adaptFindingsByResourceResponse({
data: {
id: "finding-1",
attributes: { uid: "uid-1", check_id: "s3_check", status: "FAIL" },
relationships: { resources: { data: [] }, scan: { data: null } },
},
});
expect(finding.providerId).toBe("");
});
});
@@ -64,6 +64,7 @@ export interface ResourceDrawerFinding {
resourceDetails: string | null;
resourceMetadata: Record<string, unknown> | string | null;
// Provider
providerId: string;
providerType: ProviderType;
providerAlias: string;
providerUid: string;
@@ -280,6 +281,7 @@ export function adaptFindingsByResourceResponse(
| null
| undefined) ?? null,
// Provider
providerId: providerRelId ?? "",
providerType: ((providerAttrs.provider as string | undefined) ||
"aws") as ProviderType,
providerAlias: (providerAttrs.alias as string | undefined) || "",
+87
View File
@@ -0,0 +1,87 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const { fetchMock, getAuthHeadersMock, revalidatePathMock } = vi.hoisted(
() => ({
fetchMock: vi.fn(),
getAuthHeadersMock: vi.fn(),
revalidatePathMock: vi.fn(),
}),
);
vi.mock("@/lib/helper", () => ({
apiBaseUrl: "https://api.test/api/v1",
getAuthHeaders: getAuthHeadersMock,
}));
vi.mock("next/cache", () => ({
revalidatePath: revalidatePathMock,
}));
import { createMuteRule } from "./mute-rules";
const NAME_CONFLICT_DETAIL = "A mute rule with this name already exists.";
const errorResponse = (contentType: string, body: string, status = 400) =>
new Response(body, {
status,
headers: { "Content-Type": contentType },
});
const nameConflictBody = JSON.stringify({
errors: [
{
detail: NAME_CONFLICT_DETAIL,
status: "400",
source: { pointer: "/data/attributes/name" },
code: "invalid",
},
],
});
const muteRuleFormData = () => {
const formData = new FormData();
formData.set("name", "Root account has a hardware MFA device enabled");
formData.set("reason", "Not our approach here with SSO");
formData.set("finding_ids", JSON.stringify(["finding-1"]));
return formData;
};
beforeEach(() => {
vi.clearAllMocks();
vi.stubGlobal("fetch", fetchMock);
vi.spyOn(console, "error").mockImplementation(() => {});
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
});
describe("createMuteRule", () => {
it("should return only the error detail for a JSON:API error response", async () => {
fetchMock.mockResolvedValue(
errorResponse("application/vnd.api+json", nameConflictBody),
);
const result = await createMuteRule(null, muteRuleFormData());
expect(result?.errors?.general).toBe(NAME_CONFLICT_DETAIL);
expect(revalidatePathMock).not.toHaveBeenCalled();
});
it("should return only the error detail for a plain JSON error response", async () => {
fetchMock.mockResolvedValue(
errorResponse("application/json", nameConflictBody),
);
const result = await createMuteRule(null, muteRuleFormData());
expect(result?.errors?.general).toBe(NAME_CONFLICT_DETAIL);
});
it("should return the response text for a non-JSON error response", async () => {
fetchMock.mockResolvedValue(
errorResponse("text/plain", "Bad gateway", 502),
);
const result = await createMuteRule(null, muteRuleFormData());
expect(result?.errors?.general).toBe("Bad gateway");
});
});
+2 -1
View File
@@ -156,7 +156,8 @@ export const createMuteRule = async (
let errorMessage = `Failed to create mute rule: ${response.statusText}`;
const responseContentType = response.headers.get("content-type");
try {
if (responseContentType?.includes("application/json")) {
// The API answers with application/vnd.api+json
if (responseContentType?.includes("json")) {
const errorData = await response.json();
const jsonApiError = (
errorData as {
+1 -1
View File
@@ -546,7 +546,7 @@ export const applyDiscovery = async (
);
// No `include`: the apply view rejects the parameter outright and fails the
// whole request. The created providers' uids are read afterwards instead, with
// `getProviderUidsByIds`.
// `getProviderUidsAndConnectionBaselines`.
const attributes = buildApplyAttributes(payload);
+91 -18
View File
@@ -151,28 +151,28 @@ export const getProvider = async (formData: FormData) => {
const PROVIDERS_PAGE_MAX = 100;
/**
* Uids of the given providers, keyed by provider id. A provider's `uid` is the
* candidate it was created for (AWS account id / GCP project id), so this is what
* matches an apply's created providers back to the selection. Batched with
* `filter[id__in]` rather than one `GET /providers/{id}` per id.
* Providers matching the given ids, batched with `filter[id__in]` (page size
* `PROVIDERS_PAGE_MAX`, the server max, which also bounds the id batch size)
* rather than one `GET /providers/{id}` per id. Shared by every action below
* that resolves providers by id; a batch that fails to fetch leaves its
* providers out of the result rather than failing the rest, and it is on each
* caller to say what "missing" means for its own map. Not exported: an
* exported function in this `"use server"` module becomes a callable server
* action.
*/
export const getProviderUidsByIds = async (
const fetchProvidersByIds = async (
providerIds: string[],
): Promise<Record<string, string>> => {
): Promise<ProvidersApiResponse["data"]> => {
const uniqueIds = Array.from(new Set(providerIds.filter(Boolean)));
if (uniqueIds.length === 0) {
return {};
return [];
}
const headers = await getAuthHeaders({ contentType: false });
const batches: string[][] = [];
const providers: ProvidersApiResponse["data"] = [];
for (let start = 0; start < uniqueIds.length; start += PROVIDERS_PAGE_MAX) {
batches.push(uniqueIds.slice(start, start + PROVIDERS_PAGE_MAX));
}
const uidById: Record<string, string> = {};
for (const batch of batches) {
const batch = uniqueIds.slice(start, start + PROVIDERS_PAGE_MAX);
const url = new URL(`${apiBaseUrl}/providers`);
url.searchParams.set("filter[id__in]", batch.join(","));
url.searchParams.set("page[size]", String(PROVIDERS_PAGE_MAX));
@@ -183,18 +183,91 @@ export const getProviderUidsByIds = async (
| ProvidersApiResponse
| undefined;
for (const provider of result?.data ?? []) {
providers.push(...(result?.data ?? []));
} catch {
// A failed batch leaves its providers out of the result rather than
// failing the rest.
}
}
return providers;
};
/**
* Uids of the given providers, keyed by provider id. A provider's `uid` is the
* candidate it was created for (AWS account id / GCP project id), so this is what
* matches an apply's created providers back to the selection.
*/
export const getProviderUidsByIds = async (
providerIds: string[],
): Promise<Record<string, string>> => {
const uidById: Record<string, string> = {};
for (const provider of await fetchProvidersByIds(providerIds)) {
const uid = provider?.attributes?.uid;
if (typeof provider?.id === "string" && typeof uid === "string") {
uidById[provider.id] = uid;
}
}
} catch {
// A failed batch leaves its providers unmapped rather than failing the rest.
return uidById;
};
/**
* `connection.last_checked_at` for each given provider, keyed by id. Read before
* a batch of connection checks is dispatched, so `resolveProviderConnectionState`
* can tell a check's own result apart from an older one already on record by
* comparing values, never by comparing the browser's clock against the server's
* (see that function for why). A provider missing from the response -- the batch
* read failed, or it was deleted mid-flight -- is left out of the map rather than
* defaulted, so callers can tell "no prior check" (`null`) from "unknown".
*/
export const getProviderConnectionBaselines = async (
providerIds: string[],
): Promise<Record<string, string | null>> => {
const baselineById: Record<string, string | null> = {};
for (const provider of await fetchProvidersByIds(providerIds)) {
if (typeof provider?.id === "string") {
baselineById[provider.id] =
provider.attributes?.connection?.last_checked_at ?? null;
}
}
return uidById;
return baselineById;
};
/**
* Uid and `connection.last_checked_at` for each given provider, keyed by id, read
* with a single batched `filter[id__in]` request. The organization onboarding
* apply step needs both right after creating providers: the uid to match each one
* back to the candidate it was created for (see `getProviderUidsByIds`), and the
* baseline to compare a dispatched check's result against (see
* `getProviderConnectionBaselines`). Reading them together avoids fetching the
* same set of providers twice.
*/
export const getProviderUidsAndConnectionBaselines = async (
providerIds: string[],
): Promise<{
uidById: Record<string, string>;
baselineById: Record<string, string | null>;
}> => {
const uidById: Record<string, string> = {};
const baselineById: Record<string, string | null> = {};
for (const provider of await fetchProvidersByIds(providerIds)) {
if (typeof provider?.id !== "string") {
continue;
}
const uid = provider.attributes?.uid;
if (typeof uid === "string") {
uidById[provider.id] = uid;
}
baselineById[provider.id] =
provider.attributes?.connection?.last_checked_at ?? null;
}
return { uidById, baselineById };
};
export const updateProvider = async (formData: FormData) => {
+14 -2
View File
@@ -24,6 +24,7 @@ const REGISTRY_TASK_PATH_PREFIX = "/api/v1/tasks/";
const REGISTRY_ERROR_CODE = {
KEY_REJECTED: "registry_key_rejected",
UNAVAILABLE: "registry_unavailable",
PAGE_NOT_FOUND: "registry_page_not_found",
} as const;
// Opposite remedies, so a 409 is never read without its code.
const REGISTRY_REMOVAL_CONFLICT_CODE = {
@@ -216,18 +217,29 @@ export async function classifyRegistryFailure(
return { status: REGISTRY_FAILURE.ERROR };
}
// An enabled backend also answers 404 (missing page): tell them apart by code.
export async function isRegistryDisabledResponse(response: Response) {
if (response.status !== 404) return false;
const codes = await getRegistryErrorCodes(response);
return !codes.includes(REGISTRY_ERROR_CODE.PAGE_NOT_FOUND);
}
function isRegistryDiscoveryEndpoint(endpoint: RegistryEndpoint) {
return registryDiscoveryEndpoints.has(endpoint);
}
async function getRegistryErrorCode(response: Response) {
async function getRegistryErrorCodes(response: Response) {
const parsed = errorDocumentSchema.safeParse(
await response
.clone()
.json()
.catch(() => undefined),
);
return parsed.success ? parsed.data.errors[0]?.code : undefined;
return parsed.success ? parsed.data.errors.map(({ code }) => code) : [];
}
async function getRegistryErrorCode(response: Response) {
return (await getRegistryErrorCodes(response))[0];
}
const REGISTRY_CATALOG_PAGE_SIZE = 100;
+43 -1
View File
@@ -119,14 +119,17 @@ describe("installed Registry provider discovery", () => {
emptyMetadata = false,
failedEndpoint,
failureStatus = 500,
failureBody = {},
}: {
emptyMetadata?: boolean;
failedEndpoint?: string;
failureStatus?: number;
failureBody?: unknown;
} = {}) {
fetchMock.mockImplementation((url: string) => {
const endpoint = new URL(url).pathname.split("/").pop();
if (endpoint === failedEndpoint) return jsonResponse({}, failureStatus);
if (endpoint === failedEndpoint)
return jsonResponse(failureBody, failureStatus);
if (endpoint === "available-artifacts")
return jsonResponse({
data: [
@@ -242,6 +245,45 @@ describe("installed Registry provider discovery", () => {
});
},
);
it.each(["available-artifacts", "artifacts", "providers"])(
"hides Registry when the backend has it disabled and %s answers 404",
async (failedEndpoint) => {
mockDiscovery({ failedEndpoint, failureStatus: 404 });
expect(await getInstalledRegistryProviderOptions()).toEqual({
status: "access_denied",
});
},
);
it.each(["available-artifacts", "providers"])(
"keeps Registry visible when an enabled backend answers 404 for a missing %s page",
async (failedEndpoint) => {
mockDiscovery({
failedEndpoint,
failureStatus: 404,
failureBody: {
errors: [{ status: "404", code: "registry_page_not_found" }],
},
});
expect(await getInstalledRegistryProviderOptions()).toEqual({
status: "error",
});
},
);
it("keeps Registry visible when the missing-page code is not the first error", async () => {
mockDiscovery({
failedEndpoint: "available-artifacts",
failureStatus: 404,
failureBody: {
errors: [{ code: "not_found" }, { code: "registry_page_not_found" }],
},
});
expect(await getInstalledRegistryProviderOptions()).toEqual({
status: "error",
});
});
});
describe("Registry guarded reads", () => {
+4
View File
@@ -47,6 +47,7 @@ import {
classifyRegistryRemovalConflict,
collectCompleteRegistryCatalog,
isRegistryCollection,
isRegistryDisabledResponse,
parseRegistryArtifactSubmission,
parseRegistryCredentialSubmission,
RegistryCatalogPageError,
@@ -86,6 +87,9 @@ async function readRegistryResponse(
return { status: REGISTRY_FAILURE.ERROR };
}
if (response.ok) return response;
// A backend with Registry disabled answers 404: hide it like a denial.
if (await isRegistryDisabledResponse(response))
return { status: REGISTRY_FAILURE.ACCESS_DENIED };
return endpoint === REGISTRY_ENDPOINT.PROVIDERS ||
endpoint === REGISTRY_ENDPOINT.AVAILABLE_ARTIFACTS
+112
View File
@@ -6,12 +6,14 @@ const {
getAuthHeadersMock,
handleApiErrorMock,
handleApiResponseMock,
isReportDownloadLockedMock,
} = vi.hoisted(() => ({
addScanOperationMock: vi.fn(),
fetchMock: vi.fn(),
getAuthHeadersMock: vi.fn(),
handleApiErrorMock: vi.fn(),
handleApiResponseMock: vi.fn(),
isReportDownloadLockedMock: vi.fn(),
}));
vi.mock("@/lib", () => ({
@@ -23,6 +25,10 @@ vi.mock("@/lib", () => ({
error instanceof Error ? error.message : String(error),
}));
vi.mock("next/cache", () => ({
revalidatePath: vi.fn(),
}));
vi.mock("@/lib/server-actions-helper", () => ({
handleApiError: handleApiErrorMock,
handleApiResponse: handleApiResponseMock,
@@ -32,7 +38,17 @@ vi.mock("@/lib/sentry-breadcrumbs", () => ({
addScanOperation: addScanOperationMock,
}));
vi.mock("@/lib/report-download-access", () => ({
REPORT_DOWNLOAD_LOCKED_ERROR:
"Report downloads require an active subscription.",
isReportDownloadLocked: isReportDownloadLockedMock,
}));
import {
createPartialScan,
getComplianceCsv,
getComplianceOcsf,
getCompliancePdfReport,
getExportsZip,
launchOrganizationScans,
scheduleOrganizationDailyScans,
@@ -156,6 +172,7 @@ describe("getExportsZip", () => {
vi.clearAllMocks();
vi.stubGlobal("fetch", fetchMock);
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
isReportDownloadLockedMock.mockResolvedValue(false);
});
it("returns a generic server error when the report endpoint returns HTML", async () => {
@@ -181,3 +198,98 @@ describe("getExportsZip", () => {
});
});
});
describe("report downloads for subscription-only tenants", () => {
beforeEach(() => {
vi.clearAllMocks();
vi.stubGlobal("fetch", fetchMock);
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
isReportDownloadLockedMock.mockResolvedValue(true);
});
it.each([
{ name: "scan ZIP", download: () => getExportsZip("scan-123") },
{
name: "compliance CSV",
download: () => getComplianceCsv("scan-123", "cis_2.0_aws"),
},
{
name: "compliance OCSF",
download: () => getComplianceOcsf("scan-123", "dora_aws"),
},
{
name: "compliance PDF",
download: () => getCompliancePdfReport("scan-123", "threatscore"),
},
])("rejects the $name without calling the API", async ({ download }) => {
// When
const result = await download();
// Then
expect(fetchMock).not.toHaveBeenCalled();
expect(result).toEqual({
error: "Report downloads require an active subscription.",
});
});
});
describe("createPartialScan", () => {
beforeEach(() => {
vi.clearAllMocks();
vi.stubGlobal("fetch", fetchMock);
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
fetchMock.mockResolvedValue(new Response(null, { status: 202 }));
handleApiResponseMock.mockResolvedValue({ data: { id: "scan-1" } });
});
it("posts the resource uids as a scan of one provider", async () => {
// When
const result = await createPartialScan({
providerId: "provider-1",
resourceUids: ["arn:aws:s3:::bucket"],
});
// Then
expect(fetchMock).toHaveBeenCalledWith(
"https://api.example.com/api/v1/scans",
expect.objectContaining({
method: "POST",
body: JSON.stringify({
data: {
type: "scans",
attributes: { resource_uids: ["arn:aws:s3:::bucket"] },
relationships: {
provider: { data: { type: "providers", id: "provider-1" } },
},
},
}),
}),
);
expect(handleApiResponseMock).toHaveBeenCalledWith(
expect.any(Response),
"/scans",
);
expect(result).toEqual({ data: { id: "scan-1" } });
expect(addScanOperationMock).toHaveBeenCalledWith("start", "scan-1");
});
it("refuses more resources than the API accepts without calling it", async () => {
// Given — the API caps a partial scan at 10 resources.
const resourceUids = Array.from(
{ length: 11 },
(_, index) => `arn:aws:s3:::bucket-${index}`,
);
// When
const result = await createPartialScan({
providerId: "provider-1",
resourceUids,
});
// Then
expect(fetchMock).not.toHaveBeenCalled();
expect(result).toEqual({
error: "Select between 1 and 10 resources to re-check",
});
});
});
+77
View File
@@ -15,9 +15,14 @@ import {
} from "@/lib/compliance/compliance-report-types";
import { runWithConcurrencyLimit } from "@/lib/concurrency";
import { appendSanitizedProviderTypeFilters } from "@/lib/provider-filters";
import {
isReportDownloadLocked,
REPORT_DOWNLOAD_LOCKED_ERROR,
} from "@/lib/report-download-access";
import { addScanOperation } from "@/lib/sentry-breadcrumbs";
import { handleApiError, handleApiResponse } from "@/lib/server-actions-helper";
import { SCAN_STATES } from "@/types/attack-paths";
import { PARTIAL_SCAN_MAX_RESOURCES } from "@/types/partial-scans";
const ORGANIZATION_SCAN_CONCURRENCY_LIMIT = 5;
@@ -182,6 +187,70 @@ export const scanOnDemand = async (formData: FormData) => {
}
};
/** Prowler Cloud only: re-check up to PARTIAL_SCAN_MAX_RESOURCES resources of one provider. */
export const createPartialScan = async ({
providerId,
resourceUids,
}: {
providerId: string;
resourceUids: string[];
}) => {
if (!providerId) {
return { error: "Provider ID is required" };
}
if (
resourceUids.length === 0 ||
resourceUids.length > PARTIAL_SCAN_MAX_RESOURCES
) {
return {
error: `Select between 1 and ${PARTIAL_SCAN_MAX_RESOURCES} resources to re-check`,
};
}
const headers = await getAuthHeaders({ contentType: true });
addScanOperation("create", undefined, {
provider_id: providerId,
partial: true,
resource_count: resourceUids.length,
});
const url = new URL(`${apiBaseUrl}/scans`);
try {
const requestBody = {
data: {
type: "scans",
attributes: { resource_uids: resourceUids },
relationships: {
provider: {
data: {
type: "providers",
id: providerId,
},
},
},
},
};
const response = await fetch(url.toString(), {
method: "POST",
headers,
body: JSON.stringify(requestBody),
});
const result = await handleApiResponse(response, "/scans");
if (result?.data?.id) {
addScanOperation("start", result.data.id);
revalidatePath("/scans");
}
return result;
} catch (error) {
addScanOperation("create");
return handleApiError(error);
}
};
export const scheduleDaily = async (formData: FormData) => {
const headers = await getAuthHeaders({ contentType: true });
@@ -377,6 +446,10 @@ export const updateScan = async (formData: FormData) => {
};
export const getExportsZip = async (scanId: string) => {
if (await isReportDownloadLocked()) {
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
}
const headers = await getAuthHeaders({ contentType: false });
const url = new URL(`${apiBaseUrl}/scans/${scanId}/report`);
@@ -457,6 +530,10 @@ const _fetchScanBinary = async (
filename: string,
errorLabel: string,
): Promise<ScanBinaryResult> => {
if (await isReportDownloadLocked()) {
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
}
const headers = await getAuthHeaders({ contentType: false });
const url = new URL(`${apiBaseUrl}/scans/${scanId}/${urlPath}`);
@@ -38,6 +38,7 @@ import {
} from "@/lib/compliance/compliance-report-types";
import { LIGHTHOUSE_COMPLIANCE_CONTEXT_MODE } from "@/lib/lighthouse/context/constants";
import { buildComplianceContext } from "@/lib/lighthouse/context/contributions";
import { isReportDownloadLocked } from "@/lib/report-download-access";
import { isCloud } from "@/lib/shared/env";
import { cn } from "@/lib/utils";
import type { SearchParamsProps } from "@/types";
@@ -77,6 +78,8 @@ export default async function ComplianceDetail({
notFound();
}
const subscriptionOnlyPromise = isReportDownloadLocked();
// Cross-provider mode replaces the per-scan pipeline with the universal
// roll-up view. Prowler Cloud-only: the OSS API has no such endpoint, so
// the route is blocked in OSS the same way the compliance tab is.
@@ -85,6 +88,7 @@ export default async function ComplianceDetail({
redirect("/compliance");
}
const subscriptionOnly = await subscriptionOnlyPromise;
return (
<Suspense
key={buildSearchParamsKey(resolvedSearchParams)}
@@ -105,6 +109,7 @@ export default async function ComplianceDetail({
complianceId={complianceId}
searchParams={resolvedSearchParams}
targetSection={section}
subscriptionOnly={subscriptionOnly}
/>
</Suspense>
);
@@ -124,6 +129,7 @@ export default async function ComplianceDetail({
}
const crossAccountTitle = compliancetitle.split("-").join(" ");
const subscriptionOnly = await subscriptionOnlyPromise;
return (
<ContentLayout
title={
@@ -148,6 +154,7 @@ export default async function ComplianceDetail({
providerType={providerType}
searchParams={resolvedSearchParams}
targetSection={section}
subscriptionOnly={subscriptionOnly}
/>
</Suspense>
</ContentLayout>
@@ -172,8 +179,12 @@ export default async function ComplianceDetail({
let selectedScan: ScanEntity | null = null;
const selectedScanId = scanId || null;
const [metadataInfoData, attributesData, selectedScanResponse] =
await Promise.all([
const [
metadataInfoData,
attributesData,
selectedScanResponse,
subscriptionOnly,
] = await Promise.all([
getComplianceOverviewMetadataInfo({
filters: {
"filter[scan_id]": selectedScanId ?? undefined,
@@ -183,6 +194,7 @@ export default async function ComplianceDetail({
selectedScanId
? getScan(selectedScanId, { include: "provider" })
: Promise.resolve(null),
subscriptionOnlyPromise,
]);
// The compliance catalog is still warming after a deploy/restart. Show the
@@ -309,6 +321,7 @@ export default async function ComplianceDetail({
complianceId,
latestCisIds.has(complianceId),
)}
subscriptionOnly={subscriptionOnly}
/>
</div>
)}
@@ -5,11 +5,13 @@ const {
getAuthHeadersMock,
handleApiResponseMock,
captureExceptionMock,
isReportDownloadLockedMock,
} = vi.hoisted(() => ({
fetchMock: vi.fn(),
getAuthHeadersMock: vi.fn(),
handleApiResponseMock: vi.fn(),
captureExceptionMock: vi.fn(),
isReportDownloadLockedMock: vi.fn(),
}));
vi.mock("@/lib", () => ({
@@ -28,6 +30,12 @@ vi.mock("@sentry/nextjs", () => ({
captureException: captureExceptionMock,
}));
vi.mock("@/lib/report-download-access", () => ({
REPORT_DOWNLOAD_LOCKED_ERROR:
"Report downloads require an active subscription.",
isReportDownloadLocked: isReportDownloadLockedMock,
}));
import {
generateCrossProviderPdf,
getCrossProviderComplianceOverview,
@@ -56,6 +64,29 @@ beforeEach(() => {
Authorization: "Bearer test-token",
});
handleApiResponseMock.mockResolvedValue({ data: null });
isReportDownloadLockedMock.mockResolvedValue(false);
});
describe("cross-provider PDF reports for subscription-only tenants", () => {
it.each([
{
name: "generation",
run: () => generateCrossProviderPdf({ complianceId: "csa_ccm_4.0" }),
},
{ name: "download", run: () => getCrossProviderPdfBinary("task-1") },
])("rejects the $name without calling the API", async ({ run }) => {
// Given
isReportDownloadLockedMock.mockResolvedValue(true);
// When
const result = await run();
// Then
expect(fetchMock).not.toHaveBeenCalled();
expect(result).toEqual({
error: "Report downloads require an active subscription.",
});
});
});
describe("getCrossProviderComplianceOverview", () => {
@@ -48,6 +48,7 @@ interface CrossAccountDetailProps {
providerType: KnownProviderType;
searchParams: Record<string, string | string[] | undefined>;
targetSection?: string;
subscriptionOnly?: boolean;
}
/**
@@ -63,6 +64,7 @@ export const CrossAccountDetail = async ({
providerType,
searchParams,
targetSection,
subscriptionOnly = false,
}: CrossAccountDetailProps) => {
const filters = parseCrossAccountFilters(searchParams);
@@ -205,6 +207,7 @@ export const CrossAccountDetail = async ({
providerType={providerType}
filters={{ ...filters, scanIds: attrs.scan_ids }}
latestPdf={latestPdf}
subscriptionOnly={subscriptionOnly}
/>
}
filters={
@@ -44,6 +44,7 @@ interface CrossProviderDetailProps {
complianceId: string;
searchParams: Record<string, string | string[] | undefined>;
targetSection?: string;
subscriptionOnly?: boolean;
}
/**
@@ -57,6 +58,7 @@ export const CrossProviderDetail = async ({
complianceId,
searchParams,
targetSection,
subscriptionOnly = false,
}: CrossProviderDetailProps) => {
const filters = parseCrossProviderFilters(searchParams);
@@ -206,6 +208,7 @@ export const CrossProviderDetail = async ({
complianceId={complianceId}
filters={{ ...filters, scanIds: attrs.scan_ids }}
latestPdf={latestPdf}
subscriptionOnly={subscriptionOnly}
/>
}
filters={
@@ -2,6 +2,9 @@ import { render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { useCloudUpgradeStore } from "@/store/cloud-upgrade/store";
import { PAID_PLAN_UPGRADE_FEATURE } from "@/types/cloud-upgrade";
import { CrossProviderPdfButton } from "./cross-provider-pdf-button";
// Radix dialogs/dropdowns rely on pointer-capture and scrollIntoView, which
@@ -99,6 +102,7 @@ describe("CrossProviderPdfButton", () => {
vi.clearAllMocks();
storeState.tasks = {};
generatePdfMock.mockResolvedValue({ taskId: "task-1" });
useCloudUpgradeStore.getState().closeCloudUpgrade();
});
const openGenerateModal = async (
@@ -192,6 +196,34 @@ describe("CrossProviderPdfButton", () => {
await waitFor(() => expect(downloadPdfMock).toHaveBeenCalledWith("task-7"));
});
it.each([/download latest/i, /generate new report/i])(
"opens the paid plan upgrade instead of %s for subscription-only tenants",
async (label) => {
// Given
const user = userEvent.setup();
render(
<CrossProviderPdfButton
{...props}
latestPdf={{ taskId: "task-7", filename: "csa-latest.pdf" }}
subscriptionOnly
/>,
);
// When
await user.click(screen.getByRole("button", { name: /report/i }));
await user.click(await screen.findByRole("menuitem", { name: label }));
// Then
expect(downloadPdfMock).not.toHaveBeenCalled();
expect(
screen.queryByRole("dialog", { name: /generate/i }),
).not.toBeInTheDocument();
expect(useCloudUpgradeStore.getState().activeFeature).toBe(
PAID_PLAN_UPGRADE_FEATURE.REPORT_DOWNLOAD,
);
},
);
it("keeps a completed report downloadable after the ready toast closes", async () => {
// Given
storeState.tasks = {
@@ -12,6 +12,7 @@ import { FormButtons } from "@/components/shadcn/form";
import { Input } from "@/components/shadcn/input/input";
import { Modal } from "@/components/shadcn/modal";
import { toast } from "@/components/shadcn/toast";
import { useReportDownload } from "@/hooks/use-report-download";
import {
TASK_WATCHER_STATUS,
trackAndPollTask,
@@ -47,6 +48,8 @@ interface CrossProviderPdfButtonProps {
/** Already-generated report matching these filters, if any — offered as an
* instant download instead of forcing a re-generate. */
latestPdf: LatestCrossProviderPdf | null;
/** Prowler Cloud tenants without a paid plan cannot download reports. */
subscriptionOnly?: boolean;
}
export const CrossProviderPdfButton = ({
@@ -54,7 +57,9 @@ export const CrossProviderPdfButton = ({
providerType,
filters,
latestPdf,
subscriptionOnly = false,
}: CrossProviderPdfButtonProps) => {
const runReportDownload = useReportDownload(subscriptionOnly);
const [dialogOpen, setDialogOpen] = useState(false);
const [reportName, setReportName] = useState("");
const [submitting, setSubmitting] = useState(false);
@@ -185,13 +190,15 @@ export const CrossProviderPdfButton = ({
icon={<DownloadIcon />}
label={`Download latest${formatGeneratedAt(availablePdf.completedAt)}`}
description={availablePdf.filename}
onSelect={() => downloadPdf(availablePdf.taskId)}
onSelect={() =>
runReportDownload(() => downloadPdf(availablePdf.taskId))
}
/>
)}
<ActionDropdownItem
icon={<FileTextIcon />}
label="Generate new report…"
onSelect={() => setDialogOpen(true)}
onSelect={() => runReportDownload(() => setDialogOpen(true))}
/>
</ActionDropdown>
)}
@@ -7,6 +7,10 @@ import {
getErrorMessage,
} from "@/lib";
import { hasActionError, type ActionErrorResult } from "@/lib/action-errors";
import {
isReportDownloadLocked,
REPORT_DOWNLOAD_LOCKED_ERROR,
} from "@/lib/report-download-access";
import { handleApiResponse } from "@/lib/server-actions-helper";
import { SentryErrorSource, SentryErrorType } from "@/sentry";
@@ -171,6 +175,10 @@ export const generateAggregatedCompliancePdf = async (
url: URL,
operation: string,
): Promise<{ taskId: string } | { error: string }> => {
if (await isReportDownloadLocked()) {
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
}
const headers = await getAuthHeaders({ contentType: false });
try {
@@ -211,6 +219,10 @@ export const getAggregatedCompliancePdfBinary = async ({
operation: string;
defaultFilename: string;
}): Promise<ScanBinaryResult> => {
if (await isReportDownloadLocked()) {
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
}
const headers = await getAuthHeaders({ contentType: false });
try {
+146 -2
View File
@@ -15,18 +15,24 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import Compliance from "./page";
const {
complianceFiltersSpy,
complianceOverviewGridSpy,
getComplianceOverviewMetadataInfoMock,
getCompliancesOverviewMock,
getScanMock,
getScansMock,
getThreatScoreMock,
isCloudMock,
loadComplianceWatchlistContextMock,
} = vi.hoisted(() => ({
complianceFiltersSpy: vi.fn(),
complianceOverviewGridSpy: vi.fn(),
getComplianceOverviewMetadataInfoMock: vi.fn(),
getCompliancesOverviewMock: vi.fn(),
getScanMock: vi.fn(),
getScansMock: vi.fn(),
getThreatScoreMock: vi.fn(),
isCloudMock: vi.fn(() => false),
loadComplianceWatchlistContextMock: vi.fn(),
}));
@@ -41,12 +47,13 @@ vi.mock("@/actions/overview", () => ({
}));
vi.mock("@/actions/scans", () => ({
getScan: getScanMock,
getScans: getScansMock,
getScansByState: vi.fn(),
}));
vi.mock("@/lib/shared/env", () => ({
isCloud: () => false,
isCloud: isCloudMock,
}));
vi.mock("./_lib/watchlist-context", () => ({
@@ -83,7 +90,10 @@ vi.mock("@/components/compliance", () => ({
}));
vi.mock("@/components/compliance/compliance-header/compliance-filters", () => ({
ComplianceFilters: () => <div>Compliance filters</div>,
ComplianceFilters: (props: { scans: Array<{ id: string }> }) => {
complianceFiltersSpy(props);
return <div>Compliance filters</div>;
},
}));
vi.mock("@/components/compliance/compliance-overview-grid", () => ({
@@ -148,6 +158,8 @@ describe("Compliance overview page", () => {
describe("Compliance overview task response", () => {
beforeEach(() => {
vi.clearAllMocks();
isCloudMock.mockReturnValue(false);
getScanMock.mockResolvedValue(undefined);
getScansMock.mockResolvedValue({
data: [
{
@@ -233,6 +245,138 @@ describe("Compliance overview task response", () => {
);
});
it("keeps partial scans out of the per-scan selector", async () => {
// Given - a Cloud partial scan among the completed scans; it computes no
// compliance, so selecting it would show nothing and its downloads fail
isCloudMock.mockReturnValue(true);
getScansMock.mockResolvedValue({
data: [
{
id: "scan-1",
attributes: {
name: "Production scan",
completed_at: "2026-08-05T17:00:00Z",
},
relationships: { provider: { data: { id: "provider-1" } } },
},
{
id: "scan-partial",
attributes: {
name: "Re-check",
completed_at: "2026-08-06T09:00:00Z",
is_partial: true,
},
relationships: { provider: { data: { id: "provider-1" } } },
},
],
included: [
{
type: "providers",
id: "provider-1",
attributes: { provider: "aws", uid: "123456789012", alias: "prod" },
},
],
});
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
// When
const page = await Compliance({
searchParams: Promise.resolve({ scanId: "scan-1" }),
});
render(page as ReactElement);
// Then - only the full scan reaches the selector, and the API is asked
// for the flag that tells them apart
expect(complianceFiltersSpy).toHaveBeenCalledWith(
expect.objectContaining({
scans: [expect.objectContaining({ id: "scan-1" })],
}),
);
expect(getScansMock).toHaveBeenCalledWith(
expect.objectContaining({
// Filtered at the API too, so a page full of re-checks cannot hide
// the full scans behind it.
filters: expect.objectContaining({ "filter[is_partial]": "false" }),
fields: { scans: "name,completed_at,provider,is_partial" },
}),
);
});
it("does not send the Cloud-only partial filter outside Prowler Cloud", async () => {
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
await Compliance({ searchParams: Promise.resolve({ scanId: "scan-1" }) });
expect(getScansMock).toHaveBeenCalledWith(
expect.objectContaining({
filters: { "filter[state]": "completed" },
}),
);
});
it("falls back to the first full scan when the URL names a partial scan", async () => {
// Given - a stale link to a partial scan, absent from the eligible list
getScanMock.mockResolvedValue({
data: { id: "scan-partial", attributes: { is_partial: true } },
});
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
// When
const page = await Compliance({
searchParams: Promise.resolve({ scanId: "scan-partial" }),
});
render(page as ReactElement);
// Then - the page selects a scan that has compliance instead
expect(getScanMock).toHaveBeenCalledWith("scan-partial");
expect(complianceFiltersSpy).toHaveBeenCalledWith(
expect.objectContaining({ selectedScanId: "scan-1" }),
);
expect(getCompliancesOverviewMock).toHaveBeenCalledWith(
expect.objectContaining({ scanId: "scan-1" }),
);
});
it("falls back to the first full scan when the URL scan cannot be found", async () => {
// Given - a deleted or mistyped id: the lookup returns an error, no data
getScanMock.mockResolvedValue({ error: "Not found", status: 404 });
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
// When
const page = await Compliance({
searchParams: Promise.resolve({ scanId: "scan-missing" }),
});
render(page as ReactElement);
// Then - no compliance request goes out for an id that does not exist
expect(complianceFiltersSpy).toHaveBeenCalledWith(
expect.objectContaining({ selectedScanId: "scan-1" }),
);
expect(getCompliancesOverviewMock).not.toHaveBeenCalledWith(
expect.objectContaining({ scanId: "scan-missing" }),
);
});
it("keeps trusting a URL scan id that is older than the listed page", async () => {
// Given - a full scan beyond the first page, shaped like an OSS response
// that carries no is_partial field at all
getScanMock.mockResolvedValue({
data: { id: "scan-old", attributes: { name: "Old scan" } },
});
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
// When
const page = await Compliance({
searchParams: Promise.resolve({ scanId: "scan-old" }),
});
render(page as ReactElement);
// Then
expect(complianceFiltersSpy).toHaveBeenCalledWith(
expect.objectContaining({ selectedScanId: "scan-old" }),
);
});
it("shows the invalid scan message for a JSON:API error response", async () => {
// Given - handleApiResponse converted a client error to its error result
getCompliancesOverviewMock.mockResolvedValue({
+48 -5
View File
@@ -7,7 +7,7 @@ import {
getCompliancesOverview,
} from "@/actions/compliances";
import { getThreatScore } from "@/actions/overview";
import { getScans, getScansByState } from "@/actions/scans";
import { getScan, getScans, getScansByState } from "@/actions/scans";
import {
ComplianceSkeletonGrid,
NoScansAvailable,
@@ -20,6 +20,7 @@ import { Alert, AlertDescription } from "@/components/shadcn/alert";
import { Card, CardContent } from "@/components/shadcn/card/card";
import { ContentLayout } from "@/components/shadcn/content-layout";
import { pickLatestCisPerProvider } from "@/lib/compliance/compliance-report-types";
import { isReportDownloadLocked } from "@/lib/report-download-access";
import { isCloud } from "@/lib/shared/env";
import {
ExpandedScanData,
@@ -40,6 +41,31 @@ import {
import type { ComplianceWatchlistContext } from "./_lib/watchlist-context";
import { loadComplianceWatchlistContext } from "./_lib/watchlist-context";
/**
* A scan id from the URL is trusted when it is listed, or when a single lookup
* returns a scan that is not partial (older full scans keep working). A partial
* scan, reached through a stale link, or an id the API cannot find, has no
* compliance to show, so the caller falls back to the first eligible scan.
*/
async function resolveUrlScanId(
scanIdFromUrl: string | undefined,
eligibleScans: ExpandedScanData[],
): Promise<string | undefined> {
if (!scanIdFromUrl) return undefined;
if (eligibleScans.some((scan) => scan.id === scanIdFromUrl)) {
return scanIdFromUrl;
}
const urlScan = (await getScan(scanIdFromUrl)) as
| { data?: { attributes?: { is_partial?: boolean } } }
| undefined;
const scan = urlScan?.data;
if (!scan) return undefined;
// OSS scans carry no is_partial at all, so only an explicit true excludes.
return scan.attributes?.is_partial === true ? undefined : scanIdFromUrl;
}
export default async function Compliance({
searchParams,
}: {
@@ -124,16 +150,23 @@ export default async function Compliance({
);
}
const scansData = await getScans({
const [subscriptionOnly, scansData] = await Promise.all([
isReportDownloadLocked(),
getScans({
filters: {
"filter[state]": "completed",
// Partial scans compute no compliance. Exclude them at the API so the
// page below never fills up with them; the filter is Cloud-only.
...(isCloud() ? { "filter[is_partial]": "false" } : {}),
},
pageSize: 50,
fields: {
scans: "name,completed_at,provider",
// is_partial is Cloud-only; the OSS API ignores unknown sparse fields.
scans: "name,completed_at,provider,is_partial",
},
include: "provider",
});
}),
]);
if (!scansData?.data) {
return (
@@ -157,7 +190,10 @@ export default async function Compliance({
);
}
// Belt and braces for an API without the filter: partial scans never
// compute compliance, so they have nothing to show or download here.
const expandedScansData: ExpandedScanData[] = scansData.data
.filter((scan: ScanProps) => !scan.attributes?.is_partial)
.filter((scan: ScanProps) => scan.relationships?.provider?.data?.id)
.map((scan: ScanProps) => {
const providerId = scan.relationships!.provider!.data!.id;
@@ -187,7 +223,9 @@ export default async function Compliance({
? scanIdParam[0]
: scanIdParam;
const selectedScanId: string | null =
scanIdFromUrl || expandedScansData[0]?.id || null;
(await resolveUrlScanId(scanIdFromUrl, expandedScansData)) ||
expandedScansData[0]?.id ||
null;
const onboardingAction = selectedScanId
? { flowId: "view-compliance" }
: {
@@ -256,6 +294,7 @@ export default async function Compliance({
provider={selectedScan.providerInfo.provider}
selectedScan={selectedScanData}
sectionScores={threatScoreData.sectionScores}
subscriptionOnly={subscriptionOnly}
/>
</div>
)}
@@ -273,6 +312,7 @@ export default async function Compliance({
scanId={selectedScanId}
selectedScan={selectedScanData}
watchlistPromise={watchlistPromise}
subscriptionOnly={subscriptionOnly}
/>
</Suspense>
</>
@@ -302,11 +342,13 @@ const SSRComplianceGrid = async ({
scanId,
selectedScan,
watchlistPromise,
subscriptionOnly,
}: {
searchParams: SearchParamsProps;
scanId: string | null;
selectedScan?: ScanEntity;
watchlistPromise: Promise<ComplianceWatchlistContext>;
subscriptionOnly: boolean;
}) => {
const regionFilter = searchParams["filter[region__in]"]?.toString() || "";
@@ -388,6 +430,7 @@ const SSRComplianceGrid = async ({
catalogEntries={watchlist.entries}
providerType={providerType}
canManageWatchlist={watchlist.canManage}
subscriptionOnly={subscriptionOnly}
/>
</ComplianceOverviewPanel>
);
+15 -14
View File
@@ -59,6 +59,10 @@ export default async function RootLayout({
// Skip Cloud-only onboarding fetches and orchestrators in OSS.
const cloudEnabled = isCloud();
// Every deployment needs the provider count: it drives the first-run redirect
// and the sidebar's Add Provider action.
const providersPromise = getProviders({ page: 1, pageSize: 1 });
// One-time server-side Registry gate per request: only an ELIGIBLE answer
// shows the sidebar entry; UNKNOWN and INELIGIBLE both hide it. Started
// here so it resolves in parallel with the Cloud onboarding fetches.
@@ -69,29 +73,27 @@ export default async function RootLayout({
// Fail-open: unknown scan state is treated as "has data" so the banner never blocks
// progression on a fetch error.
let hasCompletedScan = true;
// Tri-state: true = has providers, false = zero providers, undefined = fetch failed (gate fails open).
let hasProviders: boolean | undefined = false;
// Scopes the onboarding steps' local markers, so resolving them for one
// tenant does not silence them for another.
let tenantId: string | null = null;
if (cloudEnabled) {
const [providersData, scansByState] = await Promise.all([
getProviders({ page: 1, pageSize: 1 }),
getScansByState(),
]);
const scansByState = await getScansByState();
hasCompletedScan = Array.isArray(scansByState?.data)
? scansByState.data.some(
(scan: { attributes?: { state?: string } }) =>
scan.attributes?.state === SCAN_STATES.COMPLETED,
)
: true;
hasProviders = Array.isArray(providersData?.data)
? providersData.data.length > 0
: undefined;
tenantId = (await auth())?.tenantId ?? null;
}
const providersData = await providersPromise;
// Tri-state: true = has providers, false = zero providers, undefined = fetch failed (gate fails open).
const hasProviders: boolean | undefined = Array.isArray(providersData?.data)
? providersData.data.length > 0
: undefined;
const registryEligible =
(await registryAccessPromise).status === REGISTRY_ACCESS.ELIGIBLE;
@@ -114,13 +116,12 @@ export default async function RootLayout({
<Suspense>
<NavigationProgress />
</Suspense>
{/* Store uses boolean; gate receives tri-state to fail open on fetch errors. */}
<StoreInitializer
values={{ hasProviders: hasProviders ?? false, registryEligible }}
/>
{/* Tri-state for both: an unknown count leaves the store unresolved and the gate closed. */}
<StoreInitializer values={{ hasProviders, registryEligible }} />
{/* Every deployment: an empty tenant lands on the add-provider wizard once. */}
<OnboardingGate hasProviders={hasProviders} tenantId={tenantId} />
{cloudEnabled && (
<>
<OnboardingGate hasProviders={hasProviders} />
{/* Single mount point so the watcher survives post-connect navigation. */}
<OnboardingCheckpointWatcher tenantId={tenantId} />
{/* Persistent banner shown only while a guided sequence is active. */}
@@ -54,6 +54,19 @@ export class ProvidersPageHarness extends BrowserHarness<OrgFixture> {
return this.countRequests("POST", "/apply");
}
/** `POST /providers` alone: secrets and connection checks nest under it. */
get providerCreateCallCount(): number {
return this.requestLog.filter(
(request) =>
request.method === "POST" &&
new URL(request.url).pathname.replace(/\/$/, "").endsWith("/providers"),
).length;
}
get secretCreateCallCount(): number {
return this.countRequests("POST", "/providers/secrets");
}
/**
* Whether any apply asked the endpoint to include related resources, which it
* rejects outright — a tripwire, not a preference.
@@ -83,7 +96,7 @@ export class ProvidersPageHarness extends BrowserHarness<OrgFixture> {
).length;
}
private get connectionCallCount(): number {
get connectionCallCount(): number {
return this.countRequests("POST", "/connection");
}
@@ -198,10 +211,75 @@ export class ProvidersPageHarness extends BrowserHarness<OrgFixture> {
/** Enter the AWS Organizations onboarding flow from a fresh wizard. */
async chooseAwsOrganizations(): Promise<void> {
await this.selectProviderType(/Amazon Web Services/);
await this.chooseMethod(/Add Multiple Accounts With AWS Organizations/);
// AWS hosts its single-account/organization switch as tabs on its connect step.
const tab = await this.waitFor(() =>
this.byRoleName("tab", /Full AWS Organization/),
);
await this.user.click(tab);
await this.waitForText(/Organization Details/);
}
/** Labels of the wizard's progress stepper, top to bottom. */
stepperLabels(): string[] {
return Array.from(
document.querySelectorAll('nav[aria-label="Wizard progress"] span'),
(node) => node.textContent?.trim() ?? "",
);
}
/** Wait until the AWS single-account connect step (with its method tabs) is showing. */
async waitForAwsConnectStep(): Promise<void> {
await this.waitFor(
() => this.byRoleName("tab", /Single AWS Account/),
undefined,
"AWS connect step",
);
}
/** Type the account id and static keys on the AWS one-step connect form. */
async fillAwsAccountKeys({
accountId,
accessKeyId,
secretAccessKey,
}: {
accountId: string;
accessKeyId: string;
secretAccessKey: string;
}): Promise<void> {
const accountInput = await this.waitFor(() =>
this.inputByName("providerUid"),
);
await this.user.fill(accountInput, accountId);
const keyInput = await this.waitFor(() =>
this.inputByName("aws_access_key_id"),
);
await this.user.fill(keyInput, accessKeyId);
const secretInput = await this.waitFor(() =>
this.inputByName("aws_secret_access_key"),
);
await this.user.fill(secretInput, secretAccessKey);
}
/** Submit the AWS one-step form; waits for its action to become enabled. */
async connectAccount(): Promise<void> {
await this.clickPrimary(/Connect account/);
}
/** Wait until the provider wizard reached its launch step. */
async waitForProviderLaunchStep(timeoutMs = 20000): Promise<void> {
await this.waitForText(/Scan Schedule/, timeoutMs);
}
/** Switch back to a single account from the organization flow's tabs. */
async switchToAwsSingleAccount(): Promise<void> {
const tab = await this.waitFor(
() => this.byRoleName("tab", /Single AWS Account/),
undefined,
"Single AWS Account tab",
);
await this.user.click(tab);
}
/** Select GCP and open the GCP Organization method card (no advance wait). */
async chooseGcpOrganizationsMethod(): Promise<void> {
await this.selectProviderType(/Google Cloud Platform/);

Some files were not shown because too many files have changed in this diff Show More