mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-10-03 17:54:05 +00:00
Merge remote-tracking branch 'origin/master' into mintlify/cb114598
This commit is contained in:
@@ -110,11 +110,20 @@ DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY=""
|
||||
DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN=""
|
||||
|
||||
# The AWS region where your S3 bucket is located (e.g., "us-east-1")
|
||||
# Required if the bucket uses SSE-KMS: download URLs are then signed with SigV4, which
|
||||
# is scoped to this region, so it must match the bucket's
|
||||
DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION=""
|
||||
|
||||
# The name of the S3 bucket where scan output should be stored
|
||||
DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET=""
|
||||
|
||||
# The storage endpoint the API and Celery workers use to upload and list scan output
|
||||
# (e.g. "http://minio:9000"). Leave empty on AWS S3. Set it when scan output is stored on
|
||||
# S3-compatible object storage such as MinIO instead of real S3.
|
||||
# If set without DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL below, report download URLs are
|
||||
# signed against this internal host, and a browser outside the container network cannot open them.
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL=""
|
||||
|
||||
# The storage address the browser can reach, used only to sign report download URLs
|
||||
# (e.g. "https://storage.example.com"). Leave empty on AWS S3. Set it when storage is
|
||||
# only reachable inside the container network, such as MinIO on "http://minio:9000".
|
||||
@@ -165,7 +174,7 @@ SENTRY_RELEASE=local
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.43.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.44.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -11,6 +11,7 @@ on:
|
||||
- "v5.*"
|
||||
paths:
|
||||
- ".github/workflows/ui-e2e-tests-v2.yml"
|
||||
- ".github/workflows/test-impact-analysis.yml"
|
||||
- ".github/test-impact.yml"
|
||||
- "ui/**"
|
||||
- "api/**" # API changes can affect UI E2E
|
||||
@@ -237,8 +238,8 @@ jobs:
|
||||
|
||||
- name: Add AWS credentials for testing
|
||||
run: |
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
echo "AWS_ACCESS_KEY_ID=${E2E_AWS_PROVIDER_ACCESS_KEY}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${E2E_AWS_PROVIDER_SECRET_KEY}" >> .env
|
||||
|
||||
- name: Build API image from current code
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
|
||||
+11
@@ -115,3 +115,14 @@ ignore:
|
||||
- vulnerability: CVE-2026-9669
|
||||
package:
|
||||
name: python
|
||||
# CVE-2026-82049 (tarfile data/tar filter bypass via a hard link to a symlink) has no
|
||||
# fixed CPython release on any branch: the fix is merged on main and 3.13 only, and the
|
||||
# 3.12 backport is still open. Grype records 3.14.0b1 as the fix, so only-fixed does not
|
||||
# drop it, yet python:3.12.14-slim-trixie reports it too. Prowler never extracts tar
|
||||
# archives to disk: the ECR image inspection reads members in memory with extractfile().
|
||||
# Remove once the base image ships a 3.12 release that includes the backport.
|
||||
# https://github.com/python/cpython/issues/157190
|
||||
# https://github.com/python/cpython/pull/157454
|
||||
- vulnerability: CVE-2026-82049
|
||||
package:
|
||||
name: python
|
||||
|
||||
@@ -4,6 +4,22 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [1.44.0] (Prowler v5.43.0)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Report download URLs can be signed against a browser-reachable storage host via `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL`, so downloads complete on deployments where storage is only reachable inside the container network [(#12552)](https://github.com/prowler-cloud/prowler/pull/12552)
|
||||
- A scan report download no longer fails with a server error when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is unset, which is common on storage with no meaningful region [(#12552)](https://github.com/prowler-cloud/prowler/pull/12552)
|
||||
- Lapsed pending invitations are reported as expired and no longer block a new invitation for the same email [(#12831)](https://github.com/prowler-cloud/prowler/pull/12831)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `libsqlite3-0`, `gzip`, `perl-base` and `libpcre2-8-0` upgraded in the API container image, patching high Debian CVEs [(#12804)](https://github.com/prowler-cloud/prowler/pull/12804)
|
||||
- PowerShell from 7.5.9 to 7.5.11 in the API container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901 [(#12811)](https://github.com/prowler-cloud/prowler/pull/12811)
|
||||
- Bumped `anyio` to 4.14.2 to resolve CVE-2026-63374 [(#12848)](https://github.com/prowler-cloud/prowler/pull/12848)
|
||||
|
||||
---
|
||||
|
||||
## [1.43.0] (Prowler v5.42.0)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
`libsqlite3-0`, `gzip`, `perl-base` and `libpcre2-8-0` upgraded in the API container image, patching high Debian CVEs
|
||||
@@ -1 +0,0 @@
|
||||
PowerShell from 7.5.9 to 7.5.11 in the API container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901
|
||||
@@ -0,0 +1 @@
|
||||
Adds a periodic sweep that drops orphaned Attack Paths temp Neo4j scan databases left behind when a worker or Neo4j crashes mid-scan, before they accumulate unbounded
|
||||
@@ -0,0 +1 @@
|
||||
Resources no longer keep a stale failed findings count forever when a scoped or imported scan for the same provider completes after a full scan, which used to make the full scan skip its own cleanup
|
||||
@@ -1 +0,0 @@
|
||||
Lapsed pending invitations are reported as expired and no longer block a new invitation for the same email
|
||||
@@ -0,0 +1 @@
|
||||
Providers whose most recent completed scan has no `completed_at` timestamp are no longer missing from every endpoint that reports a provider's latest scan, which now falls back to scan creation order instead of skipping the provider
|
||||
@@ -0,0 +1 @@
|
||||
Unify how every endpoint resolves a provider latest completed scan, so overlapping scans no longer make findings, compliance and mute rules read from different scans
|
||||
@@ -1 +0,0 @@
|
||||
Report download URLs can be signed against a browser-reachable storage host via `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL`, so downloads complete on deployments where storage is only reachable inside the container network
|
||||
@@ -1 +0,0 @@
|
||||
A scan report download no longer fails with a server error when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is unset, which is common on storage with no meaningful region
|
||||
@@ -0,0 +1 @@
|
||||
Scan output uploads and downloads can now target S3-compatible object storage such as MinIO directly via `DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL`, instead of relying on process-wide AWS environment variables that also hijacked unrelated AWS API calls
|
||||
@@ -0,0 +1 @@
|
||||
Scan report downloads from an S3 bucket with default SSE-KMS encryption no longer fail with an `InvalidArgument` error: when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is set, presigned download URLs are signed with AWS Signature Version 4 for that region
|
||||
@@ -0,0 +1 @@
|
||||
`POST /api/v1/scans` again returns the new scan id in the response `task_args`, which had been empty since the scan broker publish moved to transaction commit
|
||||
@@ -0,0 +1 @@
|
||||
Celery loggers are now declared explicitly in `custom_logging.py` so fatal worker errors are no longer silenced by `disable_existing_loggers=True`. All long-running services in `docker-compose.yml` now have `restart: unless-stopped` so containers recover automatically after unexpected crashes.
|
||||
+2
-2
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.44.0"
|
||||
version = "1.45.0"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
@@ -137,7 +137,7 @@ constraint-dependencies = [
|
||||
"aliyun-log-fastpb==0.2.0",
|
||||
"amqp==5.3.1",
|
||||
"annotated-types==0.7.0",
|
||||
"anyio==4.12.1",
|
||||
"anyio==4.14.2",
|
||||
"applicationinsights==0.11.10",
|
||||
"apscheduler==3.11.2",
|
||||
"argcomplete==3.5.3",
|
||||
|
||||
@@ -207,6 +207,11 @@ def drop_database(database: str) -> None:
|
||||
sink_module.get_backend().drop_database(database)
|
||||
|
||||
|
||||
def list_databases() -> list[str]:
|
||||
"""List database names on the ingest cluster. Temp scan DBs always live here."""
|
||||
return ingest.list_databases()
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
return sink_module.get_backend().drop_subgraph(database, provider_id)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from api.attack_paths.ingest.driver import (
|
||||
get_session,
|
||||
get_uri,
|
||||
init_driver,
|
||||
list_databases,
|
||||
run_cypher,
|
||||
)
|
||||
|
||||
@@ -25,5 +26,6 @@ __all__ = [
|
||||
"get_session",
|
||||
"get_uri",
|
||||
"init_driver",
|
||||
"list_databases",
|
||||
"run_cypher",
|
||||
]
|
||||
|
||||
@@ -165,6 +165,14 @@ def drop_database(database: str) -> None:
|
||||
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
|
||||
|
||||
|
||||
def list_databases() -> list[str]:
|
||||
"""List every database name on the Neo4j temp-database cluster."""
|
||||
# A cluster returns one row per hosting server, so dedupe on name
|
||||
with get_session() as session:
|
||||
result = session.run("SHOW DATABASES YIELD name RETURN DISTINCT name")
|
||||
return [record["name"] for record in result]
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
"""Best-effort cache clear for a Neo4j database."""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
|
||||
@@ -409,7 +409,7 @@ def batch_delete(tenant_id, queryset, batch_size=settings.DJANGO_DELETION_BATCH_
|
||||
|
||||
Args:
|
||||
tenant_id (str): Tenant ID the queryset belongs to.
|
||||
queryset (QuerySet): The queryset of objects to delete.
|
||||
queryset: The queryset of objects to delete.
|
||||
batch_size (int): The number of objects to delete in each batch.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
from django.db import migrations
|
||||
|
||||
TASK_NAME = "attack-paths-reap-orphaned-tmp-databases"
|
||||
INTERVAL_HOURS = 6
|
||||
|
||||
|
||||
def create_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
schedule, _ = IntervalSchedule.objects.get_or_create(
|
||||
every=INTERVAL_HOURS,
|
||||
period="hours",
|
||||
)
|
||||
|
||||
PeriodicTask.objects.update_or_create(
|
||||
name=TASK_NAME,
|
||||
defaults={
|
||||
"task": TASK_NAME,
|
||||
"interval": schedule,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def delete_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
PeriodicTask.objects.filter(name=TASK_NAME).delete()
|
||||
|
||||
# Clean up the schedule if no other task references it
|
||||
IntervalSchedule.objects.filter(
|
||||
every=INTERVAL_HOURS,
|
||||
period="hours",
|
||||
periodictask__isnull=True,
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0099_delete_tenant_onboarding_profile"),
|
||||
("django_celery_beat", "0019_alter_periodictasks_options"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_periodic_task, delete_periodic_task),
|
||||
]
|
||||
@@ -617,9 +617,66 @@ class Task(RowLevelSecurityProtectedModel):
|
||||
resource_name = "tasks"
|
||||
|
||||
|
||||
class ScanQuerySet(models.QuerySet):
|
||||
"""Shared selectors for "the latest scan of a provider".
|
||||
|
||||
The queryset must already be scoped by the caller: manager, tenant, RBAC,
|
||||
providers and database alias.
|
||||
"""
|
||||
|
||||
# How "which completed scan is the provider's current one" is ordered.
|
||||
LATEST_ORDER_BY = (
|
||||
models.F("completed_at").desc(nulls_last=True),
|
||||
models.F("inserted_at").desc(),
|
||||
models.F("id").desc(),
|
||||
)
|
||||
|
||||
def _eligible_for_latest(self) -> "ScanQuerySet":
|
||||
"""Restrict to the scans that may be a provider's latest.
|
||||
|
||||
Returns:
|
||||
ScanQuerySet: The completed scans.
|
||||
"""
|
||||
return self.filter(state=StateChoices.COMPLETED)
|
||||
|
||||
def latest_per_provider(self) -> "ScanQuerySet":
|
||||
"""Pick each provider's latest scan with `DISTINCT ON (provider_id)`.
|
||||
|
||||
Returns:
|
||||
ScanQuerySet: One scan per provider, the latest one.
|
||||
"""
|
||||
return (
|
||||
self._eligible_for_latest()
|
||||
.order_by("provider_id", *self.LATEST_ORDER_BY)
|
||||
.distinct("provider_id")
|
||||
)
|
||||
|
||||
def latest_ids_per_provider(self) -> list[UUID]:
|
||||
"""Evaluate `latest_per_provider` and return the scan ids.
|
||||
|
||||
The ids are materialised so callers can pass them as a literal `IN`
|
||||
list; as a subquery Postgres misestimates the row count and picks a
|
||||
slow nested loop.
|
||||
|
||||
Returns:
|
||||
list[UUID]: The id of each provider's latest scan.
|
||||
"""
|
||||
return list(self.latest_per_provider().values_list("id", flat=True))
|
||||
|
||||
def latest_first(self) -> "ScanQuerySet":
|
||||
"""Order eligible scans newest first, without deduplicating per provider.
|
||||
|
||||
Expects the queryset to be already filtered to a single provider.
|
||||
|
||||
Returns:
|
||||
ScanQuerySet: The eligible scans, latest first.
|
||||
"""
|
||||
return self._eligible_for_latest().order_by(*self.LATEST_ORDER_BY)
|
||||
|
||||
|
||||
class Scan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
objects = ActiveProviderManager.from_queryset(ScanQuerySet)()
|
||||
all_objects = ScanQuerySet.as_manager()
|
||||
|
||||
_SCOPING_SCANNER_ARG_KEYS_CACHE: tuple[str, ...] | None = None
|
||||
|
||||
@@ -726,6 +783,12 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
name="scans_prov_state_ins_desc_idx",
|
||||
),
|
||||
# TODO This might replace `scans_prov_state_ins_desc_idx` completely. Review usage
|
||||
# Since `ScanQuerySet`, no code path reads a provider's
|
||||
# completed scans by `-inserted_at`. The only query left that
|
||||
# matches this index (and `scans_prov_state_ins_desc_idx` above)
|
||||
# is `GET /scans?filter[provider]=…&filter[state]=completed` with
|
||||
# the default sort. Both are candidates to drop in a follow-up
|
||||
# once production `pg_stat_user_indexes.idx_scan` confirms it.
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-inserted_at"],
|
||||
condition=Q(state=StateChoices.COMPLETED),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.44.0
|
||||
version: 1.45.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
|
||||
@@ -187,6 +187,27 @@ class TestRoutingByDatabasePrefix:
|
||||
sink_backend_stub.drop_database.assert_called_once_with("db-tenant-abc")
|
||||
mock_ingest.drop_database.assert_not_called()
|
||||
|
||||
def test_list_databases_always_routes_to_ingest(self, sink_backend_stub):
|
||||
with patch("api.attack_paths.database.ingest") as mock_ingest:
|
||||
mock_ingest.list_databases.return_value = ["db-tmp-scan-uuid-1"]
|
||||
|
||||
assert db_module.list_databases() == ["db-tmp-scan-uuid-1"]
|
||||
|
||||
mock_ingest.list_databases.assert_called_once_with()
|
||||
|
||||
def test_ingest_list_databases_dedupes_cluster_rows(self):
|
||||
from api.attack_paths.ingest import driver as ingest_driver
|
||||
|
||||
with patch.object(ingest_driver, "get_session") as mock_get_session:
|
||||
session = mock_get_session.return_value.__enter__.return_value
|
||||
session.run.return_value = [{"name": "db-tmp-scan-uuid-1"}]
|
||||
|
||||
assert ingest_driver.list_databases() == ["db-tmp-scan-uuid-1"]
|
||||
|
||||
session.run.assert_called_once_with(
|
||||
"SHOW DATABASES YIELD name RETURN DISTINCT name"
|
||||
)
|
||||
|
||||
def test_clear_cache_routes_temp_to_ingest(self, sink_backend_stub):
|
||||
with patch("api.attack_paths.database.ingest") as mock_ingest:
|
||||
db_module.clear_cache("db-tmp-scan-uuid-1")
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from allauth.socialaccount.models import SocialApp
|
||||
from api.db_router import MainRouter
|
||||
from api.models import (
|
||||
Provider,
|
||||
ProviderComplianceScore,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
SAMLConfiguration,
|
||||
SAMLDomainIndex,
|
||||
Scan,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
TenantComplianceSummary,
|
||||
@@ -524,3 +526,226 @@ class TestTenantComplianceSummaryModel:
|
||||
|
||||
assert summary1.id != summary2.id
|
||||
assert summary1.requirements_passed != summary2.requirements_passed
|
||||
|
||||
|
||||
def _latest_scan_fixture(tenant, provider, *, completed_at, inserted_at=None, **kwargs):
|
||||
scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=kwargs.pop("trigger", Scan.TriggerChoices.MANUAL),
|
||||
state=kwargs.pop("state", StateChoices.COMPLETED),
|
||||
completed_at=completed_at,
|
||||
**kwargs,
|
||||
)
|
||||
if inserted_at is not None:
|
||||
# `inserted_at` is auto_now_add, so it has to be forced after the fact.
|
||||
Scan.all_objects.filter(pk=scan.pk).update(inserted_at=inserted_at)
|
||||
scan.refresh_from_db()
|
||||
return scan
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetOrdering:
|
||||
def test_completed_later_wins_over_inserted_later(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""The scan that FINISHED last is current, not the one that started last."""
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
finished_last = _latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now - timedelta(hours=3),
|
||||
completed_at=now,
|
||||
)
|
||||
_latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now - timedelta(hours=1),
|
||||
completed_at=now - timedelta(hours=1),
|
||||
)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [finished_last.id]
|
||||
|
||||
def test_null_completed_at_provider_is_still_returned(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""NULLS LAST, not `completed_at__isnull=False`.
|
||||
|
||||
Excluding NULL `completed_at` would drop the provider from every
|
||||
"latest" endpoint instead of falling back to `inserted_at`.
|
||||
"""
|
||||
tenant, *_ = tenants_fixture
|
||||
only_scan = _latest_scan_fixture(tenant, aws_provider, completed_at=None)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [only_scan.id]
|
||||
|
||||
def test_null_completed_at_never_outranks_a_finished_scan(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""Postgres sorts NULLs first under DESC; NULLS LAST is what fixes it."""
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
finished = _latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now - timedelta(hours=2),
|
||||
completed_at=now - timedelta(hours=2),
|
||||
)
|
||||
_latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now,
|
||||
completed_at=None,
|
||||
)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [finished.id]
|
||||
|
||||
def test_id_breaks_an_exact_timestamp_tie_deterministically(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
scans = [
|
||||
_latest_scan_fixture(
|
||||
tenant, aws_provider, inserted_at=now, completed_at=now
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
expected = max(scan.id for scan in scans)
|
||||
|
||||
picks = {
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider()[0]
|
||||
for _ in range(5)
|
||||
}
|
||||
assert picks == {expected}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetEligibility:
|
||||
def test_unfinished_scans_are_excluded(self, tenants_fixture, aws_provider):
|
||||
tenant, *_ = tenants_fixture
|
||||
_latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
completed_at=None,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
assert (
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetManagerChoice:
|
||||
def test_active_manager_hides_soft_deleted_providers(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""`Scan.objects` drops soft-deleted providers, `all_objects` keeps them.
|
||||
|
||||
The queryset must not decide this for the caller.
|
||||
"""
|
||||
tenant, *_ = tenants_fixture
|
||||
scan = _latest_scan_fixture(
|
||||
tenant, aws_provider, completed_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
Provider.all_objects.filter(pk=aws_provider.pk).update(is_deleted=True)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [scan.id]
|
||||
assert Scan.objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetPerProviderScoping:
|
||||
def test_one_scan_per_provider(self, tenants_fixture, aws_provider_pair):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider_one, provider_two = aws_provider_pair
|
||||
now = datetime.now(UTC)
|
||||
|
||||
newest_one = _latest_scan_fixture(tenant, provider_one, completed_at=now)
|
||||
_latest_scan_fixture(tenant, provider_one, completed_at=now - timedelta(days=1))
|
||||
newest_two = _latest_scan_fixture(tenant, provider_two, completed_at=now)
|
||||
|
||||
assert set(
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider()
|
||||
) == {newest_one.id, newest_two.id}
|
||||
|
||||
def test_caller_filters_are_preserved(self, tenants_fixture, aws_provider_pair):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider_one, provider_two = aws_provider_pair
|
||||
now = datetime.now(UTC)
|
||||
|
||||
scan_one = _latest_scan_fixture(tenant, provider_one, completed_at=now)
|
||||
_latest_scan_fixture(tenant, provider_two, completed_at=now)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id, provider__in=[provider_one]
|
||||
).latest_ids_per_provider() == [scan_one.id]
|
||||
|
||||
def test_latest_first_is_ordered_not_deduplicated(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
newest = _latest_scan_fixture(tenant, aws_provider, completed_at=now)
|
||||
older = _latest_scan_fixture(
|
||||
tenant, aws_provider, completed_at=now - timedelta(days=1)
|
||||
)
|
||||
|
||||
ordered = list(
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant.id, provider_id=aws_provider.id
|
||||
).latest_first()
|
||||
)
|
||||
assert [scan.id for scan in ordered] == [newest.id, older.id]
|
||||
|
||||
def test_tenant_isolation(self, tenants_fixture, aws_provider):
|
||||
tenant, other_tenant, *_ = tenants_fixture
|
||||
_latest_scan_fixture(tenant, aws_provider, completed_at=datetime.now(UTC))
|
||||
|
||||
assert (
|
||||
Scan.all_objects.filter(tenant_id=other_tenant.id).latest_ids_per_provider()
|
||||
== []
|
||||
)
|
||||
|
||||
def test_empty_queryset_returns_empty_list(self, tenants_fixture):
|
||||
tenant, *_ = tenants_fixture
|
||||
assert (
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetPerProviderQuerysetShape:
|
||||
def test_returns_a_queryset_not_a_list(self, tenants_fixture, aws_provider):
|
||||
tenant, *_ = tenants_fixture
|
||||
_latest_scan_fixture(tenant, aws_provider, completed_at=datetime.now(UTC))
|
||||
|
||||
qs = Scan.all_objects.filter(tenant_id=tenant.id).latest_per_provider()
|
||||
# Callers chain .values(...) / .values_list(...) onto this.
|
||||
assert qs.values_list("provider_id", flat=True).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetRelatedManager:
|
||||
def test_reverse_relation_exposes_the_methods(self, tenants_fixture, aws_provider):
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
newest = _latest_scan_fixture(tenant, aws_provider, completed_at=now)
|
||||
_latest_scan_fixture(tenant, aws_provider, completed_at=now - timedelta(days=1))
|
||||
|
||||
assert aws_provider.scans.latest_first().first().id == newest.id
|
||||
|
||||
@@ -3951,6 +3951,43 @@ class TestScanViewSet:
|
||||
mock_enqueue_scan_execution.assert_called_once()
|
||||
# assert scan.scanner_args == expected_scanner_args
|
||||
|
||||
@patch("api.v1.views.enqueue_scan_execution_on_commit")
|
||||
def test_scans_create_returns_the_scan_id_in_task_args(
|
||||
self,
|
||||
mock_enqueue_scan_execution,
|
||||
authenticated_client,
|
||||
okta_provider,
|
||||
):
|
||||
"""The 202 is a task, so `task_args` is the only place the scan id is.
|
||||
|
||||
It is serialized before the on_commit publish that would otherwise fill
|
||||
the kwargs, so the record has to carry them from the start.
|
||||
"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "scans",
|
||||
"attributes": {"name": "New Scan"},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(okta_provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("scan-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
scan = Scan.objects.get()
|
||||
assert response.json()["data"]["attributes"]["task_args"] == {
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(okta_provider.id),
|
||||
}
|
||||
|
||||
@patch("tasks.tasks.perform_scan_task.apply_async")
|
||||
def test_scans_create_queues_scan_when_provider_has_active_scan(
|
||||
self,
|
||||
|
||||
@@ -2822,6 +2822,15 @@ class ScanViewSet(ProviderVisibilityMixin, BaseRLSViewSet):
|
||||
tenant_id=self.request.tenant_id,
|
||||
task_id=pre_task_id,
|
||||
task_status=(QUEUED_SCAN_TASK_STATE if active_scan else None),
|
||||
# This response is serialized before the on_commit publish,
|
||||
# so without these the caller gets a task id and no scan id.
|
||||
# Kept in step with what `enqueue_scan_execution_on_commit`
|
||||
# publishes below.
|
||||
task_kwargs={
|
||||
"tenant_id": str(self.request.tenant_id),
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
},
|
||||
)
|
||||
|
||||
if not active_scan:
|
||||
@@ -3478,12 +3487,8 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
latest_scans = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
Scan.all_objects.filter(tenant_id=tenant_id)
|
||||
.latest_per_provider()
|
||||
.values("provider_id")
|
||||
)
|
||||
|
||||
@@ -3615,11 +3620,9 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
query_params = request.query_params
|
||||
|
||||
latest_scans_queryset = (
|
||||
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
)
|
||||
latest_scans_queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_per_provider()
|
||||
|
||||
queryset = ResourceScanSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
@@ -4206,12 +4209,9 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
latest_scan_ids = list(
|
||||
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scan_ids = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_ids_per_provider()
|
||||
filtered_queryset = filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
@@ -4234,11 +4234,9 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
query_params = request.query_params
|
||||
|
||||
latest_scans_queryset = (
|
||||
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
)
|
||||
latest_scans_queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_per_provider()
|
||||
raw_latest_scans_ids = list(
|
||||
latest_scans_queryset.values_list("id", "unique_resource_count")
|
||||
)
|
||||
@@ -4979,11 +4977,7 @@ class ComplianceOverviewViewSet(
|
||||
if provider_filters:
|
||||
scans = scans.filter(**provider_filters)
|
||||
|
||||
return list(
|
||||
scans.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
return scans.latest_ids_per_provider()
|
||||
|
||||
def _filtered_queryset_for_latest_provider_scans(self, latest_scan_ids=None):
|
||||
if latest_scan_ids is None:
|
||||
@@ -5725,14 +5719,9 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scan_ids = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, **provider_filter
|
||||
).latest_ids_per_provider()
|
||||
|
||||
return filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
@@ -5759,16 +5748,10 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
|
||||
|
||||
def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None):
|
||||
provider_filter = self._get_provider_filter()
|
||||
queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
queryset = Scan.all_objects.filter(tenant_id=tenant_id, **provider_filter)
|
||||
if provider_filters:
|
||||
queryset = queryset.filter(**provider_filters)
|
||||
return (
|
||||
queryset.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
return queryset.latest_ids_per_provider()
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="providers")
|
||||
def providers(self, request):
|
||||
@@ -5780,14 +5763,9 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scan_ids = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, **provider_filter
|
||||
).latest_ids_per_provider()
|
||||
|
||||
findings_aggregated = (
|
||||
queryset.filter(scan_id__in=latest_scan_ids)
|
||||
@@ -7934,18 +7912,13 @@ class FindingGroupViewSet(JsonApiFilterMixin, BaseRLSViewSet):
|
||||
|
||||
def _get_latest_findings_per_provider(self, filtered_queryset):
|
||||
"""Keep only findings from each provider's most recent completed scan."""
|
||||
# Materialize to a literal IN list. Left as a subquery, Postgres can't
|
||||
# estimate the match count and picks a serial nested loop on
|
||||
# resource_finding_mappings when one scan dominates findings
|
||||
latest_scan_ids = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=self.request.tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
# `latest_ids_per_provider` materializes to a literal IN list.
|
||||
# Left as a subquery, Postgres can't estimate the match count and picks
|
||||
# a serial nested loop on resource_finding_mappings when one scan
|
||||
# dominates findings
|
||||
latest_scan_ids = Scan.objects.filter(
|
||||
tenant_id=self.request.tenant_id
|
||||
).latest_ids_per_provider()
|
||||
return filtered_queryset.filter(scan_id__in=latest_scan_ids)
|
||||
|
||||
def _post_process_aggregation(self, aggregated_data):
|
||||
@@ -8891,16 +8864,14 @@ class FindingGroupViewSet(JsonApiFilterMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
queryset = self._get_finding_queryset()
|
||||
|
||||
# Order by -completed_at (matching the /latest summary path and the
|
||||
# daily summary upsert keyed on midnight(completed_at)) so that
|
||||
# overlapping scans do not make /resources and /latest read from
|
||||
# different scans and report diverging counts.
|
||||
latest_scan_ids = (
|
||||
Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
# The shared selector orders by -completed_at (matching the /latest
|
||||
# summary path and the daily summary upsert keyed on
|
||||
# midnight(completed_at)) so that overlapping scans do not make
|
||||
# /resources and /latest read from different scans and report
|
||||
# diverging counts.
|
||||
latest_scan_ids = Scan.objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_ids_per_provider()
|
||||
|
||||
normalized_params = self._normalize_jsonapi_params(request.query_params)
|
||||
# Remove date filters since we're using latest
|
||||
|
||||
@@ -231,6 +231,62 @@ LOGGING = {
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
# Celery loggers must be declared explicitly because
|
||||
# disable_existing_loggers=True silences any logger that exists at
|
||||
# dictConfig time but is not named here. Without these, fatal worker
|
||||
# errors (e.g. celery.worker CRITICAL) produce no output.
|
||||
# "celery" must keep propagating: get_task_logger() parents task
|
||||
# loggers under celery.task, so blocking here hides them from root.
|
||||
"celery": {
|
||||
"level": LEVEL,
|
||||
"propagate": True,
|
||||
},
|
||||
"celery.worker": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"celery.worker.consumer": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"celery.worker.consumer.consumer": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"kombu": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"kombu.transport.redis": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"billiard": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"amqp": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
# WARNING keeps task failures but skips one "succeeded" line per task.
|
||||
"celery.app.trace": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": "WARNING",
|
||||
"propagate": False,
|
||||
},
|
||||
"celery.beat": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
},
|
||||
# Gunicorn required configuration
|
||||
"root": {
|
||||
|
||||
@@ -7,6 +7,7 @@ from config.settings.eventstream import * # noqa
|
||||
from config.settings.partitions import * # noqa
|
||||
from config.settings.sentry import * # noqa
|
||||
from config.settings.social_login import * # noqa
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
SECRET_KEY = env("SECRET_KEY", default="secret")
|
||||
DEBUG = env.bool("DJANGO_DEBUG", default=False)
|
||||
@@ -295,6 +296,9 @@ DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY = env.str(
|
||||
)
|
||||
DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN = env.str("DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN", "")
|
||||
DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION = env.str("DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION", "")
|
||||
# Storage endpoint the API and Celery workers use to talk to S3-compatible object storage
|
||||
# such as MinIO. Empty means the real AWS S3 endpoint, which is unaffected.
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL = env.str("DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL", "")
|
||||
# Browser-reachable storage host used to sign download URLs. Empty means sign against the
|
||||
# same endpoint the API talks to, which is what Prowler Cloud on S3 does.
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL = env.str(
|
||||
@@ -325,6 +329,17 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 960
|
||||
) # 16h
|
||||
|
||||
# Minimum age (of the scan row, or of the scan id itself when the row is gone) before
|
||||
# the periodic reaper will drop an orphaned temp Neo4j database. Keeps a scan that is
|
||||
# still legitimately in flight from ever losing its staging database mid-run.
|
||||
ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS = env.int(
|
||||
"ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS", 6
|
||||
)
|
||||
if ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS <= 0:
|
||||
raise ImproperlyConfigured(
|
||||
"ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS must be a positive number of hours"
|
||||
)
|
||||
|
||||
# Selects where the persistent attack-paths graph is stored. The scan
|
||||
# temporary database is always Neo4j; only the sink is configurable.
|
||||
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Periodic reaper for orphaned temp Neo4j scan databases.
|
||||
|
||||
`scan.py` creates a throw-away `db-tmp-scan-<attack_paths_scan_id>` database per
|
||||
scan and drops it once the scan finishes, success or failure. When the worker
|
||||
or Neo4j itself dies mid-scan, that drop never runs and nothing else ever
|
||||
revisits the database - it sits there forever. This sweep lists every temp
|
||||
database on the ingest cluster and drops the ones whose scan is gone or has
|
||||
been finished for longer than the configured safety margin.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
from api.models import AttackPathsScan, StateChoices
|
||||
from api.uuid_utils import datetime_from_uuid7
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS
|
||||
from uuid6 import UUID as UUID7
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
TERMINAL_STATES = (
|
||||
StateChoices.COMPLETED,
|
||||
StateChoices.FAILED,
|
||||
StateChoices.CANCELLED,
|
||||
)
|
||||
|
||||
|
||||
def reap_orphaned_tmp_databases() -> dict:
|
||||
"""Drop temp Neo4j scan databases whose scan is gone or long finished.
|
||||
|
||||
A failure listing databases aborts the whole sweep (nothing to iterate).
|
||||
A failure reaping one database is logged and skipped so the rest of the
|
||||
sweep still runs.
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
safety_margin = timedelta(hours=ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS)
|
||||
|
||||
try:
|
||||
databases = graph_database.list_databases()
|
||||
except Exception:
|
||||
logger.exception("Failed to list ingest Neo4j databases for temp-db reap")
|
||||
return {"dropped_count": 0, "databases": []}
|
||||
|
||||
tmp_databases = [
|
||||
name for name in databases if name.startswith(graph_database.TEMP_DB_PREFIX)
|
||||
]
|
||||
|
||||
dropped: list[str] = []
|
||||
for database in tmp_databases:
|
||||
try:
|
||||
if _is_orphaned(database, now, safety_margin):
|
||||
graph_database.drop_database(database)
|
||||
dropped.append(database)
|
||||
logger.info(f"Dropped orphaned temp Neo4j database `{database}`")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to reap temp Neo4j database `{database}`")
|
||||
|
||||
logger.info(f"Temp Neo4j database reap: {len(dropped)} dropped")
|
||||
return {"dropped_count": len(dropped), "databases": dropped}
|
||||
|
||||
|
||||
def _is_orphaned(database: str, now: datetime, safety_margin: timedelta) -> bool:
|
||||
"""Decide whether a temp database is safe to drop.
|
||||
|
||||
No scan row: the row was hard-deleted (tenant/provider cleanup) or was
|
||||
never created. Falls back to the scan id's own UUIDv7 timestamp so a
|
||||
database created moments ago is never touched even without a row to check.
|
||||
|
||||
Scan row present: only reapable once it reached a terminal state and has
|
||||
been finished for longer than the safety margin, so a scan still
|
||||
legitimately executing is never touched.
|
||||
"""
|
||||
scan_id = database[len(graph_database.TEMP_DB_PREFIX) :]
|
||||
|
||||
try:
|
||||
scan_uuid = UUID7(scan_id)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Temp database `{database}` has an unparseable scan id, skipping"
|
||||
)
|
||||
return False
|
||||
|
||||
# Global sweep with no tenant context: admin_db bypasses RLS on purpose, the same
|
||||
# way cleanup_stale_attack_paths_scans finds stale scans across every tenant.
|
||||
scan = (
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
.filter(id=scan_uuid)
|
||||
.first()
|
||||
)
|
||||
|
||||
if scan is None:
|
||||
if scan_uuid.version != 7:
|
||||
logger.warning(
|
||||
f"Temp database `{database}` has no scan row and a non-UUIDv7 id, "
|
||||
"skipping"
|
||||
)
|
||||
return False
|
||||
return now - datetime_from_uuid7(scan_uuid) >= safety_margin
|
||||
|
||||
if scan.state not in TERMINAL_STATES:
|
||||
return False
|
||||
|
||||
# `mark_scan_finished` does not touch `updated_at`, so prefer `completed_at`
|
||||
finished_at = scan.completed_at or scan.updated_at
|
||||
return now - finished_at >= safety_margin
|
||||
@@ -499,10 +499,13 @@ def backfill_provider_compliance_scores(tenant_id: str) -> dict:
|
||||
provider_id__in=existing_providers
|
||||
)
|
||||
|
||||
# `completed_scans` keeps its own `completed_at__isnull=False`: this
|
||||
# task writes a *dated* ProviderComplianceScore row, so unlike the read
|
||||
# paths it genuinely cannot use a scan without a `completed_at`.
|
||||
scan_info = list(
|
||||
completed_scans.order_by("provider_id", "-completed_at")
|
||||
.distinct("provider_id")
|
||||
.values("id", "provider_id", "completed_at")
|
||||
completed_scans.latest_per_provider().values(
|
||||
"id", "provider_id", "completed_at"
|
||||
)
|
||||
)
|
||||
|
||||
if not scan_info:
|
||||
|
||||
@@ -207,15 +207,21 @@ def get_s3_client():
|
||||
This function attempts to initialize an S3 client by reading the AWS access key, secret key,
|
||||
session token, and region from environment variables. It then validates the client by listing
|
||||
available S3 buckets. If an error occurs during this process (for example, due to missing or
|
||||
invalid credentials), it falls back to creating an S3 client without explicitly provided credentials,
|
||||
which may rely on other configuration sources (e.g., IAM roles).
|
||||
invalid credentials), it falls back to creating an S3 client without explicitly provided
|
||||
credentials, which may rely on other configuration sources (e.g., IAM roles).
|
||||
|
||||
That fallback is only safe when no explicit endpoint is configured: with an endpoint set, the
|
||||
explicit client already targets the intended S3-compatible storage, and the fallback client
|
||||
would go to the AWS default provider chain instead, an unrelated real-AWS account reachable
|
||||
from the host. So when an endpoint is configured, the original error propagates instead.
|
||||
|
||||
Returns:
|
||||
boto3.client: A configured S3 client instance.
|
||||
|
||||
Raises:
|
||||
ClientError, NoCredentialsError, or ParamValidationError if both attempts to create a client fail.
|
||||
ClientError, NoCredentialsError, or ParamValidationError if the client cannot be created.
|
||||
"""
|
||||
endpoint = settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL
|
||||
s3_client = None
|
||||
try:
|
||||
s3_client = boto3.client(
|
||||
@@ -226,9 +232,12 @@ def get_s3_client():
|
||||
# Storage that has no meaningful region, MinIO among it, is usually configured
|
||||
# without one, and botocore rejects an empty region before any request is made.
|
||||
region_name=settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION or "us-east-1",
|
||||
endpoint_url=endpoint or None,
|
||||
)
|
||||
s3_client.list_buckets()
|
||||
except (ClientError, NoCredentialsError, ParamValidationError, ValueError):
|
||||
if endpoint:
|
||||
raise
|
||||
s3_client = boto3.client("s3")
|
||||
s3_client.list_buckets()
|
||||
|
||||
@@ -236,13 +245,21 @@ def get_s3_client():
|
||||
|
||||
|
||||
def get_s3_presign_client():
|
||||
"""Return a client that signs URLs against the public storage host.
|
||||
"""Return a client that signs download URLs with SigV4.
|
||||
|
||||
None means no public host is configured and the caller should presign with its own
|
||||
client, which leaves deployments on real S3 with the URL they get today.
|
||||
It is used when a public or internal storage host is configured, or when the bucket's
|
||||
region is: boto3 otherwise presigns S3 URLs with SigV2, which S3 rejects for SSE-KMS
|
||||
objects. None means none of those is set and the caller should presign with its own
|
||||
client, which leaves those deployments with the URL they get today.
|
||||
|
||||
The public endpoint wins when both are set: the internal endpoint may only be reachable
|
||||
from inside the cluster, and a URL signed against it would not open in a browser.
|
||||
"""
|
||||
public_endpoint = settings.DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL
|
||||
if not public_endpoint:
|
||||
endpoint = (
|
||||
settings.DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL
|
||||
or settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL
|
||||
)
|
||||
if not endpoint and not settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION:
|
||||
return None
|
||||
|
||||
# Blank keys are signed as-is (empty credential scope) instead of deferring to the
|
||||
@@ -266,9 +283,10 @@ def get_s3_presign_client():
|
||||
# SigV4 puts the region in the credential scope, and MinIO answers to us-east-1
|
||||
# unless it was told otherwise, so an empty region would sign an unusable URL.
|
||||
region_name=settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION or "us-east-1",
|
||||
endpoint_url=public_endpoint,
|
||||
endpoint_url=endpoint or None,
|
||||
# The signature covers the host, so the addressing style has to be pinned rather
|
||||
# than guessed from the endpoint: MinIO serves path-style.
|
||||
# than guessed from the endpoint: MinIO serves path-style, and on AWS it keeps the
|
||||
# regional host instead of the global one, which redirects for new buckets.
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from collections.abc import Iterable
|
||||
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Finding, MuteRule, Scan, StateChoices
|
||||
from api.models import Finding, MuteRule, Scan
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
@@ -39,17 +39,9 @@ def mute_findings_in_latest_scans(
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
mute_rule = MuteRule.objects.get(id=mute_rule_id, tenant_id=tenant_id)
|
||||
latest_scans = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id__in=provider_ids,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at", "-id")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scans = Scan.objects.filter(
|
||||
tenant_id=tenant_id, provider_id__in=provider_ids
|
||||
).latest_ids_per_provider()
|
||||
|
||||
changed_scan_ids = []
|
||||
findings_muted = 0
|
||||
|
||||
@@ -84,6 +84,7 @@ _SKIP_RECOVERY = {
|
||||
"scan-perform-scheduled",
|
||||
"attack-paths-scan-perform",
|
||||
"attack-paths-cleanup-stale-scans",
|
||||
"attack-paths-reap-orphaned-tmp-databases",
|
||||
"reconcile-orphan-tasks",
|
||||
}
|
||||
|
||||
|
||||
@@ -2700,20 +2700,37 @@ def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dic
|
||||
# refreshed). Wiping based on the older scan would zero counts the newer
|
||||
# scan just set. Skip and let the newer scan's reset task do the work; if
|
||||
# this task was delayed in the queue, that's the correct outcome.
|
||||
# `completed_at__isnull=False` is required: Postgres orders NULL first in
|
||||
# DESC, so a sibling COMPLETED scan with a missing completed_at would sort
|
||||
# as "newest" and incorrectly cause us to skip.
|
||||
#
|
||||
# The comparison must be against the newest *full-scope* scan, which is
|
||||
# what this variable has always been named after but did not use to be:
|
||||
# the query filtered nothing about scope, so any newer scan that is not
|
||||
# full-scope (an imported one, for instance) made the full-scope scan
|
||||
# skip its own cleanup and leave ephemeral resources with a stale
|
||||
# failed_findings_count permanently.
|
||||
#
|
||||
# `is_full_scope()` reads `trigger` plus the scoping keys inside
|
||||
# `scanner_args`, which is not expressible as a WHERE clause, so the
|
||||
# candidates are walked newest-first in Python until the first full-scope
|
||||
# one. The walk needs no cap: `scan` is itself a full-scope candidate, so
|
||||
# it stops at `scan` at the latest, after reading only the scans newer than
|
||||
# it. A fixed window would return None once more newer scoped scans had
|
||||
# landed than it inspected, and skip the cleanup exactly like the bug above.
|
||||
#
|
||||
# NULL `completed_at` no longer needs an explicit filter here: the shared
|
||||
# ordering in `ScanQuerySet.LATEST_ORDER_BY` sorts NULLs last
|
||||
# rather than excluding them, which also fixes the case where a provider
|
||||
# whose completed scans all have a NULL `completed_at` resolved to None and
|
||||
# therefore never ran the reset at all.
|
||||
with rls_transaction(tenant_id):
|
||||
latest_full_scope_scan_id = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=scan.provider_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("-completed_at", "-inserted_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
candidates = (
|
||||
Scan.objects.filter(tenant_id=tenant_id, provider_id=scan.provider_id)
|
||||
.latest_first()
|
||||
.only("id", "trigger", "scanner_args")
|
||||
.iterator(chunk_size=100)
|
||||
)
|
||||
latest_full_scope_scan_id = next(
|
||||
(candidate.id for candidate in candidates if candidate.is_full_scope()),
|
||||
None,
|
||||
)
|
||||
if latest_full_scope_scan_id != scan.id:
|
||||
logger.info(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -42,6 +43,7 @@ from tasks.jobs.attack_paths import (
|
||||
)
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
from tasks.jobs.backfill import (
|
||||
aggregate_scan_category_summaries,
|
||||
aggregate_scan_resource_group_summaries,
|
||||
@@ -163,13 +165,28 @@ def create_scan_task_record(
|
||||
task_id: str,
|
||||
task_name: str = "scan-perform",
|
||||
task_status: str | None = states.PENDING,
|
||||
task_kwargs: dict | None = None,
|
||||
) -> Task:
|
||||
"""Pre-create the TaskResult + Task rows for a pre-generated task id.
|
||||
|
||||
Pass ``task_kwargs`` when the response built from this record is serialized
|
||||
before the broker publish. ``task_kwargs`` is otherwise only written by the
|
||||
``before_task_publish`` signal (``api/signals.py``), and the scan publish is
|
||||
deferred to ``on_commit``, so the 202 would carry an empty ``task_args`` and
|
||||
the caller would have no way to learn the scan id it was just handed a task
|
||||
for. The publish later overwrites the field with the same kwargs as a Python
|
||||
repr; both forms decode to the same dict (``decode_celery_field``).
|
||||
"""
|
||||
if task_status is None:
|
||||
task_status = states.PENDING
|
||||
|
||||
defaults = {"status": task_status, "task_name": task_name}
|
||||
if task_kwargs is not None:
|
||||
defaults["task_kwargs"] = json.dumps(task_kwargs)
|
||||
|
||||
task_result, _ = TaskResult.objects.update_or_create(
|
||||
task_id=str(task_id),
|
||||
defaults={"status": task_status, "task_name": task_name},
|
||||
defaults=defaults,
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=str(task_id),
|
||||
@@ -711,6 +728,13 @@ def cleanup_stale_attack_paths_scans_task():
|
||||
return cleanup_stale_attack_paths_scans()
|
||||
|
||||
|
||||
@shared_task(
|
||||
name="attack-paths-reap-orphaned-tmp-databases", queue="attack-paths-scans"
|
||||
)
|
||||
def reap_orphaned_attack_paths_tmp_databases_task():
|
||||
return reap_orphaned_tmp_databases()
|
||||
|
||||
|
||||
@shared_task(name="reconcile-orphan-tasks", queue="celery")
|
||||
def reconcile_orphan_tasks_task():
|
||||
"""Periodic watchdog: recover tasks whose worker is gone (deploys, crashes)."""
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.database import TEMP_DB_PREFIX
|
||||
from api.models import AttackPathsScan, StateChoices
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from config.django.base import ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS
|
||||
|
||||
MARGIN = timedelta(hours=ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS)
|
||||
|
||||
|
||||
def _tmp_db_name(scan_uuid) -> str:
|
||||
return f"{TEMP_DB_PREFIX}{scan_uuid}"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestReapOrphanedTmpDatabases:
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_ignores_databases_without_the_temp_prefix(self, mock_list, mock_drop):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
mock_list.return_value = ["db-tenant-abc123", "system", "neo4j"]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_temp_db_with_no_scan_row_past_safety_margin(
|
||||
self, mock_list, mock_drop
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
old_scan_id = datetime_to_uuid7(
|
||||
datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
)
|
||||
database = _tmp_db_name(old_scan_id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [database]}
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_temp_db_with_no_scan_row_inside_safety_margin(
|
||||
self, mock_list, mock_drop
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
recent_scan_id = datetime_to_uuid7(datetime.now(tz=UTC) - timedelta(minutes=5))
|
||||
database = _tmp_db_name(recent_scan_id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_temp_db_with_unparseable_scan_id(self, mock_list, mock_drop):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
database = f"{TEMP_DB_PREFIX}not-a-uuid"
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_terminal_scan_past_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old_updated_at = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=old_updated_at)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [database]}
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state",
|
||||
[StateChoices.FAILED, StateChoices.CANCELLED],
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_other_terminal_states_past_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider, state
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old_updated_at = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=state,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=old_updated_at)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result["dropped_count"] == 1
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_terminal_scan_inside_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_margin_counts_from_completed_at_not_updated_at(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(
|
||||
updated_at=old, completed_at=datetime.now(tz=UTC)
|
||||
)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_scan_completed_past_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.FAILED,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(
|
||||
updated_at=old, completed_at=old
|
||||
)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [database]}
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_never_drops_an_executing_scan_regardless_of_age(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
very_old = datetime.now(tz=UTC) - timedelta(days=30)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=very_old)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_one_failed_drop_does_not_stop_the_rest_of_the_sweep(
|
||||
self, mock_list, mock_drop
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
old_time = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
failing_scan_id = datetime_to_uuid7(old_time)
|
||||
succeeding_scan_id = datetime_to_uuid7(old_time)
|
||||
failing_db = _tmp_db_name(failing_scan_id)
|
||||
succeeding_db = _tmp_db_name(succeeding_scan_id)
|
||||
mock_list.return_value = [failing_db, succeeding_db]
|
||||
mock_drop.side_effect = [Exception("boom"), None]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [succeeding_db]}
|
||||
assert mock_drop.call_count == 2
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_returns_empty_result_when_listing_databases_fails(self, mock_list):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
mock_list.side_effect = Exception("neo4j unreachable")
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_temp_db_with_random_uuid_and_no_row(self, mock_list, mock_drop):
|
||||
"""A non-UUIDv7 id with no matching row has no reliable timestamp, so it
|
||||
must be left alone rather than guessed at."""
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
database = _tmp_db_name(uuid4())
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
|
||||
class TestReapOrphanedTmpDatabasesTask:
|
||||
@patch(
|
||||
"tasks.tasks.reap_orphaned_tmp_databases",
|
||||
return_value={"dropped_count": 2, "databases": ["db-tmp-scan-a"]},
|
||||
)
|
||||
def test_task_invokes_the_reaper(self, mock_reap):
|
||||
from tasks.tasks import reap_orphaned_attack_paths_tmp_databases_task
|
||||
|
||||
result = reap_orphaned_attack_paths_tmp_databases_task.run()
|
||||
|
||||
assert result == {"dropped_count": 2, "databases": ["db-tmp-scan-a"]}
|
||||
mock_reap.assert_called_once_with()
|
||||
@@ -3,9 +3,10 @@ import uuid
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import boto3
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
from django.test import override_settings
|
||||
@@ -63,15 +64,46 @@ class TestOutputs:
|
||||
|
||||
assert mock_boto_client.call_args.kwargs["region_name"] == "us-east-1"
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000")
|
||||
def test_get_s3_client_passes_the_endpoint_when_set(self, mock_boto_client):
|
||||
get_s3_client()
|
||||
|
||||
assert mock_boto_client.call_args.kwargs["endpoint_url"] == "http://minio:9000"
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="")
|
||||
def test_get_s3_client_endpoint_empty_by_default(self, mock_boto_client):
|
||||
"""Empty keeps today's behavior: no endpoint override, real S3 is used."""
|
||||
get_s3_client()
|
||||
|
||||
assert mock_boto_client.call_args.kwargs["endpoint_url"] is None
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000")
|
||||
def test_get_s3_client_does_not_fall_back_when_endpoint_set(self, mock_boto_client):
|
||||
"""A configured endpoint means the explicit client failed talking to it. The fallback
|
||||
goes to the default provider chain (e.g. an EC2 instance role) against real AWS, so it
|
||||
must not be used: the original error propagates instead."""
|
||||
error = ClientError({"Error": {"Code": "403"}}, "ListBuckets")
|
||||
mock_boto_client.side_effect = error
|
||||
|
||||
with pytest.raises(ClientError):
|
||||
get_s3_client()
|
||||
|
||||
mock_boto_client.assert_called_once()
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@patch("tasks.jobs.export.settings")
|
||||
def test_get_s3_client_fallback(self, mock_settings, mock_boto_client):
|
||||
mock_settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL = ""
|
||||
mock_boto_client.side_effect = [
|
||||
ClientError({"Error": {"Code": "403"}}, "ListBuckets"),
|
||||
MagicMock(),
|
||||
]
|
||||
client = get_s3_client()
|
||||
assert client is not None
|
||||
assert mock_boto_client.call_args_list[1] == call("s3")
|
||||
|
||||
@patch("tasks.jobs.export.get_s3_client")
|
||||
@patch("tasks.jobs.export.base")
|
||||
@@ -278,10 +310,70 @@ def _presign(client):
|
||||
|
||||
|
||||
class TestS3PresignClient:
|
||||
@override_settings(**PRESIGN_SETTINGS, DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="")
|
||||
def test_no_public_endpoint_returns_none(self):
|
||||
@override_settings(
|
||||
**{**PRESIGN_SETTINGS, "DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION": ""},
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
|
||||
)
|
||||
def test_no_public_endpoint_and_no_region_returns_none(self):
|
||||
# Without a region, SigV4 would have to guess one and break other regions.
|
||||
assert get_s3_presign_client() is None
|
||||
|
||||
@override_settings(**PRESIGN_SETTINGS, DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="")
|
||||
def test_region_without_public_endpoint_signs_sigv4_on_the_regional_host(self):
|
||||
# SSE-KMS objects reject the SigV2 URLs boto3 presigns by default, and the
|
||||
# global host redirects for new buckets, which breaks a SigV4 signature.
|
||||
url = urlparse(_presign(get_s3_presign_client()))
|
||||
query = parse_qs(url.query)
|
||||
|
||||
assert url.netloc == "s3.eu-west-1.amazonaws.com"
|
||||
assert url.path == "/output-bucket/tenant/scan/report.zip"
|
||||
assert query["X-Amz-Algorithm"] == ["AWS4-HMAC-SHA256"]
|
||||
assert "/eu-west-1/s3/aws4_request" in query["X-Amz-Credential"][0]
|
||||
|
||||
@override_settings(
|
||||
**{
|
||||
**PRESIGN_SETTINGS,
|
||||
"DJANGO_OUTPUT_S3_AWS_ACCESS_KEY_ID": "",
|
||||
"DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY": "",
|
||||
},
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
|
||||
)
|
||||
def test_region_without_static_keys_signs_with_the_default_chain(self, monkeypatch):
|
||||
# An ECS task role reaches boto3 through the default chain, like the env here.
|
||||
# A fresh default session keeps these keys from being cached for later tests.
|
||||
monkeypatch.setattr(boto3, "DEFAULT_SESSION", None)
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "role-access-key")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "role-secret-key")
|
||||
monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
|
||||
|
||||
query = parse_qs(urlparse(_presign(get_s3_presign_client())).query)
|
||||
|
||||
assert query["X-Amz-Credential"][0].startswith("role-access-key/")
|
||||
assert "/eu-west-1/s3/aws4_request" in query["X-Amz-Credential"][0]
|
||||
|
||||
@override_settings(
|
||||
**{**PRESIGN_SETTINGS, "DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION": ""},
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000",
|
||||
)
|
||||
def test_internal_endpoint_without_public_endpoint_signs_against_it(self):
|
||||
# No browser-reachable host was configured, so the internal one is the best
|
||||
# available target instead of falling through to the real AWS host.
|
||||
url = urlparse(_presign(get_s3_presign_client()))
|
||||
|
||||
assert url.netloc == "minio:9000"
|
||||
assert url.path == "/output-bucket/tenant/scan/report.zip"
|
||||
|
||||
@override_settings(
|
||||
**PRESIGN_SETTINGS,
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="https://storage.example.com",
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000",
|
||||
)
|
||||
def test_public_endpoint_wins_over_the_internal_endpoint(self):
|
||||
url = urlparse(_presign(get_s3_presign_client()))
|
||||
|
||||
assert url.netloc == "storage.example.com"
|
||||
|
||||
@override_settings(
|
||||
**PRESIGN_SETTINGS,
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="https://storage.example.com",
|
||||
|
||||
@@ -5967,6 +5967,112 @@ class TestResetEphemeralResourceFindingsCount:
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 5
|
||||
|
||||
def test_runs_when_newer_scan_is_not_full_scope(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
"""A newer scoped scan must not block the full-scope scan's cleanup.
|
||||
|
||||
The race guard used to pick the newest COMPLETED scan of any scope
|
||||
despite being named after full-scope ones, so a single scoped scan
|
||||
landing after a complete one made the complete scan skip its cleanup
|
||||
and leave ephemeral resources with a stale count permanently.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
newer_completed_at = scan1.completed_at + timedelta(minutes=5)
|
||||
Scan.objects.create(
|
||||
name="Newer scoped scan",
|
||||
provider=aws_provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
started_at=newer_completed_at,
|
||||
completed_at=newer_completed_at,
|
||||
scanner_args={"checks": ["check1"]},
|
||||
)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_runs_when_many_newer_scans_are_not_full_scope(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
"""The walk must not give up before it reaches the full-scope scan.
|
||||
|
||||
A fixed look-back window returned None once more newer scoped scans
|
||||
had landed than it inspected, and skipped the cleanup exactly like the
|
||||
original bug did with one.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
for minutes in range(1, 41):
|
||||
newer_completed_at = scan1.completed_at + timedelta(minutes=minutes)
|
||||
Scan.objects.create(
|
||||
name=f"Newer scoped scan {minutes}",
|
||||
provider=aws_provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
started_at=newer_completed_at,
|
||||
completed_at=newer_completed_at,
|
||||
scanner_args={"checks": ["check1"]},
|
||||
)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_runs_when_completed_at_is_null(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
"""NULL `completed_at` used to make the reset never run at all.
|
||||
|
||||
The old guard filtered `completed_at__isnull=False`, so a provider
|
||||
whose completed scans all had a NULL `completed_at` resolved the
|
||||
"latest" scan to None, which never equals `scan.id`.
|
||||
"""
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Scan.all_objects.filter(id=scan1.id).update(completed_at=None)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_does_not_touch_other_providers_resources(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
@@ -14,6 +15,7 @@ from api.models import (
|
||||
StateChoices,
|
||||
Task,
|
||||
)
|
||||
from api.v1.serializers import TaskSerializer
|
||||
from botocore.exceptions import ClientError
|
||||
from celery import states
|
||||
from django_celery_beat.models import IntervalSchedule, PeriodicTask
|
||||
@@ -32,6 +34,7 @@ from tasks.tasks import (
|
||||
_scan_tmp_output_directory,
|
||||
check_integrations_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
create_scan_task_record,
|
||||
generate_outputs_task,
|
||||
mute_findings_in_latest_scans_task,
|
||||
perform_attack_paths_scan_task,
|
||||
@@ -3384,3 +3387,79 @@ class TestTaskTimeLimits:
|
||||
"lighthouse-provider-connection-check",
|
||||
):
|
||||
assert celery_app.tasks[name].time_limit < default
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCreateScanTaskRecord:
|
||||
"""`task_kwargs` is what a response built before the publish can report."""
|
||||
|
||||
def _scan(self, tenant, provider):
|
||||
"""A manual scan, like the one `POST /api/v1/scans` creates."""
|
||||
return Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
|
||||
def _publish_kwargs(self, tenant, scan):
|
||||
"""What `enqueue_scan_execution_on_commit` publishes for this scan."""
|
||||
return {
|
||||
"tenant_id": str(tenant.id),
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
}
|
||||
|
||||
def _task_args(self, task):
|
||||
"""Read the record back the way `TaskSerializer` does."""
|
||||
return TaskSerializer(task).data["task_args"]
|
||||
|
||||
def test_the_stored_kwargs_are_the_ones_the_publish_would_send(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""The 202 reports what is stored here, so it has to be the dispatch kwargs."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = self._scan(tenant, aws_provider)
|
||||
|
||||
task = create_scan_task_record(
|
||||
tenant_id=str(tenant.id),
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_kwargs=self._publish_kwargs(tenant, scan),
|
||||
)
|
||||
|
||||
assert self._task_args(task) == {
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(aws_provider.id),
|
||||
}
|
||||
|
||||
def test_a_record_created_without_kwargs_reports_none(self, tenants_fixture):
|
||||
"""The argument is optional, so the other callers keep their behaviour."""
|
||||
task = create_scan_task_record(
|
||||
tenant_id=str(tenants_fixture[0].id),
|
||||
task_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
assert self._task_args(task) == {}
|
||||
|
||||
def test_the_publish_can_overwrite_the_stored_kwargs(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""django-celery-results stores a Python repr; both must decode alike."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = self._scan(tenant, aws_provider)
|
||||
task_id = str(uuid.uuid4())
|
||||
kwargs = self._publish_kwargs(tenant, scan)
|
||||
|
||||
task = create_scan_task_record(
|
||||
tenant_id=str(tenant.id), task_id=task_id, task_kwargs=kwargs
|
||||
)
|
||||
before = self._task_args(task)
|
||||
|
||||
# What `before_task_publish` writes once the task reaches the broker.
|
||||
task_result = TaskResult.objects.get(task_id=task_id)
|
||||
task_result.task_kwargs = json.dumps(repr(kwargs))
|
||||
task_result.save(update_fields=["task_kwargs"])
|
||||
task.refresh_from_db()
|
||||
|
||||
assert self._task_args(task) == before
|
||||
|
||||
Generated
+5
-5
@@ -53,7 +53,7 @@ constraints = [
|
||||
{ name = "aliyun-log-fastpb", specifier = "==0.2.0" },
|
||||
{ name = "amqp", specifier = "==5.3.1" },
|
||||
{ name = "annotated-types", specifier = "==0.7.0" },
|
||||
{ name = "anyio", specifier = "==4.12.1" },
|
||||
{ name = "anyio", specifier = "==4.14.2" },
|
||||
{ name = "applicationinsights", specifier = "==0.11.10" },
|
||||
{ name = "apscheduler", specifier = "==3.11.2" },
|
||||
{ name = "argcomplete", specifier = "==3.5.3" },
|
||||
@@ -969,15 +969,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.12.1"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "idna" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4938,7 +4938,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.44.0"
|
||||
version = "1.45.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
api:
|
||||
hostname: "prowler-api"
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
@@ -44,6 +45,7 @@ services:
|
||||
|
||||
ui:
|
||||
image: prowlercloud/prowler-ui:${PROWLER_UI_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
@@ -61,6 +63,7 @@ services:
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
|
||||
restart: unless-stopped
|
||||
hostname: "postgres-db"
|
||||
volumes:
|
||||
- ./_data/postgres:/var/lib/postgresql/data
|
||||
@@ -81,6 +84,7 @@ services:
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8-alpine@sha256:a038175878d66b9d274fbf8be73c0305e93798b83917647f167e18cef3c71eec
|
||||
restart: unless-stopped
|
||||
hostname: "valkey"
|
||||
volumes:
|
||||
- ./_data/valkey:/data
|
||||
@@ -97,6 +101,7 @@ services:
|
||||
|
||||
neo4j:
|
||||
image: graphstack/dozerdb:5.26.27.0@sha256:9b54d6b3a98a76c00bd23e8e78d8c82081ff168162aebd47b25c234e092cb0a0
|
||||
restart: unless-stopped
|
||||
hostname: "neo4j"
|
||||
volumes:
|
||||
- ./_data/neo4j:/data
|
||||
@@ -129,6 +134,7 @@ services:
|
||||
|
||||
worker:
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
|
||||
stop_grace_period: 120s
|
||||
env_file:
|
||||
@@ -149,6 +155,7 @@ services:
|
||||
|
||||
worker-beat:
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- path: ./.env
|
||||
required: false
|
||||
@@ -165,6 +172,7 @@ services:
|
||||
|
||||
mcp-server:
|
||||
image: prowlercloud/prowler-mcp:${PROWLER_MCP_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PROWLER_MCP_TRANSPORT_MODE=http
|
||||
env_file:
|
||||
|
||||
@@ -4,6 +4,73 @@ description: "Track new features, improvements, provider updates, and bug fixes
|
||||
rss: true
|
||||
---
|
||||
|
||||
<Update label="v5.43.0" description="September 21, 2026">
|
||||
### 🏛️ Compliance — FedRAMP 20x Consolidated Rules 2026
|
||||
|
||||
The FedRAMP 20x Phase One pilot frameworks (`fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp`) are replaced by two universal frameworks built from the FedRAMP Consolidated Rules for 2026, each covering AWS, Azure, GCP, Kubernetes and Microsoft 365 from a single definition:
|
||||
|
||||
- **FedRAMP 20x KSI** (`fedramp_20x_ksi_2026`): the 46 Key Security Indicators across 10 themes. There is one indicator catalog for every class instead of a separate Low fork; each indicator carries its class applicability and NIST SP 800-53 controls.
|
||||
- **FedRAMP 20x Class C FRR** (`fedramp_20x_frr_class_c_2026`): the 158 FedRAMP Rules of the Class C ruleset that bind cloud service providers. Most are program obligations (reports, notifications, certification package) and stay manual; checks are mapped only where they evidence part of the rule text. Configurable checks carry configuration requirements, so a relaxed `audit_config` cannot turn a requirement green.
|
||||
|
||||
Automation or stored results that reference the pilot framework IDs need to move to `fedramp_20x_ksi_2026`. In Prowler App, both frameworks show the per-provider breakdown in the cross-provider compliance view and can be downloaded as OCSF.
|
||||
|
||||
Read more in the [Compliance documentation](https://docs.prowler.com/user-guide/compliance/tutorials/compliance).
|
||||
|
||||
### 🔎 AWS — Inspector Coverage, CISA KEV and FIPS Checks
|
||||
|
||||
Seven new AWS checks back the vulnerability detection and cryptography rules of FedRAMP 20x Class C:
|
||||
|
||||
- `inspector2_coverage_scan_status_active` and `inspector2_coverage_recently_scanned` report resources Amazon Inspector is not scanning, or last scanned more than `inspector2_max_days_since_last_scan` days ago (default 3).
|
||||
- `inspector2_active_findings_no_known_exploited_vulnerabilities` and `inspector2_active_findings_kev_within_due_date` report active findings whose CVE is in the CISA Known Exploited Vulnerabilities catalog, and those still open past the CISA due date. The KEV data comes from Inspector itself through `inspector2:BatchGetFindingDetails`, so no external feed is needed.
|
||||
- `inspector2_active_findings_within_max_age` reports active findings first observed more than `inspector2_active_finding_max_age_days` days ago (default 192).
|
||||
- `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` report HTTPS/TLS load balancer listeners and Transfer Family servers without a FIPS security policy.
|
||||
|
||||
`inspector2:BatchGetFindingDetails` is not part of `SecurityAudit`, so it is now included in the Prowler additions policy and the CloudFormation scan role. Without it, the KEV checks report `MANUAL` naming the missing permission instead of a false `FAIL`.
|
||||
|
||||
Explore all AWS checks at [Prowler Hub](https://hub.prowler.com/check?provider=aws).
|
||||
|
||||
### ☁️ AWS — Partition Bootstrap Failover
|
||||
|
||||
When `PROWLER_AWS_PARTITION` is set, the bootstrap STS calls (validating credentials, assuming a role and getting an MFA session token) now try up to two more regions of the partition if the first one cannot be reached. A GovCloud host whose configured region belongs to another partition was still sent to `us-gov-east-1`, and on a network that routes only to `us-gov-west-1` the connection check and the scan failed on perfectly valid credentials. Only connection errors and timeouts move on to the next region; credential errors are reported from the first one as before. Later STS calls reuse the region that answered, and nothing changes when `PROWLER_AWS_PARTITION` is unset.
|
||||
|
||||
Read more in the [AWS Regions and Partitions documentation](https://docs.prowler.com/user-guide/providers/aws/regions-and-partitions).
|
||||
|
||||
### 🌐 Azure — Sovereign Cloud Endpoints for Defender and Key Vault
|
||||
|
||||
Defender security contacts and Key Vault key rotation policies now use the endpoints of the cloud selected with `--azure-region` instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud`. Key Vault clients are built from the vault URI that Azure returns for each vault.
|
||||
|
||||
Read more in the [Azure non-default cloud documentation](https://docs.prowler.com/user-guide/providers/azure/use-non-default-cloud).
|
||||
|
||||
### ✉️ Invitations — Expired Invitations No Longer Block Re-Invites
|
||||
|
||||
A pending invitation past its expiry date is now reported as expired, and inviting the same email again marks it as expired and creates the new invitation instead of returning a generic error. The Invitations table disables Edit and Revoke on expired and revoked invitations, and `filter[state__in]` on the invitations endpoint no longer returns a server error.
|
||||
|
||||
In Prowler Cloud, new organizations are offered an **Invite your team** step once the first provider is connected, and Prowler Private Cloud deployments can set `UI_SELF_REGISTRATION_ENABLED=false` to make sign-up invitation-only.
|
||||
|
||||
Read more in the [Invitations documentation](https://docs.prowler.com/user-guide/tutorials/prowler-app-rbac#invitations).
|
||||
|
||||
### 📄 Reports — Downloads on Self-Hosted Storage
|
||||
|
||||
Report downloads no longer depend on the storage host being the same inside and outside the container network. `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL` signs the download URL against a browser-reachable host, so a deployment whose object storage answers only on an internal address serves the file instead of a link the browser cannot open. Leaving `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` unset, common on S3-compatible storage with no meaningful region, no longer makes the download fail with a server error.
|
||||
|
||||
### 🔍 Checks
|
||||
|
||||
- **Huawei Cloud:** new `smn_topic_subscriptions` check reports SMN topics without any subscription.
|
||||
- **Cloudflare:** the API token links in the provider wizard request the SSL and Certificates, Bot Management and Zone WAF read permissions the checks need, so a token created from the wizard no longer produces failures on permissions it was never granted.
|
||||
- **Microsoft 365:** five Defender malware, anti-phishing and inbound anti-spam checks no longer fail with `KeyError` on tenants that use the Standard or Strict preset security policies, which dropped every finding of those checks. Preset policies are covered by `defender_strict_preset_security_policy_enabled`.
|
||||
- **Google Workspace:** `security_2sv_enforced` reports domain-wide 2-Step Verification failures as `FAIL` even when every failing setting is overridden for a group or organizational unit.
|
||||
|
||||
Explore all checks at [Prowler Hub](https://hub.prowler.com/check).
|
||||
|
||||
### 🔐 Security Updates
|
||||
|
||||
- `libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK and API container images, patching high-severity Debian CVEs.
|
||||
- PowerShell upgraded to 7.5.11 in the SDK and API container images, bundling .NET runtime 9.0.20 and patching CVE-2026-62901.
|
||||
- `anyio` upgraded to 4.14.2 in the SDK, the API and the MCP Server, patching CVE-2026-63374.
|
||||
|
||||
See the [full release notes on GitHub](https://github.com/prowler-cloud/prowler/releases/tag/5.43.0) for the complete list of changes.
|
||||
</Update>
|
||||
|
||||
<Update label="v5.42.0" description="September 11, 2026">
|
||||
### ☁️ AWS — ISO Partitions
|
||||
|
||||
|
||||
@@ -129,8 +129,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.42.0"
|
||||
PROWLER_API_VERSION="5.42.0"
|
||||
PROWLER_UI_VERSION="5.43.0"
|
||||
PROWLER_API_VERSION="5.43.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -76,7 +76,7 @@ Prowler supports a wide range of providers organized by category:
|
||||
| [LLM](/user-guide/providers/llm/getting-started-llm) | Official | Models | CLI |
|
||||
| [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | Tenants | UI, API, CLI |
|
||||
| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | Organizations | UI, API, CLI |
|
||||
| [Okta](/user-guide/providers/okta/getting-started-okta) | Official | Organizations | CLI |
|
||||
| [Okta](/user-guide/providers/okta/getting-started-okta) | Official | Organizations | UI, API, CLI |
|
||||
| [Vercel](/user-guide/providers/vercel/getting-started-vercel) | Official | Teams / Projects | UI, API, CLI |
|
||||
|
||||
### Kubernetes
|
||||
|
||||
@@ -30,6 +30,22 @@ Boto3 defaults both timeouts to 60 seconds. In networks with restricted egress (
|
||||
|
||||
</Note>
|
||||
|
||||
## Retries Configuration
|
||||
|
||||
<VersionBadge version="5.44.0" />
|
||||
|
||||
The number of retries is set with `--aws-retries-max-attempts`, where `0` disables retries. It can also be set through an environment variable, which is the way to tune it in Prowler Cloud and other deployments without a CLI:
|
||||
|
||||
```console
|
||||
export PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS=0
|
||||
```
|
||||
|
||||
The CLI flag takes precedence over the environment variable. The value must be a non-negative integer; when neither is set, Prowler uses 3 retries.
|
||||
|
||||
<Warning>
|
||||
The environment variable is process-wide: it applies to every AWS provider built in the process where it is set, not only to a connection check. A scan started in that same process picks it up too. Boto3's Standard retry mode, which Prowler uses, also retries service-side throttling responses (see the errors listed below), so `0` disables retries for those as well. On a large account a scan can hit throttling under normal load, and with retries disabled that throttling becomes a hard failure instead of a retried call. Set the variable only on the processes that run connection checks; leave scan workers on the default, or raise their retry count instead of lowering it.
|
||||
</Warning>
|
||||
|
||||
## Retry Behavior Overview
|
||||
|
||||
Boto3's Standard retry mode includes the following mechanisms:
|
||||
|
||||
@@ -89,6 +89,10 @@ After adding your cloud account credentials, click the `Check connection` button
|
||||
|
||||
<img src="/images/test-connection-button.png" alt="Test Connection" width="700" />
|
||||
|
||||
<Note>
|
||||
For a single AWS account, Prowler tests the connection as part of the `Connect account` step, so the wizard moves straight to launching the scan.
|
||||
</Note>
|
||||
|
||||
## Step 6: Scan Started
|
||||
After the connection check succeeds, save the provider and start your first scan with the `Launch Scan` button. The `Scans` section shows the scan in progress:
|
||||
|
||||
|
||||
@@ -4,6 +4,14 @@ All notable changes to the **Prowler MCP Server** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [0.12.2] (Prowler v5.43.0)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- Bumped `anyio` to 4.14.2 to resolve CVE-2026-63374 [(#12848)](https://github.com/prowler-cloud/prowler/pull/12848)
|
||||
|
||||
---
|
||||
|
||||
## [0.12.1] (Prowler v5.42.0)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
Generated
+3
-3
@@ -39,15 +39,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.13.0"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "idna" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -4,6 +4,34 @@ All notable changes to the **Prowler SDK** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [5.43.0] (Prowler v5.43.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `FedRAMP-20x-KSI` universal compliance framework (`fedramp_20x_ksi_2026`) with the 46 Key Security Indicators from the FedRAMP Consolidated Rules 2026 mapped for AWS, Azure, GCP, Kubernetes and M365 [(#11701)](https://github.com/prowler-cloud/prowler/pull/11701)
|
||||
- `smn_topic_subscriptions` check for Huawei Cloud provider: SMN topics have at least one subscription configured [(#12186)](https://github.com/prowler-cloud/prowler/pull/12186)
|
||||
- `inspector2_coverage_scan_status_active`, `inspector2_coverage_recently_scanned`, `inspector2_active_findings_no_known_exploited_vulnerabilities`, `inspector2_active_findings_kev_within_due_date`, `inspector2_active_findings_within_max_age`, `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` checks for AWS provider, covering FedRAMP 20x Class C vulnerability detection, CISA KEV remediation and FIPS cryptography rules; the KEV checks require `inspector2:BatchGetFindingDetails`, now in the Prowler additions policy [(#12808)](https://github.com/prowler-cloud/prowler/pull/12808)
|
||||
- `FedRAMP-20x-FRR-Class-C` universal compliance framework (`fedramp_20x_frr_class_c_2026`) with the 158 provider rules of the FedRAMP 20x Class C ruleset from the FedRAMP Consolidated Rules 2026 for AWS, Azure, GCP, Kubernetes and M365 [(#12808)](https://github.com/prowler-cloud/prowler/pull/12808)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- FedRAMP 20x Phase One pilot frameworks `fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp` replaced by `fedramp_20x_ksi_2026` [(#12855)](https://github.com/prowler-cloud/prowler/pull/12855)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `security_2sv_enforced` reports domain-wide 2-Step Verification failures as FAIL even when every failing setting is overridden for a group or organizational unit [(#12700)](https://github.com/prowler-cloud/prowler/pull/12700)
|
||||
- Bootstrap STS calls now try up to two more regions of the partition declared in `PROWLER_AWS_PARTITION` when the first one cannot be reached, so a deployment that routes to only one region of its partition no longer fails on an endpoint it has no path to. This covers validating credentials, assuming a role and getting an MFA session token [(#12799)](https://github.com/prowler-cloud/prowler/pull/12799)
|
||||
- `KeyError` in M365 Defender malware, anti-phishing and inbound anti-spam checks when the tenant has Standard or Strict preset security policies [(#12809)](https://github.com/prowler-cloud/prowler/pull/12809)
|
||||
- Azure Defender security contacts and Key Vault key rotation policies now use the endpoints of the selected cloud (`--azure-region`) instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud` [(#12813)](https://github.com/prowler-cloud/prowler/pull/12813)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK container image, patching nine high Debian CVEs [(#12804)](https://github.com/prowler-cloud/prowler/pull/12804)
|
||||
- PowerShell from 7.5.9 to 7.5.11 in the SDK container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901 [(#12811)](https://github.com/prowler-cloud/prowler/pull/12811)
|
||||
- Bumped `anyio` to 4.14.2 to resolve CVE-2026-63374 [(#12848)](https://github.com/prowler-cloud/prowler/pull/12848)
|
||||
|
||||
---
|
||||
|
||||
## [5.42.0] (Prowler v5.42.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
`inspector2_coverage_scan_status_active`, `inspector2_coverage_recently_scanned`, `inspector2_active_findings_no_known_exploited_vulnerabilities`, `inspector2_active_findings_kev_within_due_date`, `inspector2_active_findings_within_max_age`, `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` checks for AWS provider, covering FedRAMP 20x Class C vulnerability detection, CISA KEV remediation and FIPS cryptography rules; the KEV checks require `inspector2:BatchGetFindingDetails`, now in the Prowler additions policy
|
||||
@@ -1 +0,0 @@
|
||||
Bootstrap STS calls now try up to two more regions of the partition declared in `PROWLER_AWS_PARTITION` when the first one cannot be reached, so a deployment that routes to only one region of its partition no longer fails on an endpoint it has no path to. This covers validating credentials, assuming a role and getting an MFA session token
|
||||
@@ -0,0 +1 @@
|
||||
`PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS` environment variable to set the Boto3 retries for deployments without CLI flags
|
||||
@@ -0,0 +1 @@
|
||||
STS calls after role assumption use the answering region, avoiding a second wait for an unreachable partition region
|
||||
@@ -1 +0,0 @@
|
||||
Azure Defender security contacts and Key Vault key rotation policies now use the endpoints of the selected cloud (`--azure-region`) instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud`
|
||||
@@ -1 +0,0 @@
|
||||
`FedRAMP-20x-FRR-Class-C` universal compliance framework (`fedramp_20x_frr_class_c_2026`) with the 158 provider rules of the FedRAMP 20x Class C ruleset from the FedRAMP Consolidated Rules 2026 for AWS, Azure, GCP, Kubernetes and M365
|
||||
@@ -1 +0,0 @@
|
||||
`FedRAMP-20x-KSI` universal compliance framework (`fedramp_20x_ksi_2026`) with the 46 Key Security Indicators from the FedRAMP Consolidated Rules 2026 mapped for AWS, Azure, GCP, Kubernetes and M365
|
||||
@@ -1 +0,0 @@
|
||||
`fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp` FedRAMP 20x Phase One pilot frameworks, superseded by `fedramp_20x_ksi_2026`
|
||||
@@ -1 +0,0 @@
|
||||
`security_2sv_enforced` reports domain-wide 2-Step Verification failures as FAIL even when every failing setting is overridden for a group or organizational unit
|
||||
@@ -1 +0,0 @@
|
||||
`KeyError` in M365 Defender malware, anti-phishing and inbound anti-spam checks when the tenant has Standard or Strict preset security policies
|
||||
@@ -1 +0,0 @@
|
||||
`libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK container image, patching nine high Debian CVEs
|
||||
@@ -1 +0,0 @@
|
||||
PowerShell from 7.5.9 to 7.5.11 in the SDK container image, bundling .NET runtime 9.0.20 and patching CVE-2026-62901
|
||||
@@ -1 +0,0 @@
|
||||
`smn_topic_subscriptions` check for Huawei Cloud provider: SMN topics have at least one subscription configured
|
||||
@@ -0,0 +1 @@
|
||||
Pass the E2E AWS credentials to the UI E2E workflow through environment variables instead of template expansion
|
||||
@@ -52,7 +52,7 @@ class _MutableTimestamp:
|
||||
|
||||
timestamp = _MutableTimestamp(datetime.today())
|
||||
timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc))
|
||||
prowler_version = "5.43.0"
|
||||
prowler_version = "5.44.0"
|
||||
html_logo_url = "https://github.com/prowler-cloud/prowler/"
|
||||
square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png"
|
||||
aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png"
|
||||
|
||||
@@ -112,7 +112,7 @@ class AwsProvider(Provider):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
role_arn: str = None,
|
||||
session_duration: int = 3600,
|
||||
external_id: str = None,
|
||||
@@ -141,6 +141,7 @@ class AwsProvider(Provider):
|
||||
|
||||
Args:
|
||||
- retries_max_attempts: The maximum number of retries for the AWS client.
|
||||
Defaults to the PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS environment variable or, if unset, to 3.
|
||||
- role_arn: The ARN of the IAM role to assume.
|
||||
- session_duration: The duration of the session in seconds, between 900 and 43200.
|
||||
- external_id: The external ID to use when assuming the IAM role.
|
||||
@@ -1230,7 +1231,8 @@ class AwsProvider(Provider):
|
||||
|
||||
Args:
|
||||
- session: The AWS session object
|
||||
- assumed_role_info: The AWSAssumeRoleInfo object
|
||||
- assumed_role_info: The AWSAssumeRoleInfo object. Its sts_region is
|
||||
updated to the region that answered, so later calls go straight there
|
||||
|
||||
Returns:
|
||||
- AWSCredentials: The AWS credentials for the assumed role
|
||||
@@ -1256,11 +1258,14 @@ class AwsProvider(Provider):
|
||||
mfa_info = AwsProvider.input_role_mfa_token_and_code()
|
||||
assume_role_arguments["SerialNumber"] = mfa_info.arn
|
||||
assume_role_arguments["TokenCode"] = mfa_info.totp
|
||||
_, assumed_credentials = AwsProvider.sts_call_with_partition_failover(
|
||||
session,
|
||||
assumed_role_info.sts_region,
|
||||
lambda sts_client: sts_client.assume_role(**assume_role_arguments),
|
||||
sts_region, assumed_credentials = (
|
||||
AwsProvider.sts_call_with_partition_failover(
|
||||
session,
|
||||
assumed_role_info.sts_region,
|
||||
lambda sts_client: sts_client.assume_role(**assume_role_arguments),
|
||||
)
|
||||
)
|
||||
assumed_role_info.sts_region = sts_region
|
||||
# Convert the UTC datetime object to your local timezone
|
||||
credentials_expiration_local_time = (
|
||||
assumed_credentials["Credentials"]["Expiration"]
|
||||
@@ -1558,6 +1563,8 @@ class AwsProvider(Provider):
|
||||
session,
|
||||
assumed_role_information,
|
||||
)
|
||||
# Validate where the role was assumed, not where it timed out
|
||||
aws_region = assumed_role_information.sts_region
|
||||
session = Session(
|
||||
aws_access_key_id=assumed_role_credentials.aws_access_key_id,
|
||||
aws_secret_access_key=assumed_role_credentials.aws_secret_access_key,
|
||||
|
||||
@@ -122,6 +122,21 @@
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"account-access": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-southeast-1",
|
||||
"us-west-1"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"acm": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
@@ -242,6 +257,24 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"agent-registry": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-northeast-1",
|
||||
"ap-southeast-2",
|
||||
"eu-west-1",
|
||||
"us-east-1",
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"ahl": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
@@ -266,14 +299,20 @@
|
||||
"aiops": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-east-1",
|
||||
"ap-northeast-1",
|
||||
"ap-south-1",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-5",
|
||||
"ap-southeast-7",
|
||||
"eu-central-1",
|
||||
"eu-north-1",
|
||||
"eu-south-2"
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
@@ -973,12 +1012,15 @@
|
||||
"aws": [
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-5",
|
||||
"ca-central-1",
|
||||
"ca-west-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
@@ -1545,37 +1587,11 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"awstransform": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-south-1",
|
||||
"ap-southeast-2",
|
||||
"ca-central-1",
|
||||
"eu-central-1",
|
||||
"eu-west-2",
|
||||
"sa-east-1",
|
||||
"us-east-1"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"b2bi": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-south-2",
|
||||
"ap-southeast-2",
|
||||
"ca-central-1",
|
||||
"eu-central-1",
|
||||
"eu-west-1",
|
||||
"eu-west-3",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
"us-west-2"
|
||||
@@ -1889,6 +1905,7 @@
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-5",
|
||||
@@ -1904,6 +1921,7 @@
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
"us-west-1",
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
@@ -4202,16 +4220,33 @@
|
||||
"dax": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"af-south-1",
|
||||
"ap-east-1",
|
||||
"ap-east-2",
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-4",
|
||||
"ap-southeast-5",
|
||||
"ap-southeast-6",
|
||||
"ap-southeast-7",
|
||||
"ca-central-1",
|
||||
"ca-west-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-north-1",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"il-central-1",
|
||||
"mx-central-1",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
@@ -6006,8 +6041,10 @@
|
||||
"evs": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-east-2",
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
@@ -6019,9 +6056,11 @@
|
||||
"eu-central-2",
|
||||
"eu-north-1",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"il-central-1",
|
||||
"mx-central-1",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
@@ -9218,9 +9257,12 @@
|
||||
"aws": [
|
||||
"af-south-1",
|
||||
"ap-east-1",
|
||||
"ap-east-2",
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-5",
|
||||
@@ -9228,12 +9270,17 @@
|
||||
"ca-central-1",
|
||||
"ca-west-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-north-1",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"il-central-1",
|
||||
"me-central-1",
|
||||
"me-south-1",
|
||||
"mx-central-1",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
@@ -9965,7 +10012,9 @@
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
"aws-us-gov": [
|
||||
"us-gov-west-1"
|
||||
]
|
||||
}
|
||||
},
|
||||
"neptune": {
|
||||
@@ -10427,7 +10476,9 @@
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-eusc": [
|
||||
"eusc-de-east-1"
|
||||
],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
@@ -10902,7 +10953,10 @@
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
"aws-us-gov": [
|
||||
"us-gov-east-1",
|
||||
"us-gov-west-1"
|
||||
]
|
||||
}
|
||||
},
|
||||
"pca-connector-scep": {
|
||||
@@ -10964,6 +11018,8 @@
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-4",
|
||||
"ap-southeast-6",
|
||||
"eu-central-1",
|
||||
"eu-north-1",
|
||||
"eu-south-1",
|
||||
@@ -13414,6 +13470,21 @@
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"securityagent": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"us-east-1",
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"securityhub": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
@@ -13877,6 +13948,7 @@
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-4",
|
||||
"ap-southeast-5",
|
||||
"ap-southeast-6",
|
||||
"ap-southeast-7",
|
||||
"ca-central-1",
|
||||
"ca-west-1",
|
||||
@@ -14839,6 +14911,18 @@
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"supportauthz": {
|
||||
"regions": {
|
||||
"aws": [],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"sustainability": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
@@ -15062,8 +15146,10 @@
|
||||
"aws": [
|
||||
"ap-northeast-1",
|
||||
"ap-south-1",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"eu-central-1",
|
||||
"eu-north-1",
|
||||
"eu-west-1",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
@@ -15083,20 +15169,28 @@
|
||||
"timestream-influxdb": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"af-south-1",
|
||||
"ap-east-1",
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-4",
|
||||
"ap-southeast-7",
|
||||
"ca-central-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-north-1",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"il-central-1",
|
||||
"me-central-1",
|
||||
"mx-central-1",
|
||||
"sa-east-1",
|
||||
@@ -15162,29 +15256,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"tnb": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"ap-northeast-2",
|
||||
"ap-southeast-2",
|
||||
"ca-central-1",
|
||||
"eu-central-1",
|
||||
"eu-north-1",
|
||||
"eu-south-2",
|
||||
"eu-west-3",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
"aws-iso": [],
|
||||
"aws-iso-b": [],
|
||||
"aws-iso-e": [],
|
||||
"aws-iso-f": [],
|
||||
"aws-us-gov": []
|
||||
}
|
||||
},
|
||||
"transcribe": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
@@ -15437,7 +15508,38 @@
|
||||
"uxc": {
|
||||
"regions": {
|
||||
"aws": [
|
||||
"us-east-1"
|
||||
"af-south-1",
|
||||
"ap-east-1",
|
||||
"ap-east-2",
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-4",
|
||||
"ap-southeast-5",
|
||||
"ap-southeast-6",
|
||||
"ap-southeast-7",
|
||||
"ca-central-1",
|
||||
"ca-west-1",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-north-1",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"il-central-1",
|
||||
"mx-central-1",
|
||||
"sa-east-1",
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
"us-west-1",
|
||||
"us-west-2"
|
||||
],
|
||||
"aws-cn": [],
|
||||
"aws-eusc": [],
|
||||
@@ -16123,6 +16225,7 @@
|
||||
"aws": [
|
||||
"ap-east-1",
|
||||
"ap-northeast-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-5",
|
||||
"eu-south-2",
|
||||
"me-central-1",
|
||||
|
||||
@@ -2,7 +2,10 @@ import os
|
||||
|
||||
from botocore.config import Config
|
||||
|
||||
from prowler.providers.aws.exceptions.exceptions import AWSInvalidBoto3TimeoutError
|
||||
from prowler.providers.aws.exceptions.exceptions import (
|
||||
AWSInvalidBoto3RetriesError,
|
||||
AWSInvalidBoto3TimeoutError,
|
||||
)
|
||||
|
||||
AWS_STS_GLOBAL_ENDPOINT_REGION = "us-east-1"
|
||||
AWS_REGION_US_EAST_1 = "us-east-1"
|
||||
@@ -27,10 +30,28 @@ def get_boto3_timeout_from_env(name: str, default: int) -> int:
|
||||
return int(raw)
|
||||
|
||||
|
||||
def get_boto3_retries_from_env(name: str, default: int) -> int:
|
||||
"""Non-negative integer retries read from the environment, or default when unset."""
|
||||
raw = os.getenv(name, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
if not raw.isdecimal():
|
||||
raise AWSInvalidBoto3RetriesError(
|
||||
file=os.path.basename(__file__),
|
||||
message=f"{name} must be a non-negative integer number of retries, got {raw!r}",
|
||||
)
|
||||
return int(raw)
|
||||
|
||||
|
||||
def get_default_session_config() -> Config:
|
||||
return Config(
|
||||
user_agent_extra=BOTO3_USER_AGENT_EXTRA,
|
||||
retries={"max_attempts": BOTO3_RETRIES_MAX_ATTEMPTS, "mode": "standard"},
|
||||
retries={
|
||||
"max_attempts": get_boto3_retries_from_env(
|
||||
"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", BOTO3_RETRIES_MAX_ATTEMPTS
|
||||
),
|
||||
"mode": "standard",
|
||||
},
|
||||
connect_timeout=get_boto3_timeout_from_env(
|
||||
"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", BOTO3_CONNECT_TIMEOUT
|
||||
),
|
||||
|
||||
@@ -82,6 +82,10 @@ class AWSBaseException(ProwlerException):
|
||||
"message": "The Boto3 timeout configured through the environment is invalid",
|
||||
"remediation": "Set PROWLER_AWS_BOTO3_CONNECT_TIMEOUT and PROWLER_AWS_BOTO3_READ_TIMEOUT to a positive integer number of seconds.",
|
||||
},
|
||||
(1919, "AWSInvalidBoto3RetriesError"): {
|
||||
"message": "The Boto3 retries configured through the environment are invalid",
|
||||
"remediation": "Set PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS to a non-negative integer, 0 disables retries.",
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, code, file=None, original_exception=None, message=None):
|
||||
@@ -244,3 +248,12 @@ class AWSInvalidBoto3TimeoutError(AWSBaseException):
|
||||
super().__init__(
|
||||
1918, file=file, original_exception=original_exception, message=message
|
||||
)
|
||||
|
||||
|
||||
class AWSInvalidBoto3RetriesError(AWSBaseException):
|
||||
"""Boto3 retries configured through the environment are not a non-negative integer."""
|
||||
|
||||
def __init__(self, file=None, original_exception=None, message=None):
|
||||
super().__init__(
|
||||
1919, file=file, original_exception=original_exception, message=message
|
||||
)
|
||||
|
||||
@@ -85,7 +85,7 @@ class S3:
|
||||
aws_access_key_id: str = None,
|
||||
aws_secret_access_key: str = None,
|
||||
aws_session_token: Optional[str] = None,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
regions: set = set(),
|
||||
) -> None:
|
||||
"""
|
||||
|
||||
@@ -106,7 +106,7 @@ class SecurityHub:
|
||||
aws_access_key_id: str = None,
|
||||
aws_secret_access_key: str = None,
|
||||
aws_session_token: Optional[str] = None,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
regions: set = set(),
|
||||
) -> "SecurityHub":
|
||||
"""
|
||||
|
||||
@@ -40,7 +40,7 @@ class AwsSetUpSession:
|
||||
aws_access_key_id: str = None,
|
||||
aws_secret_access_key: str = None,
|
||||
aws_session_token: Optional[str] = None,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
regions: set = set(),
|
||||
connect_timeout: Optional[int] = None,
|
||||
read_timeout: Optional[int] = None,
|
||||
@@ -106,6 +106,8 @@ class AwsSetUpSession:
|
||||
session=self._session.current_session,
|
||||
aws_region=sts_region,
|
||||
)
|
||||
# Later STS calls go where validation got an answer, not where it timed out
|
||||
sts_region = caller_identity.region
|
||||
|
||||
logger.info("Credentials validated")
|
||||
########
|
||||
|
||||
+2
-2
@@ -144,7 +144,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"}
|
||||
name = "prowler"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.14"
|
||||
version = "5.43.0"
|
||||
version = "5.44.0"
|
||||
|
||||
[project.scripts]
|
||||
prowler = "prowler.__main__:prowler"
|
||||
@@ -214,7 +214,7 @@ constraint-dependencies = [
|
||||
"aliyun-log-fastpb==0.3.0",
|
||||
"annotated-types==0.7.0",
|
||||
"antlr4-python3-runtime==4.13.2",
|
||||
"anyio==4.13.0",
|
||||
"anyio==4.14.2",
|
||||
"apscheduler==3.11.2",
|
||||
"astroid==3.3.11",
|
||||
"async-timeout==5.0.1",
|
||||
|
||||
@@ -28,15 +28,19 @@ from prowler.providers.aws.config import (
|
||||
AWS_STS_GLOBAL_ENDPOINT_REGION,
|
||||
BOTO3_CONNECT_TIMEOUT,
|
||||
BOTO3_READ_TIMEOUT,
|
||||
BOTO3_RETRIES_MAX_ATTEMPTS,
|
||||
BOTO3_USER_AGENT_EXTRA,
|
||||
ROLE_SESSION_NAME,
|
||||
get_boto3_retries_from_env,
|
||||
get_boto3_timeout_from_env,
|
||||
get_default_session_config,
|
||||
)
|
||||
from prowler.providers.aws.exceptions.exceptions import (
|
||||
AWSAccessKeyIDInvalidError,
|
||||
AWSArgumentTypeValidationError,
|
||||
AWSAssumeRoleError,
|
||||
AWSIAMRoleARNInvalidResourceTypeError,
|
||||
AWSInvalidBoto3RetriesError,
|
||||
AWSInvalidBoto3TimeoutError,
|
||||
AWSInvalidPartitionError,
|
||||
AWSInvalidProviderIdError,
|
||||
@@ -1845,6 +1849,143 @@ aws:
|
||||
]
|
||||
assert isinstance(credentials, AWSCredentials)
|
||||
assert credentials.aws_access_key_id == "AKIAIOSFODNN7EXAMPLE"
|
||||
# Refreshing the credentials later goes straight to the region that answered
|
||||
assert assumed_role_info.sts_region == AWS_REGION_GOV_CLOUD_US_WEST_1
|
||||
|
||||
def test_assume_role_does_not_retry_a_credential_error(self, monkeypatch):
|
||||
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
|
||||
current_session = session.Session(region_name=AWS_REGION_US_EAST_1)
|
||||
attempted_regions = []
|
||||
|
||||
def create_sts_session(session, aws_region):
|
||||
attempted_regions.append(aws_region)
|
||||
sts_client = mock.MagicMock()
|
||||
sts_client.assume_role.side_effect = botocore.exceptions.ClientError(
|
||||
{"Error": {"Code": "AccessDenied", "Message": "denied"}},
|
||||
"AssumeRole",
|
||||
)
|
||||
return sts_client
|
||||
|
||||
assumed_role_info = AWSAssumeRoleInfo(
|
||||
role_arn=ARN(
|
||||
arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role"
|
||||
),
|
||||
session_duration=3600,
|
||||
external_id=None,
|
||||
mfa_enabled=False,
|
||||
role_session_name=ROLE_SESSION_NAME,
|
||||
sts_region=AWS_REGION_GOV_CLOUD_US_EAST_1,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.create_sts_session",
|
||||
new=create_sts_session,
|
||||
):
|
||||
with raises(AWSAssumeRoleError):
|
||||
AwsProvider.assume_role(current_session, assumed_role_info)
|
||||
|
||||
assert attempted_regions == [AWS_REGION_GOV_CLOUD_US_EAST_1]
|
||||
assert assumed_role_info.sts_region == AWS_REGION_GOV_CLOUD_US_EAST_1
|
||||
|
||||
def test_test_connection_role_validates_where_the_role_was_assumed(
|
||||
self, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
|
||||
monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
|
||||
attempted_calls = []
|
||||
|
||||
def create_sts_session(session, aws_region):
|
||||
if aws_region == AWS_REGION_GOV_CLOUD_US_EAST_1:
|
||||
attempted_calls.append(aws_region)
|
||||
raise botocore.exceptions.ConnectTimeoutError(
|
||||
endpoint_url=f"https://sts.{aws_region}.amazonaws.com"
|
||||
)
|
||||
sts_client = mock.MagicMock()
|
||||
|
||||
def assume_role(**_):
|
||||
attempted_calls.append(("AssumeRole", aws_region))
|
||||
return {
|
||||
"Credentials": {
|
||||
"AccessKeyId": "AKIAIOSFODNN7EXAMPLE",
|
||||
"SecretAccessKey": "secret",
|
||||
"SessionToken": "token",
|
||||
"Expiration": datetime.now() + timedelta(seconds=3600),
|
||||
}
|
||||
}
|
||||
|
||||
def get_caller_identity():
|
||||
attempted_calls.append(("GetCallerIdentity", aws_region))
|
||||
return {
|
||||
"UserId": "test-user-id",
|
||||
"Account": AWS_ACCOUNT_NUMBER,
|
||||
"Arn": AWS_GOV_CLOUD_ACCOUNT_ARN,
|
||||
}
|
||||
|
||||
sts_client.assume_role.side_effect = assume_role
|
||||
sts_client.get_caller_identity.side_effect = get_caller_identity
|
||||
return sts_client
|
||||
|
||||
with patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.create_sts_session",
|
||||
new=create_sts_session,
|
||||
):
|
||||
connection = AwsProvider.test_connection(
|
||||
role_arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role",
|
||||
aws_access_key_id="test-access-key",
|
||||
aws_secret_access_key="test-secret-key",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
assert connection.is_connected
|
||||
# The unreachable region is paid for once, not again for the validation
|
||||
assert attempted_calls == [
|
||||
AWS_REGION_GOV_CLOUD_US_EAST_1,
|
||||
("AssumeRole", AWS_REGION_GOV_CLOUD_US_WEST_1),
|
||||
("GetCallerIdentity", AWS_REGION_GOV_CLOUD_US_WEST_1),
|
||||
]
|
||||
|
||||
@mock_aws
|
||||
def test_aws_set_up_session_assumes_the_role_where_validation_got_an_answer(
|
||||
self, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
|
||||
monkeypatch.setenv("AWS_DEFAULT_REGION", AWS_REGION_US_EAST_1)
|
||||
answered = AWSCallerIdentity(
|
||||
user_id="test-user-id",
|
||||
account=AWS_ACCOUNT_NUMBER,
|
||||
arn=ARN(AWS_GOV_CLOUD_ACCOUNT_ARN),
|
||||
region=AWS_REGION_GOV_CLOUD_US_WEST_1,
|
||||
)
|
||||
sts_regions = []
|
||||
|
||||
class RoleAssumed(Exception):
|
||||
pass
|
||||
|
||||
def assume_role(session, assumed_role_info):
|
||||
sts_regions.append(assumed_role_info.sts_region)
|
||||
raise RoleAssumed
|
||||
|
||||
with (
|
||||
patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.validate_credentials",
|
||||
return_value=answered,
|
||||
),
|
||||
patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.assume_role",
|
||||
side_effect=assume_role,
|
||||
),
|
||||
):
|
||||
with raises(RoleAssumed):
|
||||
AwsSetUpSession(
|
||||
role_arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role",
|
||||
session_duration=900,
|
||||
external_id="test-external-id",
|
||||
role_session_name=ROLE_SESSION_NAME,
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
)
|
||||
|
||||
assert sts_regions == [AWS_REGION_GOV_CLOUD_US_WEST_1]
|
||||
|
||||
def test_setup_session_mfa_falls_back_to_the_next_partition_region(
|
||||
self, monkeypatch
|
||||
@@ -3352,6 +3493,123 @@ aws:
|
||||
):
|
||||
get_boto3_timeout_from_env("PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", 10)
|
||||
|
||||
def test_get_default_session_config_retries_from_env(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
|
||||
):
|
||||
config = get_default_session_config()
|
||||
|
||||
assert config.retries == {"max_attempts": 1, "mode": "standard"}
|
||||
|
||||
def test_get_default_session_config_retries_from_env_0_disables_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
):
|
||||
config = get_default_session_config()
|
||||
|
||||
assert config.retries == {"max_attempts": 0, "mode": "standard"}
|
||||
|
||||
def test_set_session_config_argument_overrides_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
|
||||
):
|
||||
config = AwsProvider.set_session_config(5)
|
||||
|
||||
assert config.retries == {"max_attempts": 5, "mode": "standard"}
|
||||
|
||||
@mock_aws
|
||||
def test_aws_provider_without_retries_argument_uses_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
):
|
||||
aws_provider = AwsProvider()
|
||||
client = aws_provider.session.current_session.client(
|
||||
"ec2", region_name=AWS_REGION_US_EAST_1
|
||||
)
|
||||
|
||||
# botocore rewrites max_attempts into total_max_attempts (retries + 1)
|
||||
assert client.meta.config.retries["total_max_attempts"] == 1
|
||||
|
||||
@mock_aws
|
||||
def test_aws_provider_retries_argument_overrides_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
):
|
||||
aws_provider = AwsProvider(retries_max_attempts=7)
|
||||
client = aws_provider.session.current_session.client(
|
||||
"ec2", region_name=AWS_REGION_US_EAST_1
|
||||
)
|
||||
|
||||
assert client.meta.config.retries["total_max_attempts"] == 8
|
||||
|
||||
@mock_aws
|
||||
def test_aws_set_up_session_without_retries_argument_uses_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
|
||||
):
|
||||
aws_session = AwsSetUpSession(
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
)
|
||||
client = aws_session._session.current_session.client(
|
||||
"ec2", region_name=AWS_REGION_US_EAST_1
|
||||
)
|
||||
|
||||
assert client.meta.config.retries["total_max_attempts"] == 2
|
||||
|
||||
def test_test_connection_session_uses_env_retries(self):
|
||||
with (
|
||||
mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
),
|
||||
mock.patch.object(
|
||||
AwsProvider,
|
||||
"validate_credentials",
|
||||
return_value=AWSCallerIdentity(
|
||||
user_id="test-user-id",
|
||||
account=AWS_ACCOUNT_NUMBER,
|
||||
arn=ARN(AWS_ACCOUNT_ARN),
|
||||
region=AWS_REGION_US_EAST_1,
|
||||
),
|
||||
) as mock_validate_credentials,
|
||||
):
|
||||
connection = AwsProvider.test_connection(
|
||||
aws_access_key_id="test-access-key",
|
||||
aws_secret_access_key="test-secret-key",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
assert connection.is_connected
|
||||
validated_session = mock_validate_credentials.call_args.args[0]
|
||||
assert validated_session._session.get_default_client_config().retries == {
|
||||
"max_attempts": 0,
|
||||
"mode": "standard",
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize("raw", ["-1", "three", "1.5"])
|
||||
def test_get_boto3_retries_from_env_rejects_anything_but_non_negative_integers(
|
||||
self, raw
|
||||
):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": raw}
|
||||
):
|
||||
with raises(
|
||||
AWSInvalidBoto3RetriesError,
|
||||
match="PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS",
|
||||
):
|
||||
get_boto3_retries_from_env("PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", 3)
|
||||
|
||||
def test_get_boto3_retries_from_env_blank_falls_back_to_default(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": " "}
|
||||
):
|
||||
assert (
|
||||
get_boto3_retries_from_env(
|
||||
"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", BOTO3_RETRIES_MAX_ATTEMPTS
|
||||
)
|
||||
== BOTO3_RETRIES_MAX_ATTEMPTS
|
||||
)
|
||||
|
||||
def test_get_boto3_timeout_from_env_blank_falls_back_to_default(self):
|
||||
with mock.patch.dict(os.environ, {"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": " "}):
|
||||
assert (
|
||||
|
||||
@@ -4,6 +4,24 @@ All notable changes to the **Prowler UI** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [1.43.0] (Prowler v5.43.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Registry marketplace and external provider onboarding for Private Cloud, with permission-based access independent of billing, confirmed artifact installation, schema-driven credentials, connection checks, and scan launch [(#12494)](https://github.com/prowler-cloud/prowler/pull/12494)
|
||||
- AWS Marketplace button variant with outlined styling for light and dark themes [(#12803)](https://github.com/prowler-cloud/prowler/pull/12803)
|
||||
- `UI_SELF_REGISTRATION_ENABLED` flag for Prowler Private Cloud deployments; when `"false"`, `/sign-up` only opens with an invitation, the sign-in page drops the "Sign up" link and the profile hides **Create organization** [(#12815)](https://github.com/prowler-cloud/prowler/pull/12815)
|
||||
- "Invite your team" step offered once after the first provider is connected, before the onboarding checkpoint, reusing the invitation form tagged with `source=onboarding` [(#12819)](https://github.com/prowler-cloud/prowler/pull/12819)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Automatic onboarding stays hidden on billing pages and remains available after leaving billing [(#12803)](https://github.com/prowler-cloud/prowler/pull/12803)
|
||||
- Per-provider breakdown and OCSF download for FedRAMP 20x KSI and Class C FRR in the cross-provider compliance view [(#12810)](https://github.com/prowler-cloud/prowler/pull/12810)
|
||||
- Edit and Revoke actions are disabled for expired and revoked invitations [(#12831)](https://github.com/prowler-cloud/prowler/pull/12831)
|
||||
- Cloudflare API token links in the provider wizard request the SSL and Certificates, Bot Management and Zone WAF read permissions the scan needs [(#12842)](https://github.com/prowler-cloud/prowler/pull/12842)
|
||||
|
||||
---
|
||||
|
||||
## [1.42.0] (Prowler v5.42.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
@@ -302,6 +302,7 @@ export const handlersForOrganizations = (
|
||||
organizations.map((o) => o.secretId).filter((id): id is string => !!id),
|
||||
);
|
||||
let orgSeq = 0;
|
||||
let providerSeq = 0;
|
||||
let secretSeq = 0;
|
||||
/** Reads per connection task, so `executingPolls` can hold one task running. */
|
||||
const connectionTaskReads = new Map<string, number>();
|
||||
@@ -565,6 +566,37 @@ export const handlersForOrganizations = (
|
||||
HttpResponse.json({ data: [], meta: collectionMeta(0) }),
|
||||
),
|
||||
|
||||
// --- single-account connect (AWS one-step form) -----------------------
|
||||
http.post(`${API}/providers`, async ({ request }) => {
|
||||
const body = (await request.json()) as {
|
||||
data: { attributes: { provider: string; uid: string; alias?: string } };
|
||||
};
|
||||
providerSeq += 1;
|
||||
return HttpResponse.json(
|
||||
{
|
||||
data: {
|
||||
id: `provider-created-${providerSeq}`,
|
||||
type: "providers",
|
||||
attributes: {
|
||||
...body.data.attributes,
|
||||
connection: { connected: false, last_checked_at: null },
|
||||
},
|
||||
},
|
||||
},
|
||||
{ status: 201 },
|
||||
);
|
||||
}),
|
||||
|
||||
http.post(`${API}/providers/secrets`, () => {
|
||||
secretSeq += 1;
|
||||
return HttpResponse.json(
|
||||
{
|
||||
data: { id: `secret-created-${secretSeq}`, type: "provider-secrets" },
|
||||
},
|
||||
{ status: 201 },
|
||||
);
|
||||
}),
|
||||
|
||||
// --- providers (uid resolution) + connection testing -----------------
|
||||
http.get<{ id: string }>(`${API}/providers/:id`, ({ params }) => {
|
||||
const provider = fx.providers.find((p) => p.id === params.id);
|
||||
|
||||
@@ -215,3 +215,72 @@ describe("adaptFindingsByResourceResponse — malformed input", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("adaptFindingsByResourceResponse — provider id", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("should carry the provider id resolved through the scan include", () => {
|
||||
// Given — scan.provider include path, as the drawer requests it
|
||||
createDictMock.mockImplementation((type: string) => {
|
||||
if (type === "scans") {
|
||||
return {
|
||||
"scan-1": {
|
||||
id: "scan-1",
|
||||
attributes: {},
|
||||
relationships: { provider: { data: { id: "provider-1" } } },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (type === "providers") {
|
||||
return {
|
||||
"provider-1": {
|
||||
id: "provider-1",
|
||||
attributes: { provider: "aws", alias: "prod", uid: "123" },
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const input = {
|
||||
data: {
|
||||
id: "finding-1",
|
||||
attributes: {
|
||||
uid: "uid-1",
|
||||
check_id: "s3_check",
|
||||
status: "FAIL",
|
||||
severity: "high",
|
||||
check_metadata: {},
|
||||
},
|
||||
relationships: {
|
||||
resources: { data: [] },
|
||||
scan: { data: { id: "scan-1" } },
|
||||
},
|
||||
},
|
||||
included: [],
|
||||
};
|
||||
|
||||
// When
|
||||
const [finding] = adaptFindingsByResourceResponse(input);
|
||||
|
||||
// Then — the partial-scan request needs the id, not only the uid
|
||||
expect(finding.providerId).toBe("provider-1");
|
||||
expect(finding.providerUid).toBe("123");
|
||||
});
|
||||
|
||||
it("should leave the provider id empty when the scan is not included", () => {
|
||||
createDictMock.mockReturnValue({});
|
||||
|
||||
const [finding] = adaptFindingsByResourceResponse({
|
||||
data: {
|
||||
id: "finding-1",
|
||||
attributes: { uid: "uid-1", check_id: "s3_check", status: "FAIL" },
|
||||
relationships: { resources: { data: [] }, scan: { data: null } },
|
||||
},
|
||||
});
|
||||
|
||||
expect(finding.providerId).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,6 +64,7 @@ export interface ResourceDrawerFinding {
|
||||
resourceDetails: string | null;
|
||||
resourceMetadata: Record<string, unknown> | string | null;
|
||||
// Provider
|
||||
providerId: string;
|
||||
providerType: ProviderType;
|
||||
providerAlias: string;
|
||||
providerUid: string;
|
||||
@@ -280,6 +281,7 @@ export function adaptFindingsByResourceResponse(
|
||||
| null
|
||||
| undefined) ?? null,
|
||||
// Provider
|
||||
providerId: providerRelId ?? "",
|
||||
providerType: ((providerAttrs.provider as string | undefined) ||
|
||||
"aws") as ProviderType,
|
||||
providerAlias: (providerAttrs.alias as string | undefined) || "",
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { fetchMock, getAuthHeadersMock, revalidatePathMock } = vi.hoisted(
|
||||
() => ({
|
||||
fetchMock: vi.fn(),
|
||||
getAuthHeadersMock: vi.fn(),
|
||||
revalidatePathMock: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
vi.mock("@/lib/helper", () => ({
|
||||
apiBaseUrl: "https://api.test/api/v1",
|
||||
getAuthHeaders: getAuthHeadersMock,
|
||||
}));
|
||||
|
||||
vi.mock("next/cache", () => ({
|
||||
revalidatePath: revalidatePathMock,
|
||||
}));
|
||||
|
||||
import { createMuteRule } from "./mute-rules";
|
||||
|
||||
const NAME_CONFLICT_DETAIL = "A mute rule with this name already exists.";
|
||||
|
||||
const errorResponse = (contentType: string, body: string, status = 400) =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "Content-Type": contentType },
|
||||
});
|
||||
|
||||
const nameConflictBody = JSON.stringify({
|
||||
errors: [
|
||||
{
|
||||
detail: NAME_CONFLICT_DETAIL,
|
||||
status: "400",
|
||||
source: { pointer: "/data/attributes/name" },
|
||||
code: "invalid",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const muteRuleFormData = () => {
|
||||
const formData = new FormData();
|
||||
formData.set("name", "Root account has a hardware MFA device enabled");
|
||||
formData.set("reason", "Not our approach here with SSO");
|
||||
formData.set("finding_ids", JSON.stringify(["finding-1"]));
|
||||
return formData;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
});
|
||||
|
||||
describe("createMuteRule", () => {
|
||||
it("should return only the error detail for a JSON:API error response", async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
errorResponse("application/vnd.api+json", nameConflictBody),
|
||||
);
|
||||
|
||||
const result = await createMuteRule(null, muteRuleFormData());
|
||||
|
||||
expect(result?.errors?.general).toBe(NAME_CONFLICT_DETAIL);
|
||||
expect(revalidatePathMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should return only the error detail for a plain JSON error response", async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
errorResponse("application/json", nameConflictBody),
|
||||
);
|
||||
|
||||
const result = await createMuteRule(null, muteRuleFormData());
|
||||
|
||||
expect(result?.errors?.general).toBe(NAME_CONFLICT_DETAIL);
|
||||
});
|
||||
|
||||
it("should return the response text for a non-JSON error response", async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
errorResponse("text/plain", "Bad gateway", 502),
|
||||
);
|
||||
|
||||
const result = await createMuteRule(null, muteRuleFormData());
|
||||
|
||||
expect(result?.errors?.general).toBe("Bad gateway");
|
||||
});
|
||||
});
|
||||
@@ -156,7 +156,8 @@ export const createMuteRule = async (
|
||||
let errorMessage = `Failed to create mute rule: ${response.statusText}`;
|
||||
const responseContentType = response.headers.get("content-type");
|
||||
try {
|
||||
if (responseContentType?.includes("application/json")) {
|
||||
// The API answers with application/vnd.api+json
|
||||
if (responseContentType?.includes("json")) {
|
||||
const errorData = await response.json();
|
||||
const jsonApiError = (
|
||||
errorData as {
|
||||
|
||||
@@ -546,7 +546,7 @@ export const applyDiscovery = async (
|
||||
);
|
||||
// No `include`: the apply view rejects the parameter outright and fails the
|
||||
// whole request. The created providers' uids are read afterwards instead, with
|
||||
// `getProviderUidsByIds`.
|
||||
// `getProviderUidsAndConnectionBaselines`.
|
||||
|
||||
const attributes = buildApplyAttributes(payload);
|
||||
|
||||
|
||||
@@ -151,28 +151,28 @@ export const getProvider = async (formData: FormData) => {
|
||||
const PROVIDERS_PAGE_MAX = 100;
|
||||
|
||||
/**
|
||||
* Uids of the given providers, keyed by provider id. A provider's `uid` is the
|
||||
* candidate it was created for (AWS account id / GCP project id), so this is what
|
||||
* matches an apply's created providers back to the selection. Batched with
|
||||
* `filter[id__in]` rather than one `GET /providers/{id}` per id.
|
||||
* Providers matching the given ids, batched with `filter[id__in]` (page size
|
||||
* `PROVIDERS_PAGE_MAX`, the server max, which also bounds the id batch size)
|
||||
* rather than one `GET /providers/{id}` per id. Shared by every action below
|
||||
* that resolves providers by id; a batch that fails to fetch leaves its
|
||||
* providers out of the result rather than failing the rest, and it is on each
|
||||
* caller to say what "missing" means for its own map. Not exported: an
|
||||
* exported function in this `"use server"` module becomes a callable server
|
||||
* action.
|
||||
*/
|
||||
export const getProviderUidsByIds = async (
|
||||
const fetchProvidersByIds = async (
|
||||
providerIds: string[],
|
||||
): Promise<Record<string, string>> => {
|
||||
): Promise<ProvidersApiResponse["data"]> => {
|
||||
const uniqueIds = Array.from(new Set(providerIds.filter(Boolean)));
|
||||
if (uniqueIds.length === 0) {
|
||||
return {};
|
||||
return [];
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
const batches: string[][] = [];
|
||||
const providers: ProvidersApiResponse["data"] = [];
|
||||
|
||||
for (let start = 0; start < uniqueIds.length; start += PROVIDERS_PAGE_MAX) {
|
||||
batches.push(uniqueIds.slice(start, start + PROVIDERS_PAGE_MAX));
|
||||
}
|
||||
|
||||
const uidById: Record<string, string> = {};
|
||||
|
||||
for (const batch of batches) {
|
||||
const batch = uniqueIds.slice(start, start + PROVIDERS_PAGE_MAX);
|
||||
const url = new URL(`${apiBaseUrl}/providers`);
|
||||
url.searchParams.set("filter[id__in]", batch.join(","));
|
||||
url.searchParams.set("page[size]", String(PROVIDERS_PAGE_MAX));
|
||||
@@ -183,20 +183,93 @@ export const getProviderUidsByIds = async (
|
||||
| ProvidersApiResponse
|
||||
| undefined;
|
||||
|
||||
for (const provider of result?.data ?? []) {
|
||||
const uid = provider?.attributes?.uid;
|
||||
if (typeof provider?.id === "string" && typeof uid === "string") {
|
||||
uidById[provider.id] = uid;
|
||||
}
|
||||
}
|
||||
providers.push(...(result?.data ?? []));
|
||||
} catch {
|
||||
// A failed batch leaves its providers unmapped rather than failing the rest.
|
||||
// A failed batch leaves its providers out of the result rather than
|
||||
// failing the rest.
|
||||
}
|
||||
}
|
||||
|
||||
return providers;
|
||||
};
|
||||
|
||||
/**
|
||||
* Uids of the given providers, keyed by provider id. A provider's `uid` is the
|
||||
* candidate it was created for (AWS account id / GCP project id), so this is what
|
||||
* matches an apply's created providers back to the selection.
|
||||
*/
|
||||
export const getProviderUidsByIds = async (
|
||||
providerIds: string[],
|
||||
): Promise<Record<string, string>> => {
|
||||
const uidById: Record<string, string> = {};
|
||||
|
||||
for (const provider of await fetchProvidersByIds(providerIds)) {
|
||||
const uid = provider?.attributes?.uid;
|
||||
if (typeof provider?.id === "string" && typeof uid === "string") {
|
||||
uidById[provider.id] = uid;
|
||||
}
|
||||
}
|
||||
|
||||
return uidById;
|
||||
};
|
||||
|
||||
/**
|
||||
* `connection.last_checked_at` for each given provider, keyed by id. Read before
|
||||
* a batch of connection checks is dispatched, so `resolveProviderConnectionState`
|
||||
* can tell a check's own result apart from an older one already on record by
|
||||
* comparing values, never by comparing the browser's clock against the server's
|
||||
* (see that function for why). A provider missing from the response -- the batch
|
||||
* read failed, or it was deleted mid-flight -- is left out of the map rather than
|
||||
* defaulted, so callers can tell "no prior check" (`null`) from "unknown".
|
||||
*/
|
||||
export const getProviderConnectionBaselines = async (
|
||||
providerIds: string[],
|
||||
): Promise<Record<string, string | null>> => {
|
||||
const baselineById: Record<string, string | null> = {};
|
||||
|
||||
for (const provider of await fetchProvidersByIds(providerIds)) {
|
||||
if (typeof provider?.id === "string") {
|
||||
baselineById[provider.id] =
|
||||
provider.attributes?.connection?.last_checked_at ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
return baselineById;
|
||||
};
|
||||
|
||||
/**
|
||||
* Uid and `connection.last_checked_at` for each given provider, keyed by id, read
|
||||
* with a single batched `filter[id__in]` request. The organization onboarding
|
||||
* apply step needs both right after creating providers: the uid to match each one
|
||||
* back to the candidate it was created for (see `getProviderUidsByIds`), and the
|
||||
* baseline to compare a dispatched check's result against (see
|
||||
* `getProviderConnectionBaselines`). Reading them together avoids fetching the
|
||||
* same set of providers twice.
|
||||
*/
|
||||
export const getProviderUidsAndConnectionBaselines = async (
|
||||
providerIds: string[],
|
||||
): Promise<{
|
||||
uidById: Record<string, string>;
|
||||
baselineById: Record<string, string | null>;
|
||||
}> => {
|
||||
const uidById: Record<string, string> = {};
|
||||
const baselineById: Record<string, string | null> = {};
|
||||
|
||||
for (const provider of await fetchProvidersByIds(providerIds)) {
|
||||
if (typeof provider?.id !== "string") {
|
||||
continue;
|
||||
}
|
||||
const uid = provider.attributes?.uid;
|
||||
if (typeof uid === "string") {
|
||||
uidById[provider.id] = uid;
|
||||
}
|
||||
baselineById[provider.id] =
|
||||
provider.attributes?.connection?.last_checked_at ?? null;
|
||||
}
|
||||
|
||||
return { uidById, baselineById };
|
||||
};
|
||||
|
||||
export const updateProvider = async (formData: FormData) => {
|
||||
const headers = await getAuthHeaders({ contentType: true });
|
||||
const providerId = formData.get(ProviderCredentialFields.PROVIDER_ID);
|
||||
|
||||
@@ -24,6 +24,7 @@ const REGISTRY_TASK_PATH_PREFIX = "/api/v1/tasks/";
|
||||
const REGISTRY_ERROR_CODE = {
|
||||
KEY_REJECTED: "registry_key_rejected",
|
||||
UNAVAILABLE: "registry_unavailable",
|
||||
PAGE_NOT_FOUND: "registry_page_not_found",
|
||||
} as const;
|
||||
// Opposite remedies, so a 409 is never read without its code.
|
||||
const REGISTRY_REMOVAL_CONFLICT_CODE = {
|
||||
@@ -216,18 +217,29 @@ export async function classifyRegistryFailure(
|
||||
return { status: REGISTRY_FAILURE.ERROR };
|
||||
}
|
||||
|
||||
// An enabled backend also answers 404 (missing page): tell them apart by code.
|
||||
export async function isRegistryDisabledResponse(response: Response) {
|
||||
if (response.status !== 404) return false;
|
||||
const codes = await getRegistryErrorCodes(response);
|
||||
return !codes.includes(REGISTRY_ERROR_CODE.PAGE_NOT_FOUND);
|
||||
}
|
||||
|
||||
function isRegistryDiscoveryEndpoint(endpoint: RegistryEndpoint) {
|
||||
return registryDiscoveryEndpoints.has(endpoint);
|
||||
}
|
||||
|
||||
async function getRegistryErrorCode(response: Response) {
|
||||
async function getRegistryErrorCodes(response: Response) {
|
||||
const parsed = errorDocumentSchema.safeParse(
|
||||
await response
|
||||
.clone()
|
||||
.json()
|
||||
.catch(() => undefined),
|
||||
);
|
||||
return parsed.success ? parsed.data.errors[0]?.code : undefined;
|
||||
return parsed.success ? parsed.data.errors.map(({ code }) => code) : [];
|
||||
}
|
||||
|
||||
async function getRegistryErrorCode(response: Response) {
|
||||
return (await getRegistryErrorCodes(response))[0];
|
||||
}
|
||||
|
||||
const REGISTRY_CATALOG_PAGE_SIZE = 100;
|
||||
|
||||
@@ -119,14 +119,17 @@ describe("installed Registry provider discovery", () => {
|
||||
emptyMetadata = false,
|
||||
failedEndpoint,
|
||||
failureStatus = 500,
|
||||
failureBody = {},
|
||||
}: {
|
||||
emptyMetadata?: boolean;
|
||||
failedEndpoint?: string;
|
||||
failureStatus?: number;
|
||||
failureBody?: unknown;
|
||||
} = {}) {
|
||||
fetchMock.mockImplementation((url: string) => {
|
||||
const endpoint = new URL(url).pathname.split("/").pop();
|
||||
if (endpoint === failedEndpoint) return jsonResponse({}, failureStatus);
|
||||
if (endpoint === failedEndpoint)
|
||||
return jsonResponse(failureBody, failureStatus);
|
||||
if (endpoint === "available-artifacts")
|
||||
return jsonResponse({
|
||||
data: [
|
||||
@@ -242,6 +245,45 @@ describe("installed Registry provider discovery", () => {
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["available-artifacts", "artifacts", "providers"])(
|
||||
"hides Registry when the backend has it disabled and %s answers 404",
|
||||
async (failedEndpoint) => {
|
||||
mockDiscovery({ failedEndpoint, failureStatus: 404 });
|
||||
expect(await getInstalledRegistryProviderOptions()).toEqual({
|
||||
status: "access_denied",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["available-artifacts", "providers"])(
|
||||
"keeps Registry visible when an enabled backend answers 404 for a missing %s page",
|
||||
async (failedEndpoint) => {
|
||||
mockDiscovery({
|
||||
failedEndpoint,
|
||||
failureStatus: 404,
|
||||
failureBody: {
|
||||
errors: [{ status: "404", code: "registry_page_not_found" }],
|
||||
},
|
||||
});
|
||||
expect(await getInstalledRegistryProviderOptions()).toEqual({
|
||||
status: "error",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps Registry visible when the missing-page code is not the first error", async () => {
|
||||
mockDiscovery({
|
||||
failedEndpoint: "available-artifacts",
|
||||
failureStatus: 404,
|
||||
failureBody: {
|
||||
errors: [{ code: "not_found" }, { code: "registry_page_not_found" }],
|
||||
},
|
||||
});
|
||||
expect(await getInstalledRegistryProviderOptions()).toEqual({
|
||||
status: "error",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Registry guarded reads", () => {
|
||||
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
classifyRegistryRemovalConflict,
|
||||
collectCompleteRegistryCatalog,
|
||||
isRegistryCollection,
|
||||
isRegistryDisabledResponse,
|
||||
parseRegistryArtifactSubmission,
|
||||
parseRegistryCredentialSubmission,
|
||||
RegistryCatalogPageError,
|
||||
@@ -86,6 +87,9 @@ async function readRegistryResponse(
|
||||
return { status: REGISTRY_FAILURE.ERROR };
|
||||
}
|
||||
if (response.ok) return response;
|
||||
// A backend with Registry disabled answers 404: hide it like a denial.
|
||||
if (await isRegistryDisabledResponse(response))
|
||||
return { status: REGISTRY_FAILURE.ACCESS_DENIED };
|
||||
|
||||
return endpoint === REGISTRY_ENDPOINT.PROVIDERS ||
|
||||
endpoint === REGISTRY_ENDPOINT.AVAILABLE_ARTIFACTS
|
||||
|
||||
@@ -6,12 +6,14 @@ const {
|
||||
getAuthHeadersMock,
|
||||
handleApiErrorMock,
|
||||
handleApiResponseMock,
|
||||
isReportDownloadLockedMock,
|
||||
} = vi.hoisted(() => ({
|
||||
addScanOperationMock: vi.fn(),
|
||||
fetchMock: vi.fn(),
|
||||
getAuthHeadersMock: vi.fn(),
|
||||
handleApiErrorMock: vi.fn(),
|
||||
handleApiResponseMock: vi.fn(),
|
||||
isReportDownloadLockedMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib", () => ({
|
||||
@@ -23,6 +25,10 @@ vi.mock("@/lib", () => ({
|
||||
error instanceof Error ? error.message : String(error),
|
||||
}));
|
||||
|
||||
vi.mock("next/cache", () => ({
|
||||
revalidatePath: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/server-actions-helper", () => ({
|
||||
handleApiError: handleApiErrorMock,
|
||||
handleApiResponse: handleApiResponseMock,
|
||||
@@ -32,7 +38,17 @@ vi.mock("@/lib/sentry-breadcrumbs", () => ({
|
||||
addScanOperation: addScanOperationMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/report-download-access", () => ({
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR:
|
||||
"Report downloads require an active subscription.",
|
||||
isReportDownloadLocked: isReportDownloadLockedMock,
|
||||
}));
|
||||
|
||||
import {
|
||||
createPartialScan,
|
||||
getComplianceCsv,
|
||||
getComplianceOcsf,
|
||||
getCompliancePdfReport,
|
||||
getExportsZip,
|
||||
launchOrganizationScans,
|
||||
scheduleOrganizationDailyScans,
|
||||
@@ -156,6 +172,7 @@ describe("getExportsZip", () => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
isReportDownloadLockedMock.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it("returns a generic server error when the report endpoint returns HTML", async () => {
|
||||
@@ -181,3 +198,98 @@ describe("getExportsZip", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("report downloads for subscription-only tenants", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
isReportDownloadLockedMock.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "scan ZIP", download: () => getExportsZip("scan-123") },
|
||||
{
|
||||
name: "compliance CSV",
|
||||
download: () => getComplianceCsv("scan-123", "cis_2.0_aws"),
|
||||
},
|
||||
{
|
||||
name: "compliance OCSF",
|
||||
download: () => getComplianceOcsf("scan-123", "dora_aws"),
|
||||
},
|
||||
{
|
||||
name: "compliance PDF",
|
||||
download: () => getCompliancePdfReport("scan-123", "threatscore"),
|
||||
},
|
||||
])("rejects the $name without calling the API", async ({ download }) => {
|
||||
// When
|
||||
const result = await download();
|
||||
|
||||
// Then
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
error: "Report downloads require an active subscription.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("createPartialScan", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
fetchMock.mockResolvedValue(new Response(null, { status: 202 }));
|
||||
handleApiResponseMock.mockResolvedValue({ data: { id: "scan-1" } });
|
||||
});
|
||||
|
||||
it("posts the resource uids as a scan of one provider", async () => {
|
||||
// When
|
||||
const result = await createPartialScan({
|
||||
providerId: "provider-1",
|
||||
resourceUids: ["arn:aws:s3:::bucket"],
|
||||
});
|
||||
|
||||
// Then
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.example.com/api/v1/scans",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
type: "scans",
|
||||
attributes: { resource_uids: ["arn:aws:s3:::bucket"] },
|
||||
relationships: {
|
||||
provider: { data: { type: "providers", id: "provider-1" } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(handleApiResponseMock).toHaveBeenCalledWith(
|
||||
expect.any(Response),
|
||||
"/scans",
|
||||
);
|
||||
expect(result).toEqual({ data: { id: "scan-1" } });
|
||||
expect(addScanOperationMock).toHaveBeenCalledWith("start", "scan-1");
|
||||
});
|
||||
|
||||
it("refuses more resources than the API accepts without calling it", async () => {
|
||||
// Given — the API caps a partial scan at 10 resources.
|
||||
const resourceUids = Array.from(
|
||||
{ length: 11 },
|
||||
(_, index) => `arn:aws:s3:::bucket-${index}`,
|
||||
);
|
||||
|
||||
// When
|
||||
const result = await createPartialScan({
|
||||
providerId: "provider-1",
|
||||
resourceUids,
|
||||
});
|
||||
|
||||
// Then
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
error: "Select between 1 and 10 resources to re-check",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,9 +15,14 @@ import {
|
||||
} from "@/lib/compliance/compliance-report-types";
|
||||
import { runWithConcurrencyLimit } from "@/lib/concurrency";
|
||||
import { appendSanitizedProviderTypeFilters } from "@/lib/provider-filters";
|
||||
import {
|
||||
isReportDownloadLocked,
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR,
|
||||
} from "@/lib/report-download-access";
|
||||
import { addScanOperation } from "@/lib/sentry-breadcrumbs";
|
||||
import { handleApiError, handleApiResponse } from "@/lib/server-actions-helper";
|
||||
import { SCAN_STATES } from "@/types/attack-paths";
|
||||
import { PARTIAL_SCAN_MAX_RESOURCES } from "@/types/partial-scans";
|
||||
|
||||
const ORGANIZATION_SCAN_CONCURRENCY_LIMIT = 5;
|
||||
|
||||
@@ -182,6 +187,70 @@ export const scanOnDemand = async (formData: FormData) => {
|
||||
}
|
||||
};
|
||||
|
||||
/** Prowler Cloud only: re-check up to PARTIAL_SCAN_MAX_RESOURCES resources of one provider. */
|
||||
export const createPartialScan = async ({
|
||||
providerId,
|
||||
resourceUids,
|
||||
}: {
|
||||
providerId: string;
|
||||
resourceUids: string[];
|
||||
}) => {
|
||||
if (!providerId) {
|
||||
return { error: "Provider ID is required" };
|
||||
}
|
||||
if (
|
||||
resourceUids.length === 0 ||
|
||||
resourceUids.length > PARTIAL_SCAN_MAX_RESOURCES
|
||||
) {
|
||||
return {
|
||||
error: `Select between 1 and ${PARTIAL_SCAN_MAX_RESOURCES} resources to re-check`,
|
||||
};
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: true });
|
||||
|
||||
addScanOperation("create", undefined, {
|
||||
provider_id: providerId,
|
||||
partial: true,
|
||||
resource_count: resourceUids.length,
|
||||
});
|
||||
|
||||
const url = new URL(`${apiBaseUrl}/scans`);
|
||||
|
||||
try {
|
||||
const requestBody = {
|
||||
data: {
|
||||
type: "scans",
|
||||
attributes: { resource_uids: resourceUids },
|
||||
relationships: {
|
||||
provider: {
|
||||
data: {
|
||||
type: "providers",
|
||||
id: providerId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
const result = await handleApiResponse(response, "/scans");
|
||||
if (result?.data?.id) {
|
||||
addScanOperation("start", result.data.id);
|
||||
revalidatePath("/scans");
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
addScanOperation("create");
|
||||
return handleApiError(error);
|
||||
}
|
||||
};
|
||||
|
||||
export const scheduleDaily = async (formData: FormData) => {
|
||||
const headers = await getAuthHeaders({ contentType: true });
|
||||
|
||||
@@ -377,6 +446,10 @@ export const updateScan = async (formData: FormData) => {
|
||||
};
|
||||
|
||||
export const getExportsZip = async (scanId: string) => {
|
||||
if (await isReportDownloadLocked()) {
|
||||
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
|
||||
const url = new URL(`${apiBaseUrl}/scans/${scanId}/report`);
|
||||
@@ -457,6 +530,10 @@ const _fetchScanBinary = async (
|
||||
filename: string,
|
||||
errorLabel: string,
|
||||
): Promise<ScanBinaryResult> => {
|
||||
if (await isReportDownloadLocked()) {
|
||||
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
const url = new URL(`${apiBaseUrl}/scans/${scanId}/${urlPath}`);
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
} from "@/lib/compliance/compliance-report-types";
|
||||
import { LIGHTHOUSE_COMPLIANCE_CONTEXT_MODE } from "@/lib/lighthouse/context/constants";
|
||||
import { buildComplianceContext } from "@/lib/lighthouse/context/contributions";
|
||||
import { isReportDownloadLocked } from "@/lib/report-download-access";
|
||||
import { isCloud } from "@/lib/shared/env";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { SearchParamsProps } from "@/types";
|
||||
@@ -77,6 +78,8 @@ export default async function ComplianceDetail({
|
||||
notFound();
|
||||
}
|
||||
|
||||
const subscriptionOnlyPromise = isReportDownloadLocked();
|
||||
|
||||
// Cross-provider mode replaces the per-scan pipeline with the universal
|
||||
// roll-up view. Prowler Cloud-only: the OSS API has no such endpoint, so
|
||||
// the route is blocked in OSS the same way the compliance tab is.
|
||||
@@ -85,6 +88,7 @@ export default async function ComplianceDetail({
|
||||
redirect("/compliance");
|
||||
}
|
||||
|
||||
const subscriptionOnly = await subscriptionOnlyPromise;
|
||||
return (
|
||||
<Suspense
|
||||
key={buildSearchParamsKey(resolvedSearchParams)}
|
||||
@@ -105,6 +109,7 @@ export default async function ComplianceDetail({
|
||||
complianceId={complianceId}
|
||||
searchParams={resolvedSearchParams}
|
||||
targetSection={section}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</Suspense>
|
||||
);
|
||||
@@ -124,6 +129,7 @@ export default async function ComplianceDetail({
|
||||
}
|
||||
|
||||
const crossAccountTitle = compliancetitle.split("-").join(" ");
|
||||
const subscriptionOnly = await subscriptionOnlyPromise;
|
||||
return (
|
||||
<ContentLayout
|
||||
title={
|
||||
@@ -148,6 +154,7 @@ export default async function ComplianceDetail({
|
||||
providerType={providerType}
|
||||
searchParams={resolvedSearchParams}
|
||||
targetSection={section}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</Suspense>
|
||||
</ContentLayout>
|
||||
@@ -172,18 +179,23 @@ export default async function ComplianceDetail({
|
||||
let selectedScan: ScanEntity | null = null;
|
||||
const selectedScanId = scanId || null;
|
||||
|
||||
const [metadataInfoData, attributesData, selectedScanResponse] =
|
||||
await Promise.all([
|
||||
getComplianceOverviewMetadataInfo({
|
||||
filters: {
|
||||
"filter[scan_id]": selectedScanId ?? undefined,
|
||||
},
|
||||
}),
|
||||
getComplianceAttributes(complianceId, selectedScanId ?? undefined),
|
||||
selectedScanId
|
||||
? getScan(selectedScanId, { include: "provider" })
|
||||
: Promise.resolve(null),
|
||||
]);
|
||||
const [
|
||||
metadataInfoData,
|
||||
attributesData,
|
||||
selectedScanResponse,
|
||||
subscriptionOnly,
|
||||
] = await Promise.all([
|
||||
getComplianceOverviewMetadataInfo({
|
||||
filters: {
|
||||
"filter[scan_id]": selectedScanId ?? undefined,
|
||||
},
|
||||
}),
|
||||
getComplianceAttributes(complianceId, selectedScanId ?? undefined),
|
||||
selectedScanId
|
||||
? getScan(selectedScanId, { include: "provider" })
|
||||
: Promise.resolve(null),
|
||||
subscriptionOnlyPromise,
|
||||
]);
|
||||
|
||||
// The compliance catalog is still warming after a deploy/restart. Show the
|
||||
// "still loading" state with a Try Again instead of rendering an empty page.
|
||||
@@ -309,6 +321,7 @@ export default async function ComplianceDetail({
|
||||
complianceId,
|
||||
latestCisIds.has(complianceId),
|
||||
)}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -5,11 +5,13 @@ const {
|
||||
getAuthHeadersMock,
|
||||
handleApiResponseMock,
|
||||
captureExceptionMock,
|
||||
isReportDownloadLockedMock,
|
||||
} = vi.hoisted(() => ({
|
||||
fetchMock: vi.fn(),
|
||||
getAuthHeadersMock: vi.fn(),
|
||||
handleApiResponseMock: vi.fn(),
|
||||
captureExceptionMock: vi.fn(),
|
||||
isReportDownloadLockedMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib", () => ({
|
||||
@@ -28,6 +30,12 @@ vi.mock("@sentry/nextjs", () => ({
|
||||
captureException: captureExceptionMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/report-download-access", () => ({
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR:
|
||||
"Report downloads require an active subscription.",
|
||||
isReportDownloadLocked: isReportDownloadLockedMock,
|
||||
}));
|
||||
|
||||
import {
|
||||
generateCrossProviderPdf,
|
||||
getCrossProviderComplianceOverview,
|
||||
@@ -56,6 +64,29 @@ beforeEach(() => {
|
||||
Authorization: "Bearer test-token",
|
||||
});
|
||||
handleApiResponseMock.mockResolvedValue({ data: null });
|
||||
isReportDownloadLockedMock.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
describe("cross-provider PDF reports for subscription-only tenants", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "generation",
|
||||
run: () => generateCrossProviderPdf({ complianceId: "csa_ccm_4.0" }),
|
||||
},
|
||||
{ name: "download", run: () => getCrossProviderPdfBinary("task-1") },
|
||||
])("rejects the $name without calling the API", async ({ run }) => {
|
||||
// Given
|
||||
isReportDownloadLockedMock.mockResolvedValue(true);
|
||||
|
||||
// When
|
||||
const result = await run();
|
||||
|
||||
// Then
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
error: "Report downloads require an active subscription.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("getCrossProviderComplianceOverview", () => {
|
||||
|
||||
@@ -48,6 +48,7 @@ interface CrossAccountDetailProps {
|
||||
providerType: KnownProviderType;
|
||||
searchParams: Record<string, string | string[] | undefined>;
|
||||
targetSection?: string;
|
||||
subscriptionOnly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -63,6 +64,7 @@ export const CrossAccountDetail = async ({
|
||||
providerType,
|
||||
searchParams,
|
||||
targetSection,
|
||||
subscriptionOnly = false,
|
||||
}: CrossAccountDetailProps) => {
|
||||
const filters = parseCrossAccountFilters(searchParams);
|
||||
|
||||
@@ -205,6 +207,7 @@ export const CrossAccountDetail = async ({
|
||||
providerType={providerType}
|
||||
filters={{ ...filters, scanIds: attrs.scan_ids }}
|
||||
latestPdf={latestPdf}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
}
|
||||
filters={
|
||||
|
||||
@@ -44,6 +44,7 @@ interface CrossProviderDetailProps {
|
||||
complianceId: string;
|
||||
searchParams: Record<string, string | string[] | undefined>;
|
||||
targetSection?: string;
|
||||
subscriptionOnly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -57,6 +58,7 @@ export const CrossProviderDetail = async ({
|
||||
complianceId,
|
||||
searchParams,
|
||||
targetSection,
|
||||
subscriptionOnly = false,
|
||||
}: CrossProviderDetailProps) => {
|
||||
const filters = parseCrossProviderFilters(searchParams);
|
||||
|
||||
@@ -206,6 +208,7 @@ export const CrossProviderDetail = async ({
|
||||
complianceId={complianceId}
|
||||
filters={{ ...filters, scanIds: attrs.scan_ids }}
|
||||
latestPdf={latestPdf}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
}
|
||||
filters={
|
||||
|
||||
@@ -2,6 +2,9 @@ import { render, screen, waitFor } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useCloudUpgradeStore } from "@/store/cloud-upgrade/store";
|
||||
import { PAID_PLAN_UPGRADE_FEATURE } from "@/types/cloud-upgrade";
|
||||
|
||||
import { CrossProviderPdfButton } from "./cross-provider-pdf-button";
|
||||
|
||||
// Radix dialogs/dropdowns rely on pointer-capture and scrollIntoView, which
|
||||
@@ -99,6 +102,7 @@ describe("CrossProviderPdfButton", () => {
|
||||
vi.clearAllMocks();
|
||||
storeState.tasks = {};
|
||||
generatePdfMock.mockResolvedValue({ taskId: "task-1" });
|
||||
useCloudUpgradeStore.getState().closeCloudUpgrade();
|
||||
});
|
||||
|
||||
const openGenerateModal = async (
|
||||
@@ -192,6 +196,34 @@ describe("CrossProviderPdfButton", () => {
|
||||
await waitFor(() => expect(downloadPdfMock).toHaveBeenCalledWith("task-7"));
|
||||
});
|
||||
|
||||
it.each([/download latest/i, /generate new report/i])(
|
||||
"opens the paid plan upgrade instead of %s for subscription-only tenants",
|
||||
async (label) => {
|
||||
// Given
|
||||
const user = userEvent.setup();
|
||||
render(
|
||||
<CrossProviderPdfButton
|
||||
{...props}
|
||||
latestPdf={{ taskId: "task-7", filename: "csa-latest.pdf" }}
|
||||
subscriptionOnly
|
||||
/>,
|
||||
);
|
||||
|
||||
// When
|
||||
await user.click(screen.getByRole("button", { name: /report/i }));
|
||||
await user.click(await screen.findByRole("menuitem", { name: label }));
|
||||
|
||||
// Then
|
||||
expect(downloadPdfMock).not.toHaveBeenCalled();
|
||||
expect(
|
||||
screen.queryByRole("dialog", { name: /generate/i }),
|
||||
).not.toBeInTheDocument();
|
||||
expect(useCloudUpgradeStore.getState().activeFeature).toBe(
|
||||
PAID_PLAN_UPGRADE_FEATURE.REPORT_DOWNLOAD,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a completed report downloadable after the ready toast closes", async () => {
|
||||
// Given
|
||||
storeState.tasks = {
|
||||
|
||||
@@ -12,6 +12,7 @@ import { FormButtons } from "@/components/shadcn/form";
|
||||
import { Input } from "@/components/shadcn/input/input";
|
||||
import { Modal } from "@/components/shadcn/modal";
|
||||
import { toast } from "@/components/shadcn/toast";
|
||||
import { useReportDownload } from "@/hooks/use-report-download";
|
||||
import {
|
||||
TASK_WATCHER_STATUS,
|
||||
trackAndPollTask,
|
||||
@@ -47,6 +48,8 @@ interface CrossProviderPdfButtonProps {
|
||||
/** Already-generated report matching these filters, if any — offered as an
|
||||
* instant download instead of forcing a re-generate. */
|
||||
latestPdf: LatestCrossProviderPdf | null;
|
||||
/** Prowler Cloud tenants without a paid plan cannot download reports. */
|
||||
subscriptionOnly?: boolean;
|
||||
}
|
||||
|
||||
export const CrossProviderPdfButton = ({
|
||||
@@ -54,7 +57,9 @@ export const CrossProviderPdfButton = ({
|
||||
providerType,
|
||||
filters,
|
||||
latestPdf,
|
||||
subscriptionOnly = false,
|
||||
}: CrossProviderPdfButtonProps) => {
|
||||
const runReportDownload = useReportDownload(subscriptionOnly);
|
||||
const [dialogOpen, setDialogOpen] = useState(false);
|
||||
const [reportName, setReportName] = useState("");
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
@@ -185,13 +190,15 @@ export const CrossProviderPdfButton = ({
|
||||
icon={<DownloadIcon />}
|
||||
label={`Download latest${formatGeneratedAt(availablePdf.completedAt)}`}
|
||||
description={availablePdf.filename}
|
||||
onSelect={() => downloadPdf(availablePdf.taskId)}
|
||||
onSelect={() =>
|
||||
runReportDownload(() => downloadPdf(availablePdf.taskId))
|
||||
}
|
||||
/>
|
||||
)}
|
||||
<ActionDropdownItem
|
||||
icon={<FileTextIcon />}
|
||||
label="Generate new report…"
|
||||
onSelect={() => setDialogOpen(true)}
|
||||
onSelect={() => runReportDownload(() => setDialogOpen(true))}
|
||||
/>
|
||||
</ActionDropdown>
|
||||
)}
|
||||
|
||||
@@ -7,6 +7,10 @@ import {
|
||||
getErrorMessage,
|
||||
} from "@/lib";
|
||||
import { hasActionError, type ActionErrorResult } from "@/lib/action-errors";
|
||||
import {
|
||||
isReportDownloadLocked,
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR,
|
||||
} from "@/lib/report-download-access";
|
||||
import { handleApiResponse } from "@/lib/server-actions-helper";
|
||||
import { SentryErrorSource, SentryErrorType } from "@/sentry";
|
||||
|
||||
@@ -171,6 +175,10 @@ export const generateAggregatedCompliancePdf = async (
|
||||
url: URL,
|
||||
operation: string,
|
||||
): Promise<{ taskId: string } | { error: string }> => {
|
||||
if (await isReportDownloadLocked()) {
|
||||
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
|
||||
try {
|
||||
@@ -211,6 +219,10 @@ export const getAggregatedCompliancePdfBinary = async ({
|
||||
operation: string;
|
||||
defaultFilename: string;
|
||||
}): Promise<ScanBinaryResult> => {
|
||||
if (await isReportDownloadLocked()) {
|
||||
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
|
||||
try {
|
||||
|
||||
@@ -15,18 +15,24 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import Compliance from "./page";
|
||||
|
||||
const {
|
||||
complianceFiltersSpy,
|
||||
complianceOverviewGridSpy,
|
||||
getComplianceOverviewMetadataInfoMock,
|
||||
getCompliancesOverviewMock,
|
||||
getScanMock,
|
||||
getScansMock,
|
||||
getThreatScoreMock,
|
||||
isCloudMock,
|
||||
loadComplianceWatchlistContextMock,
|
||||
} = vi.hoisted(() => ({
|
||||
complianceFiltersSpy: vi.fn(),
|
||||
complianceOverviewGridSpy: vi.fn(),
|
||||
getComplianceOverviewMetadataInfoMock: vi.fn(),
|
||||
getCompliancesOverviewMock: vi.fn(),
|
||||
getScanMock: vi.fn(),
|
||||
getScansMock: vi.fn(),
|
||||
getThreatScoreMock: vi.fn(),
|
||||
isCloudMock: vi.fn(() => false),
|
||||
loadComplianceWatchlistContextMock: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -41,12 +47,13 @@ vi.mock("@/actions/overview", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("@/actions/scans", () => ({
|
||||
getScan: getScanMock,
|
||||
getScans: getScansMock,
|
||||
getScansByState: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/shared/env", () => ({
|
||||
isCloud: () => false,
|
||||
isCloud: isCloudMock,
|
||||
}));
|
||||
|
||||
vi.mock("./_lib/watchlist-context", () => ({
|
||||
@@ -83,7 +90,10 @@ vi.mock("@/components/compliance", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("@/components/compliance/compliance-header/compliance-filters", () => ({
|
||||
ComplianceFilters: () => <div>Compliance filters</div>,
|
||||
ComplianceFilters: (props: { scans: Array<{ id: string }> }) => {
|
||||
complianceFiltersSpy(props);
|
||||
return <div>Compliance filters</div>;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/components/compliance/compliance-overview-grid", () => ({
|
||||
@@ -148,6 +158,8 @@ describe("Compliance overview page", () => {
|
||||
describe("Compliance overview task response", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
isCloudMock.mockReturnValue(false);
|
||||
getScanMock.mockResolvedValue(undefined);
|
||||
getScansMock.mockResolvedValue({
|
||||
data: [
|
||||
{
|
||||
@@ -233,6 +245,138 @@ describe("Compliance overview task response", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps partial scans out of the per-scan selector", async () => {
|
||||
// Given - a Cloud partial scan among the completed scans; it computes no
|
||||
// compliance, so selecting it would show nothing and its downloads fail
|
||||
isCloudMock.mockReturnValue(true);
|
||||
getScansMock.mockResolvedValue({
|
||||
data: [
|
||||
{
|
||||
id: "scan-1",
|
||||
attributes: {
|
||||
name: "Production scan",
|
||||
completed_at: "2026-08-05T17:00:00Z",
|
||||
},
|
||||
relationships: { provider: { data: { id: "provider-1" } } },
|
||||
},
|
||||
{
|
||||
id: "scan-partial",
|
||||
attributes: {
|
||||
name: "Re-check",
|
||||
completed_at: "2026-08-06T09:00:00Z",
|
||||
is_partial: true,
|
||||
},
|
||||
relationships: { provider: { data: { id: "provider-1" } } },
|
||||
},
|
||||
],
|
||||
included: [
|
||||
{
|
||||
type: "providers",
|
||||
id: "provider-1",
|
||||
attributes: { provider: "aws", uid: "123456789012", alias: "prod" },
|
||||
},
|
||||
],
|
||||
});
|
||||
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
|
||||
|
||||
// When
|
||||
const page = await Compliance({
|
||||
searchParams: Promise.resolve({ scanId: "scan-1" }),
|
||||
});
|
||||
render(page as ReactElement);
|
||||
|
||||
// Then - only the full scan reaches the selector, and the API is asked
|
||||
// for the flag that tells them apart
|
||||
expect(complianceFiltersSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
scans: [expect.objectContaining({ id: "scan-1" })],
|
||||
}),
|
||||
);
|
||||
expect(getScansMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
// Filtered at the API too, so a page full of re-checks cannot hide
|
||||
// the full scans behind it.
|
||||
filters: expect.objectContaining({ "filter[is_partial]": "false" }),
|
||||
fields: { scans: "name,completed_at,provider,is_partial" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not send the Cloud-only partial filter outside Prowler Cloud", async () => {
|
||||
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
|
||||
|
||||
await Compliance({ searchParams: Promise.resolve({ scanId: "scan-1" }) });
|
||||
|
||||
expect(getScansMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
filters: { "filter[state]": "completed" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the first full scan when the URL names a partial scan", async () => {
|
||||
// Given - a stale link to a partial scan, absent from the eligible list
|
||||
getScanMock.mockResolvedValue({
|
||||
data: { id: "scan-partial", attributes: { is_partial: true } },
|
||||
});
|
||||
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
|
||||
|
||||
// When
|
||||
const page = await Compliance({
|
||||
searchParams: Promise.resolve({ scanId: "scan-partial" }),
|
||||
});
|
||||
render(page as ReactElement);
|
||||
|
||||
// Then - the page selects a scan that has compliance instead
|
||||
expect(getScanMock).toHaveBeenCalledWith("scan-partial");
|
||||
expect(complianceFiltersSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ selectedScanId: "scan-1" }),
|
||||
);
|
||||
expect(getCompliancesOverviewMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ scanId: "scan-1" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the first full scan when the URL scan cannot be found", async () => {
|
||||
// Given - a deleted or mistyped id: the lookup returns an error, no data
|
||||
getScanMock.mockResolvedValue({ error: "Not found", status: 404 });
|
||||
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
|
||||
|
||||
// When
|
||||
const page = await Compliance({
|
||||
searchParams: Promise.resolve({ scanId: "scan-missing" }),
|
||||
});
|
||||
render(page as ReactElement);
|
||||
|
||||
// Then - no compliance request goes out for an id that does not exist
|
||||
expect(complianceFiltersSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ selectedScanId: "scan-1" }),
|
||||
);
|
||||
expect(getCompliancesOverviewMock).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ scanId: "scan-missing" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps trusting a URL scan id that is older than the listed page", async () => {
|
||||
// Given - a full scan beyond the first page, shaped like an OSS response
|
||||
// that carries no is_partial field at all
|
||||
getScanMock.mockResolvedValue({
|
||||
data: { id: "scan-old", attributes: { name: "Old scan" } },
|
||||
});
|
||||
getCompliancesOverviewMock.mockResolvedValue({ data: [] });
|
||||
|
||||
// When
|
||||
const page = await Compliance({
|
||||
searchParams: Promise.resolve({ scanId: "scan-old" }),
|
||||
});
|
||||
render(page as ReactElement);
|
||||
|
||||
// Then
|
||||
expect(complianceFiltersSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ selectedScanId: "scan-old" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the invalid scan message for a JSON:API error response", async () => {
|
||||
// Given - handleApiResponse converted a client error to its error result
|
||||
getCompliancesOverviewMock.mockResolvedValue({
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
getCompliancesOverview,
|
||||
} from "@/actions/compliances";
|
||||
import { getThreatScore } from "@/actions/overview";
|
||||
import { getScans, getScansByState } from "@/actions/scans";
|
||||
import { getScan, getScans, getScansByState } from "@/actions/scans";
|
||||
import {
|
||||
ComplianceSkeletonGrid,
|
||||
NoScansAvailable,
|
||||
@@ -20,6 +20,7 @@ import { Alert, AlertDescription } from "@/components/shadcn/alert";
|
||||
import { Card, CardContent } from "@/components/shadcn/card/card";
|
||||
import { ContentLayout } from "@/components/shadcn/content-layout";
|
||||
import { pickLatestCisPerProvider } from "@/lib/compliance/compliance-report-types";
|
||||
import { isReportDownloadLocked } from "@/lib/report-download-access";
|
||||
import { isCloud } from "@/lib/shared/env";
|
||||
import {
|
||||
ExpandedScanData,
|
||||
@@ -40,6 +41,31 @@ import {
|
||||
import type { ComplianceWatchlistContext } from "./_lib/watchlist-context";
|
||||
import { loadComplianceWatchlistContext } from "./_lib/watchlist-context";
|
||||
|
||||
/**
|
||||
* A scan id from the URL is trusted when it is listed, or when a single lookup
|
||||
* returns a scan that is not partial (older full scans keep working). A partial
|
||||
* scan, reached through a stale link, or an id the API cannot find, has no
|
||||
* compliance to show, so the caller falls back to the first eligible scan.
|
||||
*/
|
||||
async function resolveUrlScanId(
|
||||
scanIdFromUrl: string | undefined,
|
||||
eligibleScans: ExpandedScanData[],
|
||||
): Promise<string | undefined> {
|
||||
if (!scanIdFromUrl) return undefined;
|
||||
if (eligibleScans.some((scan) => scan.id === scanIdFromUrl)) {
|
||||
return scanIdFromUrl;
|
||||
}
|
||||
|
||||
const urlScan = (await getScan(scanIdFromUrl)) as
|
||||
| { data?: { attributes?: { is_partial?: boolean } } }
|
||||
| undefined;
|
||||
const scan = urlScan?.data;
|
||||
if (!scan) return undefined;
|
||||
|
||||
// OSS scans carry no is_partial at all, so only an explicit true excludes.
|
||||
return scan.attributes?.is_partial === true ? undefined : scanIdFromUrl;
|
||||
}
|
||||
|
||||
export default async function Compliance({
|
||||
searchParams,
|
||||
}: {
|
||||
@@ -124,16 +150,23 @@ export default async function Compliance({
|
||||
);
|
||||
}
|
||||
|
||||
const scansData = await getScans({
|
||||
filters: {
|
||||
"filter[state]": "completed",
|
||||
},
|
||||
pageSize: 50,
|
||||
fields: {
|
||||
scans: "name,completed_at,provider",
|
||||
},
|
||||
include: "provider",
|
||||
});
|
||||
const [subscriptionOnly, scansData] = await Promise.all([
|
||||
isReportDownloadLocked(),
|
||||
getScans({
|
||||
filters: {
|
||||
"filter[state]": "completed",
|
||||
// Partial scans compute no compliance. Exclude them at the API so the
|
||||
// page below never fills up with them; the filter is Cloud-only.
|
||||
...(isCloud() ? { "filter[is_partial]": "false" } : {}),
|
||||
},
|
||||
pageSize: 50,
|
||||
fields: {
|
||||
// is_partial is Cloud-only; the OSS API ignores unknown sparse fields.
|
||||
scans: "name,completed_at,provider,is_partial",
|
||||
},
|
||||
include: "provider",
|
||||
}),
|
||||
]);
|
||||
|
||||
if (!scansData?.data) {
|
||||
return (
|
||||
@@ -157,7 +190,10 @@ export default async function Compliance({
|
||||
);
|
||||
}
|
||||
|
||||
// Belt and braces for an API without the filter: partial scans never
|
||||
// compute compliance, so they have nothing to show or download here.
|
||||
const expandedScansData: ExpandedScanData[] = scansData.data
|
||||
.filter((scan: ScanProps) => !scan.attributes?.is_partial)
|
||||
.filter((scan: ScanProps) => scan.relationships?.provider?.data?.id)
|
||||
.map((scan: ScanProps) => {
|
||||
const providerId = scan.relationships!.provider!.data!.id;
|
||||
@@ -187,7 +223,9 @@ export default async function Compliance({
|
||||
? scanIdParam[0]
|
||||
: scanIdParam;
|
||||
const selectedScanId: string | null =
|
||||
scanIdFromUrl || expandedScansData[0]?.id || null;
|
||||
(await resolveUrlScanId(scanIdFromUrl, expandedScansData)) ||
|
||||
expandedScansData[0]?.id ||
|
||||
null;
|
||||
const onboardingAction = selectedScanId
|
||||
? { flowId: "view-compliance" }
|
||||
: {
|
||||
@@ -256,6 +294,7 @@ export default async function Compliance({
|
||||
provider={selectedScan.providerInfo.provider}
|
||||
selectedScan={selectedScanData}
|
||||
sectionScores={threatScoreData.sectionScores}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
@@ -273,6 +312,7 @@ export default async function Compliance({
|
||||
scanId={selectedScanId}
|
||||
selectedScan={selectedScanData}
|
||||
watchlistPromise={watchlistPromise}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</Suspense>
|
||||
</>
|
||||
@@ -302,11 +342,13 @@ const SSRComplianceGrid = async ({
|
||||
scanId,
|
||||
selectedScan,
|
||||
watchlistPromise,
|
||||
subscriptionOnly,
|
||||
}: {
|
||||
searchParams: SearchParamsProps;
|
||||
scanId: string | null;
|
||||
selectedScan?: ScanEntity;
|
||||
watchlistPromise: Promise<ComplianceWatchlistContext>;
|
||||
subscriptionOnly: boolean;
|
||||
}) => {
|
||||
const regionFilter = searchParams["filter[region__in]"]?.toString() || "";
|
||||
|
||||
@@ -388,6 +430,7 @@ const SSRComplianceGrid = async ({
|
||||
catalogEntries={watchlist.entries}
|
||||
providerType={providerType}
|
||||
canManageWatchlist={watchlist.canManage}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</ComplianceOverviewPanel>
|
||||
);
|
||||
|
||||
+15
-14
@@ -59,6 +59,10 @@ export default async function RootLayout({
|
||||
// Skip Cloud-only onboarding fetches and orchestrators in OSS.
|
||||
const cloudEnabled = isCloud();
|
||||
|
||||
// Every deployment needs the provider count: it drives the first-run redirect
|
||||
// and the sidebar's Add Provider action.
|
||||
const providersPromise = getProviders({ page: 1, pageSize: 1 });
|
||||
|
||||
// One-time server-side Registry gate per request: only an ELIGIBLE answer
|
||||
// shows the sidebar entry; UNKNOWN and INELIGIBLE both hide it. Started
|
||||
// here so it resolves in parallel with the Cloud onboarding fetches.
|
||||
@@ -69,29 +73,27 @@ export default async function RootLayout({
|
||||
// Fail-open: unknown scan state is treated as "has data" so the banner never blocks
|
||||
// progression on a fetch error.
|
||||
let hasCompletedScan = true;
|
||||
// Tri-state: true = has providers, false = zero providers, undefined = fetch failed (gate fails open).
|
||||
let hasProviders: boolean | undefined = false;
|
||||
// Scopes the onboarding steps' local markers, so resolving them for one
|
||||
// tenant does not silence them for another.
|
||||
let tenantId: string | null = null;
|
||||
|
||||
if (cloudEnabled) {
|
||||
const [providersData, scansByState] = await Promise.all([
|
||||
getProviders({ page: 1, pageSize: 1 }),
|
||||
getScansByState(),
|
||||
]);
|
||||
const scansByState = await getScansByState();
|
||||
hasCompletedScan = Array.isArray(scansByState?.data)
|
||||
? scansByState.data.some(
|
||||
(scan: { attributes?: { state?: string } }) =>
|
||||
scan.attributes?.state === SCAN_STATES.COMPLETED,
|
||||
)
|
||||
: true;
|
||||
hasProviders = Array.isArray(providersData?.data)
|
||||
? providersData.data.length > 0
|
||||
: undefined;
|
||||
tenantId = (await auth())?.tenantId ?? null;
|
||||
}
|
||||
|
||||
const providersData = await providersPromise;
|
||||
// Tri-state: true = has providers, false = zero providers, undefined = fetch failed (gate fails open).
|
||||
const hasProviders: boolean | undefined = Array.isArray(providersData?.data)
|
||||
? providersData.data.length > 0
|
||||
: undefined;
|
||||
|
||||
const registryEligible =
|
||||
(await registryAccessPromise).status === REGISTRY_ACCESS.ELIGIBLE;
|
||||
|
||||
@@ -114,13 +116,12 @@ export default async function RootLayout({
|
||||
<Suspense>
|
||||
<NavigationProgress />
|
||||
</Suspense>
|
||||
{/* Store uses boolean; gate receives tri-state to fail open on fetch errors. */}
|
||||
<StoreInitializer
|
||||
values={{ hasProviders: hasProviders ?? false, registryEligible }}
|
||||
/>
|
||||
{/* Tri-state for both: an unknown count leaves the store unresolved and the gate closed. */}
|
||||
<StoreInitializer values={{ hasProviders, registryEligible }} />
|
||||
{/* Every deployment: an empty tenant lands on the add-provider wizard once. */}
|
||||
<OnboardingGate hasProviders={hasProviders} tenantId={tenantId} />
|
||||
{cloudEnabled && (
|
||||
<>
|
||||
<OnboardingGate hasProviders={hasProviders} />
|
||||
{/* Single mount point so the watcher survives post-connect navigation. */}
|
||||
<OnboardingCheckpointWatcher tenantId={tenantId} />
|
||||
{/* Persistent banner shown only while a guided sequence is active. */}
|
||||
|
||||
@@ -54,6 +54,19 @@ export class ProvidersPageHarness extends BrowserHarness<OrgFixture> {
|
||||
return this.countRequests("POST", "/apply");
|
||||
}
|
||||
|
||||
/** `POST /providers` alone: secrets and connection checks nest under it. */
|
||||
get providerCreateCallCount(): number {
|
||||
return this.requestLog.filter(
|
||||
(request) =>
|
||||
request.method === "POST" &&
|
||||
new URL(request.url).pathname.replace(/\/$/, "").endsWith("/providers"),
|
||||
).length;
|
||||
}
|
||||
|
||||
get secretCreateCallCount(): number {
|
||||
return this.countRequests("POST", "/providers/secrets");
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether any apply asked the endpoint to include related resources, which it
|
||||
* rejects outright — a tripwire, not a preference.
|
||||
@@ -83,7 +96,7 @@ export class ProvidersPageHarness extends BrowserHarness<OrgFixture> {
|
||||
).length;
|
||||
}
|
||||
|
||||
private get connectionCallCount(): number {
|
||||
get connectionCallCount(): number {
|
||||
return this.countRequests("POST", "/connection");
|
||||
}
|
||||
|
||||
@@ -198,10 +211,75 @@ export class ProvidersPageHarness extends BrowserHarness<OrgFixture> {
|
||||
/** Enter the AWS Organizations onboarding flow from a fresh wizard. */
|
||||
async chooseAwsOrganizations(): Promise<void> {
|
||||
await this.selectProviderType(/Amazon Web Services/);
|
||||
await this.chooseMethod(/Add Multiple Accounts With AWS Organizations/);
|
||||
// AWS hosts its single-account/organization switch as tabs on its connect step.
|
||||
const tab = await this.waitFor(() =>
|
||||
this.byRoleName("tab", /Full AWS Organization/),
|
||||
);
|
||||
await this.user.click(tab);
|
||||
await this.waitForText(/Organization Details/);
|
||||
}
|
||||
|
||||
/** Labels of the wizard's progress stepper, top to bottom. */
|
||||
stepperLabels(): string[] {
|
||||
return Array.from(
|
||||
document.querySelectorAll('nav[aria-label="Wizard progress"] span'),
|
||||
(node) => node.textContent?.trim() ?? "",
|
||||
);
|
||||
}
|
||||
|
||||
/** Wait until the AWS single-account connect step (with its method tabs) is showing. */
|
||||
async waitForAwsConnectStep(): Promise<void> {
|
||||
await this.waitFor(
|
||||
() => this.byRoleName("tab", /Single AWS Account/),
|
||||
undefined,
|
||||
"AWS connect step",
|
||||
);
|
||||
}
|
||||
|
||||
/** Type the account id and static keys on the AWS one-step connect form. */
|
||||
async fillAwsAccountKeys({
|
||||
accountId,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
}: {
|
||||
accountId: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
}): Promise<void> {
|
||||
const accountInput = await this.waitFor(() =>
|
||||
this.inputByName("providerUid"),
|
||||
);
|
||||
await this.user.fill(accountInput, accountId);
|
||||
const keyInput = await this.waitFor(() =>
|
||||
this.inputByName("aws_access_key_id"),
|
||||
);
|
||||
await this.user.fill(keyInput, accessKeyId);
|
||||
const secretInput = await this.waitFor(() =>
|
||||
this.inputByName("aws_secret_access_key"),
|
||||
);
|
||||
await this.user.fill(secretInput, secretAccessKey);
|
||||
}
|
||||
|
||||
/** Submit the AWS one-step form; waits for its action to become enabled. */
|
||||
async connectAccount(): Promise<void> {
|
||||
await this.clickPrimary(/Connect account/);
|
||||
}
|
||||
|
||||
/** Wait until the provider wizard reached its launch step. */
|
||||
async waitForProviderLaunchStep(timeoutMs = 20000): Promise<void> {
|
||||
await this.waitForText(/Scan Schedule/, timeoutMs);
|
||||
}
|
||||
|
||||
/** Switch back to a single account from the organization flow's tabs. */
|
||||
async switchToAwsSingleAccount(): Promise<void> {
|
||||
const tab = await this.waitFor(
|
||||
() => this.byRoleName("tab", /Single AWS Account/),
|
||||
undefined,
|
||||
"Single AWS Account tab",
|
||||
);
|
||||
await this.user.click(tab);
|
||||
}
|
||||
|
||||
/** Select GCP and open the GCP Organization method card (no advance wait). */
|
||||
async chooseGcpOrganizationsMethod(): Promise<void> {
|
||||
await this.selectProviderType(/Google Cloud Platform/);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user