mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-10-04 10:14:20 +00:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c5e8f7e176 | ||
|
|
c2b8092461 | ||
|
|
26d9d24e5d | ||
|
|
ee59e35bc2 | ||
|
|
e0fa23b9ee | ||
|
|
4dbc3c7e74 | ||
|
|
576433d85d | ||
|
|
bf179212a5 | ||
|
|
60f936a10b | ||
|
|
15630f54d2 | ||
|
|
706603fe4d | ||
|
|
dc67fe4f37 | ||
|
|
2c233c2f6c | ||
|
|
69e1d19abe | ||
|
|
859421b0ec | ||
|
|
ea36f12a01 | ||
|
|
50a9138bea | ||
|
|
09821e6328 | ||
|
|
94899e20fd | ||
|
|
79da676c86 | ||
|
|
8b35b69731 | ||
|
|
2dbdcf2b50 | ||
|
|
4f145a3c0d | ||
|
|
538d5273c7 | ||
|
|
e0379250ae | ||
|
|
a7cf5690e8 |
@@ -110,11 +110,20 @@ DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY=""
|
||||
DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN=""
|
||||
|
||||
# The AWS region where your S3 bucket is located (e.g., "us-east-1")
|
||||
# Required if the bucket uses SSE-KMS: download URLs are then signed with SigV4, which
|
||||
# is scoped to this region, so it must match the bucket's
|
||||
DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION=""
|
||||
|
||||
# The name of the S3 bucket where scan output should be stored
|
||||
DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET=""
|
||||
|
||||
# The storage endpoint the API and Celery workers use to upload and list scan output
|
||||
# (e.g. "http://minio:9000"). Leave empty on AWS S3. Set it when scan output is stored on
|
||||
# S3-compatible object storage such as MinIO instead of real S3.
|
||||
# If set without DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL below, report download URLs are
|
||||
# signed against this internal host, and a browser outside the container network cannot open them.
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL=""
|
||||
|
||||
# The storage address the browser can reach, used only to sign report download URLs
|
||||
# (e.g. "https://storage.example.com"). Leave empty on AWS S3. Set it when storage is
|
||||
# only reachable inside the container network, such as MinIO on "http://minio:9000".
|
||||
@@ -165,7 +174,7 @@ SENTRY_RELEASE=local
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.43.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.44.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -11,6 +11,7 @@ on:
|
||||
- "v5.*"
|
||||
paths:
|
||||
- ".github/workflows/ui-e2e-tests-v2.yml"
|
||||
- ".github/workflows/test-impact-analysis.yml"
|
||||
- ".github/test-impact.yml"
|
||||
- "ui/**"
|
||||
- "api/**" # API changes can affect UI E2E
|
||||
@@ -237,8 +238,8 @@ jobs:
|
||||
|
||||
- name: Add AWS credentials for testing
|
||||
run: |
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
echo "AWS_ACCESS_KEY_ID=${E2E_AWS_PROVIDER_ACCESS_KEY}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${E2E_AWS_PROVIDER_SECRET_KEY}" >> .env
|
||||
|
||||
- name: Build API image from current code
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Adds a periodic sweep that drops orphaned Attack Paths temp Neo4j scan databases left behind when a worker or Neo4j crashes mid-scan, before they accumulate unbounded
|
||||
@@ -0,0 +1 @@
|
||||
Resources no longer keep a stale failed findings count forever when a scoped or imported scan for the same provider completes after a full scan, which used to make the full scan skip its own cleanup
|
||||
@@ -0,0 +1 @@
|
||||
Providers whose most recent completed scan has no `completed_at` timestamp are no longer missing from every endpoint that reports a provider's latest scan, which now falls back to scan creation order instead of skipping the provider
|
||||
@@ -0,0 +1 @@
|
||||
Unify how every endpoint resolves a provider latest completed scan, so overlapping scans no longer make findings, compliance and mute rules read from different scans
|
||||
@@ -0,0 +1 @@
|
||||
Scan output uploads and downloads can now target S3-compatible object storage such as MinIO directly via `DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL`, instead of relying on process-wide AWS environment variables that also hijacked unrelated AWS API calls
|
||||
@@ -0,0 +1 @@
|
||||
Scan report downloads from an S3 bucket with default SSE-KMS encryption no longer fail with an `InvalidArgument` error: when `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` is set, presigned download URLs are signed with AWS Signature Version 4 for that region
|
||||
@@ -0,0 +1 @@
|
||||
`POST /api/v1/scans` again returns the new scan id in the response `task_args`, which had been empty since the scan broker publish moved to transaction commit
|
||||
@@ -0,0 +1 @@
|
||||
Celery loggers are now declared explicitly in `custom_logging.py` so fatal worker errors are no longer silenced by `disable_existing_loggers=True`. All long-running services in `docker-compose.yml` now have `restart: unless-stopped` so containers recover automatically after unexpected crashes.
|
||||
+1
-1
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.44.0"
|
||||
version = "1.45.0"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
|
||||
@@ -207,6 +207,11 @@ def drop_database(database: str) -> None:
|
||||
sink_module.get_backend().drop_database(database)
|
||||
|
||||
|
||||
def list_databases() -> list[str]:
|
||||
"""List database names on the ingest cluster. Temp scan DBs always live here."""
|
||||
return ingest.list_databases()
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
return sink_module.get_backend().drop_subgraph(database, provider_id)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from api.attack_paths.ingest.driver import (
|
||||
get_session,
|
||||
get_uri,
|
||||
init_driver,
|
||||
list_databases,
|
||||
run_cypher,
|
||||
)
|
||||
|
||||
@@ -25,5 +26,6 @@ __all__ = [
|
||||
"get_session",
|
||||
"get_uri",
|
||||
"init_driver",
|
||||
"list_databases",
|
||||
"run_cypher",
|
||||
]
|
||||
|
||||
@@ -165,6 +165,14 @@ def drop_database(database: str) -> None:
|
||||
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
|
||||
|
||||
|
||||
def list_databases() -> list[str]:
|
||||
"""List every database name on the Neo4j temp-database cluster."""
|
||||
# A cluster returns one row per hosting server, so dedupe on name
|
||||
with get_session() as session:
|
||||
result = session.run("SHOW DATABASES YIELD name RETURN DISTINCT name")
|
||||
return [record["name"] for record in result]
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
"""Best-effort cache clear for a Neo4j database."""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
|
||||
@@ -409,7 +409,7 @@ def batch_delete(tenant_id, queryset, batch_size=settings.DJANGO_DELETION_BATCH_
|
||||
|
||||
Args:
|
||||
tenant_id (str): Tenant ID the queryset belongs to.
|
||||
queryset (QuerySet): The queryset of objects to delete.
|
||||
queryset: The queryset of objects to delete.
|
||||
batch_size (int): The number of objects to delete in each batch.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
from django.db import migrations
|
||||
|
||||
TASK_NAME = "attack-paths-reap-orphaned-tmp-databases"
|
||||
INTERVAL_HOURS = 6
|
||||
|
||||
|
||||
def create_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
schedule, _ = IntervalSchedule.objects.get_or_create(
|
||||
every=INTERVAL_HOURS,
|
||||
period="hours",
|
||||
)
|
||||
|
||||
PeriodicTask.objects.update_or_create(
|
||||
name=TASK_NAME,
|
||||
defaults={
|
||||
"task": TASK_NAME,
|
||||
"interval": schedule,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def delete_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
PeriodicTask.objects.filter(name=TASK_NAME).delete()
|
||||
|
||||
# Clean up the schedule if no other task references it
|
||||
IntervalSchedule.objects.filter(
|
||||
every=INTERVAL_HOURS,
|
||||
period="hours",
|
||||
periodictask__isnull=True,
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0099_delete_tenant_onboarding_profile"),
|
||||
("django_celery_beat", "0019_alter_periodictasks_options"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_periodic_task, delete_periodic_task),
|
||||
]
|
||||
@@ -617,9 +617,66 @@ class Task(RowLevelSecurityProtectedModel):
|
||||
resource_name = "tasks"
|
||||
|
||||
|
||||
class ScanQuerySet(models.QuerySet):
|
||||
"""Shared selectors for "the latest scan of a provider".
|
||||
|
||||
The queryset must already be scoped by the caller: manager, tenant, RBAC,
|
||||
providers and database alias.
|
||||
"""
|
||||
|
||||
# How "which completed scan is the provider's current one" is ordered.
|
||||
LATEST_ORDER_BY = (
|
||||
models.F("completed_at").desc(nulls_last=True),
|
||||
models.F("inserted_at").desc(),
|
||||
models.F("id").desc(),
|
||||
)
|
||||
|
||||
def _eligible_for_latest(self) -> "ScanQuerySet":
|
||||
"""Restrict to the scans that may be a provider's latest.
|
||||
|
||||
Returns:
|
||||
ScanQuerySet: The completed scans.
|
||||
"""
|
||||
return self.filter(state=StateChoices.COMPLETED)
|
||||
|
||||
def latest_per_provider(self) -> "ScanQuerySet":
|
||||
"""Pick each provider's latest scan with `DISTINCT ON (provider_id)`.
|
||||
|
||||
Returns:
|
||||
ScanQuerySet: One scan per provider, the latest one.
|
||||
"""
|
||||
return (
|
||||
self._eligible_for_latest()
|
||||
.order_by("provider_id", *self.LATEST_ORDER_BY)
|
||||
.distinct("provider_id")
|
||||
)
|
||||
|
||||
def latest_ids_per_provider(self) -> list[UUID]:
|
||||
"""Evaluate `latest_per_provider` and return the scan ids.
|
||||
|
||||
The ids are materialised so callers can pass them as a literal `IN`
|
||||
list; as a subquery Postgres misestimates the row count and picks a
|
||||
slow nested loop.
|
||||
|
||||
Returns:
|
||||
list[UUID]: The id of each provider's latest scan.
|
||||
"""
|
||||
return list(self.latest_per_provider().values_list("id", flat=True))
|
||||
|
||||
def latest_first(self) -> "ScanQuerySet":
|
||||
"""Order eligible scans newest first, without deduplicating per provider.
|
||||
|
||||
Expects the queryset to be already filtered to a single provider.
|
||||
|
||||
Returns:
|
||||
ScanQuerySet: The eligible scans, latest first.
|
||||
"""
|
||||
return self._eligible_for_latest().order_by(*self.LATEST_ORDER_BY)
|
||||
|
||||
|
||||
class Scan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
objects = ActiveProviderManager.from_queryset(ScanQuerySet)()
|
||||
all_objects = ScanQuerySet.as_manager()
|
||||
|
||||
_SCOPING_SCANNER_ARG_KEYS_CACHE: tuple[str, ...] | None = None
|
||||
|
||||
@@ -726,6 +783,12 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
name="scans_prov_state_ins_desc_idx",
|
||||
),
|
||||
# TODO This might replace `scans_prov_state_ins_desc_idx` completely. Review usage
|
||||
# Since `ScanQuerySet`, no code path reads a provider's
|
||||
# completed scans by `-inserted_at`. The only query left that
|
||||
# matches this index (and `scans_prov_state_ins_desc_idx` above)
|
||||
# is `GET /scans?filter[provider]=…&filter[state]=completed` with
|
||||
# the default sort. Both are candidates to drop in a follow-up
|
||||
# once production `pg_stat_user_indexes.idx_scan` confirms it.
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-inserted_at"],
|
||||
condition=Q(state=StateChoices.COMPLETED),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.44.0
|
||||
version: 1.45.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
|
||||
@@ -187,6 +187,27 @@ class TestRoutingByDatabasePrefix:
|
||||
sink_backend_stub.drop_database.assert_called_once_with("db-tenant-abc")
|
||||
mock_ingest.drop_database.assert_not_called()
|
||||
|
||||
def test_list_databases_always_routes_to_ingest(self, sink_backend_stub):
|
||||
with patch("api.attack_paths.database.ingest") as mock_ingest:
|
||||
mock_ingest.list_databases.return_value = ["db-tmp-scan-uuid-1"]
|
||||
|
||||
assert db_module.list_databases() == ["db-tmp-scan-uuid-1"]
|
||||
|
||||
mock_ingest.list_databases.assert_called_once_with()
|
||||
|
||||
def test_ingest_list_databases_dedupes_cluster_rows(self):
|
||||
from api.attack_paths.ingest import driver as ingest_driver
|
||||
|
||||
with patch.object(ingest_driver, "get_session") as mock_get_session:
|
||||
session = mock_get_session.return_value.__enter__.return_value
|
||||
session.run.return_value = [{"name": "db-tmp-scan-uuid-1"}]
|
||||
|
||||
assert ingest_driver.list_databases() == ["db-tmp-scan-uuid-1"]
|
||||
|
||||
session.run.assert_called_once_with(
|
||||
"SHOW DATABASES YIELD name RETURN DISTINCT name"
|
||||
)
|
||||
|
||||
def test_clear_cache_routes_temp_to_ingest(self, sink_backend_stub):
|
||||
with patch("api.attack_paths.database.ingest") as mock_ingest:
|
||||
db_module.clear_cache("db-tmp-scan-uuid-1")
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from allauth.socialaccount.models import SocialApp
|
||||
from api.db_router import MainRouter
|
||||
from api.models import (
|
||||
Provider,
|
||||
ProviderComplianceScore,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
SAMLConfiguration,
|
||||
SAMLDomainIndex,
|
||||
Scan,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
TenantComplianceSummary,
|
||||
@@ -524,3 +526,226 @@ class TestTenantComplianceSummaryModel:
|
||||
|
||||
assert summary1.id != summary2.id
|
||||
assert summary1.requirements_passed != summary2.requirements_passed
|
||||
|
||||
|
||||
def _latest_scan_fixture(tenant, provider, *, completed_at, inserted_at=None, **kwargs):
|
||||
scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=kwargs.pop("trigger", Scan.TriggerChoices.MANUAL),
|
||||
state=kwargs.pop("state", StateChoices.COMPLETED),
|
||||
completed_at=completed_at,
|
||||
**kwargs,
|
||||
)
|
||||
if inserted_at is not None:
|
||||
# `inserted_at` is auto_now_add, so it has to be forced after the fact.
|
||||
Scan.all_objects.filter(pk=scan.pk).update(inserted_at=inserted_at)
|
||||
scan.refresh_from_db()
|
||||
return scan
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetOrdering:
|
||||
def test_completed_later_wins_over_inserted_later(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""The scan that FINISHED last is current, not the one that started last."""
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
finished_last = _latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now - timedelta(hours=3),
|
||||
completed_at=now,
|
||||
)
|
||||
_latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now - timedelta(hours=1),
|
||||
completed_at=now - timedelta(hours=1),
|
||||
)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [finished_last.id]
|
||||
|
||||
def test_null_completed_at_provider_is_still_returned(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""NULLS LAST, not `completed_at__isnull=False`.
|
||||
|
||||
Excluding NULL `completed_at` would drop the provider from every
|
||||
"latest" endpoint instead of falling back to `inserted_at`.
|
||||
"""
|
||||
tenant, *_ = tenants_fixture
|
||||
only_scan = _latest_scan_fixture(tenant, aws_provider, completed_at=None)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [only_scan.id]
|
||||
|
||||
def test_null_completed_at_never_outranks_a_finished_scan(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""Postgres sorts NULLs first under DESC; NULLS LAST is what fixes it."""
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
finished = _latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now - timedelta(hours=2),
|
||||
completed_at=now - timedelta(hours=2),
|
||||
)
|
||||
_latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
inserted_at=now,
|
||||
completed_at=None,
|
||||
)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [finished.id]
|
||||
|
||||
def test_id_breaks_an_exact_timestamp_tie_deterministically(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
scans = [
|
||||
_latest_scan_fixture(
|
||||
tenant, aws_provider, inserted_at=now, completed_at=now
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
expected = max(scan.id for scan in scans)
|
||||
|
||||
picks = {
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider()[0]
|
||||
for _ in range(5)
|
||||
}
|
||||
assert picks == {expected}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetEligibility:
|
||||
def test_unfinished_scans_are_excluded(self, tenants_fixture, aws_provider):
|
||||
tenant, *_ = tenants_fixture
|
||||
_latest_scan_fixture(
|
||||
tenant,
|
||||
aws_provider,
|
||||
completed_at=None,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
assert (
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetManagerChoice:
|
||||
def test_active_manager_hides_soft_deleted_providers(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""`Scan.objects` drops soft-deleted providers, `all_objects` keeps them.
|
||||
|
||||
The queryset must not decide this for the caller.
|
||||
"""
|
||||
tenant, *_ = tenants_fixture
|
||||
scan = _latest_scan_fixture(
|
||||
tenant, aws_provider, completed_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
Provider.all_objects.filter(pk=aws_provider.pk).update(is_deleted=True)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id
|
||||
).latest_ids_per_provider() == [scan.id]
|
||||
assert Scan.objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetPerProviderScoping:
|
||||
def test_one_scan_per_provider(self, tenants_fixture, aws_provider_pair):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider_one, provider_two = aws_provider_pair
|
||||
now = datetime.now(UTC)
|
||||
|
||||
newest_one = _latest_scan_fixture(tenant, provider_one, completed_at=now)
|
||||
_latest_scan_fixture(tenant, provider_one, completed_at=now - timedelta(days=1))
|
||||
newest_two = _latest_scan_fixture(tenant, provider_two, completed_at=now)
|
||||
|
||||
assert set(
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider()
|
||||
) == {newest_one.id, newest_two.id}
|
||||
|
||||
def test_caller_filters_are_preserved(self, tenants_fixture, aws_provider_pair):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider_one, provider_two = aws_provider_pair
|
||||
now = datetime.now(UTC)
|
||||
|
||||
scan_one = _latest_scan_fixture(tenant, provider_one, completed_at=now)
|
||||
_latest_scan_fixture(tenant, provider_two, completed_at=now)
|
||||
|
||||
assert Scan.all_objects.filter(
|
||||
tenant_id=tenant.id, provider__in=[provider_one]
|
||||
).latest_ids_per_provider() == [scan_one.id]
|
||||
|
||||
def test_latest_first_is_ordered_not_deduplicated(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
newest = _latest_scan_fixture(tenant, aws_provider, completed_at=now)
|
||||
older = _latest_scan_fixture(
|
||||
tenant, aws_provider, completed_at=now - timedelta(days=1)
|
||||
)
|
||||
|
||||
ordered = list(
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant.id, provider_id=aws_provider.id
|
||||
).latest_first()
|
||||
)
|
||||
assert [scan.id for scan in ordered] == [newest.id, older.id]
|
||||
|
||||
def test_tenant_isolation(self, tenants_fixture, aws_provider):
|
||||
tenant, other_tenant, *_ = tenants_fixture
|
||||
_latest_scan_fixture(tenant, aws_provider, completed_at=datetime.now(UTC))
|
||||
|
||||
assert (
|
||||
Scan.all_objects.filter(tenant_id=other_tenant.id).latest_ids_per_provider()
|
||||
== []
|
||||
)
|
||||
|
||||
def test_empty_queryset_returns_empty_list(self, tenants_fixture):
|
||||
tenant, *_ = tenants_fixture
|
||||
assert (
|
||||
Scan.all_objects.filter(tenant_id=tenant.id).latest_ids_per_provider() == []
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetPerProviderQuerysetShape:
|
||||
def test_returns_a_queryset_not_a_list(self, tenants_fixture, aws_provider):
|
||||
tenant, *_ = tenants_fixture
|
||||
_latest_scan_fixture(tenant, aws_provider, completed_at=datetime.now(UTC))
|
||||
|
||||
qs = Scan.all_objects.filter(tenant_id=tenant.id).latest_per_provider()
|
||||
# Callers chain .values(...) / .values_list(...) onto this.
|
||||
assert qs.values_list("provider_id", flat=True).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestScanQuerySetRelatedManager:
|
||||
def test_reverse_relation_exposes_the_methods(self, tenants_fixture, aws_provider):
|
||||
tenant, *_ = tenants_fixture
|
||||
now = datetime.now(UTC)
|
||||
|
||||
newest = _latest_scan_fixture(tenant, aws_provider, completed_at=now)
|
||||
_latest_scan_fixture(tenant, aws_provider, completed_at=now - timedelta(days=1))
|
||||
|
||||
assert aws_provider.scans.latest_first().first().id == newest.id
|
||||
|
||||
@@ -3951,6 +3951,43 @@ class TestScanViewSet:
|
||||
mock_enqueue_scan_execution.assert_called_once()
|
||||
# assert scan.scanner_args == expected_scanner_args
|
||||
|
||||
@patch("api.v1.views.enqueue_scan_execution_on_commit")
|
||||
def test_scans_create_returns_the_scan_id_in_task_args(
|
||||
self,
|
||||
mock_enqueue_scan_execution,
|
||||
authenticated_client,
|
||||
okta_provider,
|
||||
):
|
||||
"""The 202 is a task, so `task_args` is the only place the scan id is.
|
||||
|
||||
It is serialized before the on_commit publish that would otherwise fill
|
||||
the kwargs, so the record has to carry them from the start.
|
||||
"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "scans",
|
||||
"attributes": {"name": "New Scan"},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(okta_provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("scan-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
scan = Scan.objects.get()
|
||||
assert response.json()["data"]["attributes"]["task_args"] == {
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(okta_provider.id),
|
||||
}
|
||||
|
||||
@patch("tasks.tasks.perform_scan_task.apply_async")
|
||||
def test_scans_create_queues_scan_when_provider_has_active_scan(
|
||||
self,
|
||||
|
||||
@@ -2822,6 +2822,15 @@ class ScanViewSet(ProviderVisibilityMixin, BaseRLSViewSet):
|
||||
tenant_id=self.request.tenant_id,
|
||||
task_id=pre_task_id,
|
||||
task_status=(QUEUED_SCAN_TASK_STATE if active_scan else None),
|
||||
# This response is serialized before the on_commit publish,
|
||||
# so without these the caller gets a task id and no scan id.
|
||||
# Kept in step with what `enqueue_scan_execution_on_commit`
|
||||
# publishes below.
|
||||
task_kwargs={
|
||||
"tenant_id": str(self.request.tenant_id),
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
},
|
||||
)
|
||||
|
||||
if not active_scan:
|
||||
@@ -3478,12 +3487,8 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
latest_scans = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
Scan.all_objects.filter(tenant_id=tenant_id)
|
||||
.latest_per_provider()
|
||||
.values("provider_id")
|
||||
)
|
||||
|
||||
@@ -3615,11 +3620,9 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
query_params = request.query_params
|
||||
|
||||
latest_scans_queryset = (
|
||||
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
)
|
||||
latest_scans_queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_per_provider()
|
||||
|
||||
queryset = ResourceScanSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
@@ -4206,12 +4209,9 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
latest_scan_ids = list(
|
||||
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scan_ids = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_ids_per_provider()
|
||||
filtered_queryset = filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
@@ -4234,11 +4234,9 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
query_params = request.query_params
|
||||
|
||||
latest_scans_queryset = (
|
||||
Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
)
|
||||
latest_scans_queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_per_provider()
|
||||
raw_latest_scans_ids = list(
|
||||
latest_scans_queryset.values_list("id", "unique_resource_count")
|
||||
)
|
||||
@@ -4979,11 +4977,7 @@ class ComplianceOverviewViewSet(
|
||||
if provider_filters:
|
||||
scans = scans.filter(**provider_filters)
|
||||
|
||||
return list(
|
||||
scans.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
return scans.latest_ids_per_provider()
|
||||
|
||||
def _filtered_queryset_for_latest_provider_scans(self, latest_scan_ids=None):
|
||||
if latest_scan_ids is None:
|
||||
@@ -5725,14 +5719,9 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scan_ids = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, **provider_filter
|
||||
).latest_ids_per_provider()
|
||||
|
||||
return filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
@@ -5759,16 +5748,10 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
|
||||
|
||||
def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None):
|
||||
provider_filter = self._get_provider_filter()
|
||||
queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
queryset = Scan.all_objects.filter(tenant_id=tenant_id, **provider_filter)
|
||||
if provider_filters:
|
||||
queryset = queryset.filter(**provider_filters)
|
||||
return (
|
||||
queryset.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
return queryset.latest_ids_per_provider()
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="providers")
|
||||
def providers(self, request):
|
||||
@@ -5780,14 +5763,9 @@ class OverviewViewSet(ProviderFilterParamsMixin, BaseRLSViewSet):
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scan_ids = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, **provider_filter
|
||||
).latest_ids_per_provider()
|
||||
|
||||
findings_aggregated = (
|
||||
queryset.filter(scan_id__in=latest_scan_ids)
|
||||
@@ -7934,18 +7912,13 @@ class FindingGroupViewSet(JsonApiFilterMixin, BaseRLSViewSet):
|
||||
|
||||
def _get_latest_findings_per_provider(self, filtered_queryset):
|
||||
"""Keep only findings from each provider's most recent completed scan."""
|
||||
# Materialize to a literal IN list. Left as a subquery, Postgres can't
|
||||
# estimate the match count and picks a serial nested loop on
|
||||
# resource_finding_mappings when one scan dominates findings
|
||||
latest_scan_ids = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=self.request.tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
# `latest_ids_per_provider` materializes to a literal IN list.
|
||||
# Left as a subquery, Postgres can't estimate the match count and picks
|
||||
# a serial nested loop on resource_finding_mappings when one scan
|
||||
# dominates findings
|
||||
latest_scan_ids = Scan.objects.filter(
|
||||
tenant_id=self.request.tenant_id
|
||||
).latest_ids_per_provider()
|
||||
return filtered_queryset.filter(scan_id__in=latest_scan_ids)
|
||||
|
||||
def _post_process_aggregation(self, aggregated_data):
|
||||
@@ -8891,16 +8864,14 @@ class FindingGroupViewSet(JsonApiFilterMixin, BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
queryset = self._get_finding_queryset()
|
||||
|
||||
# Order by -completed_at (matching the /latest summary path and the
|
||||
# daily summary upsert keyed on midnight(completed_at)) so that
|
||||
# overlapping scans do not make /resources and /latest read from
|
||||
# different scans and report diverging counts.
|
||||
latest_scan_ids = (
|
||||
Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
# The shared selector orders by -completed_at (matching the /latest
|
||||
# summary path and the daily summary upsert keyed on
|
||||
# midnight(completed_at)) so that overlapping scans do not make
|
||||
# /resources and /latest read from different scans and report
|
||||
# diverging counts.
|
||||
latest_scan_ids = Scan.objects.filter(
|
||||
tenant_id=tenant_id
|
||||
).latest_ids_per_provider()
|
||||
|
||||
normalized_params = self._normalize_jsonapi_params(request.query_params)
|
||||
# Remove date filters since we're using latest
|
||||
|
||||
@@ -231,6 +231,62 @@ LOGGING = {
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
# Celery loggers must be declared explicitly because
|
||||
# disable_existing_loggers=True silences any logger that exists at
|
||||
# dictConfig time but is not named here. Without these, fatal worker
|
||||
# errors (e.g. celery.worker CRITICAL) produce no output.
|
||||
# "celery" must keep propagating: get_task_logger() parents task
|
||||
# loggers under celery.task, so blocking here hides them from root.
|
||||
"celery": {
|
||||
"level": LEVEL,
|
||||
"propagate": True,
|
||||
},
|
||||
"celery.worker": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"celery.worker.consumer": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"celery.worker.consumer.consumer": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"kombu": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"kombu.transport.redis": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"billiard": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
"amqp": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
# WARNING keeps task failures but skips one "succeeded" line per task.
|
||||
"celery.app.trace": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": "WARNING",
|
||||
"propagate": False,
|
||||
},
|
||||
"celery.beat": {
|
||||
"handlers": ["tasks_console"],
|
||||
"level": LEVEL,
|
||||
"propagate": False,
|
||||
},
|
||||
},
|
||||
# Gunicorn required configuration
|
||||
"root": {
|
||||
|
||||
@@ -7,6 +7,7 @@ from config.settings.eventstream import * # noqa
|
||||
from config.settings.partitions import * # noqa
|
||||
from config.settings.sentry import * # noqa
|
||||
from config.settings.social_login import * # noqa
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
SECRET_KEY = env("SECRET_KEY", default="secret")
|
||||
DEBUG = env.bool("DJANGO_DEBUG", default=False)
|
||||
@@ -295,6 +296,9 @@ DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY = env.str(
|
||||
)
|
||||
DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN = env.str("DJANGO_OUTPUT_S3_AWS_SESSION_TOKEN", "")
|
||||
DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION = env.str("DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION", "")
|
||||
# Storage endpoint the API and Celery workers use to talk to S3-compatible object storage
|
||||
# such as MinIO. Empty means the real AWS S3 endpoint, which is unaffected.
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL = env.str("DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL", "")
|
||||
# Browser-reachable storage host used to sign download URLs. Empty means sign against the
|
||||
# same endpoint the API talks to, which is what Prowler Cloud on S3 does.
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL = env.str(
|
||||
@@ -325,6 +329,17 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 960
|
||||
) # 16h
|
||||
|
||||
# Minimum age (of the scan row, or of the scan id itself when the row is gone) before
|
||||
# the periodic reaper will drop an orphaned temp Neo4j database. Keeps a scan that is
|
||||
# still legitimately in flight from ever losing its staging database mid-run.
|
||||
ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS = env.int(
|
||||
"ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS", 6
|
||||
)
|
||||
if ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS <= 0:
|
||||
raise ImproperlyConfigured(
|
||||
"ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS must be a positive number of hours"
|
||||
)
|
||||
|
||||
# Selects where the persistent attack-paths graph is stored. The scan
|
||||
# temporary database is always Neo4j; only the sink is configurable.
|
||||
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Periodic reaper for orphaned temp Neo4j scan databases.
|
||||
|
||||
`scan.py` creates a throw-away `db-tmp-scan-<attack_paths_scan_id>` database per
|
||||
scan and drops it once the scan finishes, success or failure. When the worker
|
||||
or Neo4j itself dies mid-scan, that drop never runs and nothing else ever
|
||||
revisits the database - it sits there forever. This sweep lists every temp
|
||||
database on the ingest cluster and drops the ones whose scan is gone or has
|
||||
been finished for longer than the configured safety margin.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
from api.models import AttackPathsScan, StateChoices
|
||||
from api.uuid_utils import datetime_from_uuid7
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS
|
||||
from uuid6 import UUID as UUID7
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
TERMINAL_STATES = (
|
||||
StateChoices.COMPLETED,
|
||||
StateChoices.FAILED,
|
||||
StateChoices.CANCELLED,
|
||||
)
|
||||
|
||||
|
||||
def reap_orphaned_tmp_databases() -> dict:
|
||||
"""Drop temp Neo4j scan databases whose scan is gone or long finished.
|
||||
|
||||
A failure listing databases aborts the whole sweep (nothing to iterate).
|
||||
A failure reaping one database is logged and skipped so the rest of the
|
||||
sweep still runs.
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
safety_margin = timedelta(hours=ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS)
|
||||
|
||||
try:
|
||||
databases = graph_database.list_databases()
|
||||
except Exception:
|
||||
logger.exception("Failed to list ingest Neo4j databases for temp-db reap")
|
||||
return {"dropped_count": 0, "databases": []}
|
||||
|
||||
tmp_databases = [
|
||||
name for name in databases if name.startswith(graph_database.TEMP_DB_PREFIX)
|
||||
]
|
||||
|
||||
dropped: list[str] = []
|
||||
for database in tmp_databases:
|
||||
try:
|
||||
if _is_orphaned(database, now, safety_margin):
|
||||
graph_database.drop_database(database)
|
||||
dropped.append(database)
|
||||
logger.info(f"Dropped orphaned temp Neo4j database `{database}`")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to reap temp Neo4j database `{database}`")
|
||||
|
||||
logger.info(f"Temp Neo4j database reap: {len(dropped)} dropped")
|
||||
return {"dropped_count": len(dropped), "databases": dropped}
|
||||
|
||||
|
||||
def _is_orphaned(database: str, now: datetime, safety_margin: timedelta) -> bool:
|
||||
"""Decide whether a temp database is safe to drop.
|
||||
|
||||
No scan row: the row was hard-deleted (tenant/provider cleanup) or was
|
||||
never created. Falls back to the scan id's own UUIDv7 timestamp so a
|
||||
database created moments ago is never touched even without a row to check.
|
||||
|
||||
Scan row present: only reapable once it reached a terminal state and has
|
||||
been finished for longer than the safety margin, so a scan still
|
||||
legitimately executing is never touched.
|
||||
"""
|
||||
scan_id = database[len(graph_database.TEMP_DB_PREFIX) :]
|
||||
|
||||
try:
|
||||
scan_uuid = UUID7(scan_id)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Temp database `{database}` has an unparseable scan id, skipping"
|
||||
)
|
||||
return False
|
||||
|
||||
# Global sweep with no tenant context: admin_db bypasses RLS on purpose, the same
|
||||
# way cleanup_stale_attack_paths_scans finds stale scans across every tenant.
|
||||
scan = (
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
.filter(id=scan_uuid)
|
||||
.first()
|
||||
)
|
||||
|
||||
if scan is None:
|
||||
if scan_uuid.version != 7:
|
||||
logger.warning(
|
||||
f"Temp database `{database}` has no scan row and a non-UUIDv7 id, "
|
||||
"skipping"
|
||||
)
|
||||
return False
|
||||
return now - datetime_from_uuid7(scan_uuid) >= safety_margin
|
||||
|
||||
if scan.state not in TERMINAL_STATES:
|
||||
return False
|
||||
|
||||
# `mark_scan_finished` does not touch `updated_at`, so prefer `completed_at`
|
||||
finished_at = scan.completed_at or scan.updated_at
|
||||
return now - finished_at >= safety_margin
|
||||
@@ -499,10 +499,13 @@ def backfill_provider_compliance_scores(tenant_id: str) -> dict:
|
||||
provider_id__in=existing_providers
|
||||
)
|
||||
|
||||
# `completed_scans` keeps its own `completed_at__isnull=False`: this
|
||||
# task writes a *dated* ProviderComplianceScore row, so unlike the read
|
||||
# paths it genuinely cannot use a scan without a `completed_at`.
|
||||
scan_info = list(
|
||||
completed_scans.order_by("provider_id", "-completed_at")
|
||||
.distinct("provider_id")
|
||||
.values("id", "provider_id", "completed_at")
|
||||
completed_scans.latest_per_provider().values(
|
||||
"id", "provider_id", "completed_at"
|
||||
)
|
||||
)
|
||||
|
||||
if not scan_info:
|
||||
|
||||
@@ -207,15 +207,21 @@ def get_s3_client():
|
||||
This function attempts to initialize an S3 client by reading the AWS access key, secret key,
|
||||
session token, and region from environment variables. It then validates the client by listing
|
||||
available S3 buckets. If an error occurs during this process (for example, due to missing or
|
||||
invalid credentials), it falls back to creating an S3 client without explicitly provided credentials,
|
||||
which may rely on other configuration sources (e.g., IAM roles).
|
||||
invalid credentials), it falls back to creating an S3 client without explicitly provided
|
||||
credentials, which may rely on other configuration sources (e.g., IAM roles).
|
||||
|
||||
That fallback is only safe when no explicit endpoint is configured: with an endpoint set, the
|
||||
explicit client already targets the intended S3-compatible storage, and the fallback client
|
||||
would go to the AWS default provider chain instead, an unrelated real-AWS account reachable
|
||||
from the host. So when an endpoint is configured, the original error propagates instead.
|
||||
|
||||
Returns:
|
||||
boto3.client: A configured S3 client instance.
|
||||
|
||||
Raises:
|
||||
ClientError, NoCredentialsError, or ParamValidationError if both attempts to create a client fail.
|
||||
ClientError, NoCredentialsError, or ParamValidationError if the client cannot be created.
|
||||
"""
|
||||
endpoint = settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL
|
||||
s3_client = None
|
||||
try:
|
||||
s3_client = boto3.client(
|
||||
@@ -226,9 +232,12 @@ def get_s3_client():
|
||||
# Storage that has no meaningful region, MinIO among it, is usually configured
|
||||
# without one, and botocore rejects an empty region before any request is made.
|
||||
region_name=settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION or "us-east-1",
|
||||
endpoint_url=endpoint or None,
|
||||
)
|
||||
s3_client.list_buckets()
|
||||
except (ClientError, NoCredentialsError, ParamValidationError, ValueError):
|
||||
if endpoint:
|
||||
raise
|
||||
s3_client = boto3.client("s3")
|
||||
s3_client.list_buckets()
|
||||
|
||||
@@ -236,13 +245,21 @@ def get_s3_client():
|
||||
|
||||
|
||||
def get_s3_presign_client():
|
||||
"""Return a client that signs URLs against the public storage host.
|
||||
"""Return a client that signs download URLs with SigV4.
|
||||
|
||||
None means no public host is configured and the caller should presign with its own
|
||||
client, which leaves deployments on real S3 with the URL they get today.
|
||||
It is used when a public or internal storage host is configured, or when the bucket's
|
||||
region is: boto3 otherwise presigns S3 URLs with SigV2, which S3 rejects for SSE-KMS
|
||||
objects. None means none of those is set and the caller should presign with its own
|
||||
client, which leaves those deployments with the URL they get today.
|
||||
|
||||
The public endpoint wins when both are set: the internal endpoint may only be reachable
|
||||
from inside the cluster, and a URL signed against it would not open in a browser.
|
||||
"""
|
||||
public_endpoint = settings.DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL
|
||||
if not public_endpoint:
|
||||
endpoint = (
|
||||
settings.DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL
|
||||
or settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL
|
||||
)
|
||||
if not endpoint and not settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION:
|
||||
return None
|
||||
|
||||
# Blank keys are signed as-is (empty credential scope) instead of deferring to the
|
||||
@@ -266,9 +283,10 @@ def get_s3_presign_client():
|
||||
# SigV4 puts the region in the credential scope, and MinIO answers to us-east-1
|
||||
# unless it was told otherwise, so an empty region would sign an unusable URL.
|
||||
region_name=settings.DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION or "us-east-1",
|
||||
endpoint_url=public_endpoint,
|
||||
endpoint_url=endpoint or None,
|
||||
# The signature covers the host, so the addressing style has to be pinned rather
|
||||
# than guessed from the endpoint: MinIO serves path-style.
|
||||
# than guessed from the endpoint: MinIO serves path-style, and on AWS it keeps the
|
||||
# regional host instead of the global one, which redirects for new buckets.
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from collections.abc import Iterable
|
||||
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Finding, MuteRule, Scan, StateChoices
|
||||
from api.models import Finding, MuteRule, Scan
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
@@ -39,17 +39,9 @@ def mute_findings_in_latest_scans(
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
mute_rule = MuteRule.objects.get(id=mute_rule_id, tenant_id=tenant_id)
|
||||
latest_scans = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id__in=provider_ids,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at", "-id")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
latest_scans = Scan.objects.filter(
|
||||
tenant_id=tenant_id, provider_id__in=provider_ids
|
||||
).latest_ids_per_provider()
|
||||
|
||||
changed_scan_ids = []
|
||||
findings_muted = 0
|
||||
|
||||
@@ -84,6 +84,7 @@ _SKIP_RECOVERY = {
|
||||
"scan-perform-scheduled",
|
||||
"attack-paths-scan-perform",
|
||||
"attack-paths-cleanup-stale-scans",
|
||||
"attack-paths-reap-orphaned-tmp-databases",
|
||||
"reconcile-orphan-tasks",
|
||||
}
|
||||
|
||||
|
||||
@@ -2700,20 +2700,37 @@ def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dic
|
||||
# refreshed). Wiping based on the older scan would zero counts the newer
|
||||
# scan just set. Skip and let the newer scan's reset task do the work; if
|
||||
# this task was delayed in the queue, that's the correct outcome.
|
||||
# `completed_at__isnull=False` is required: Postgres orders NULL first in
|
||||
# DESC, so a sibling COMPLETED scan with a missing completed_at would sort
|
||||
# as "newest" and incorrectly cause us to skip.
|
||||
#
|
||||
# The comparison must be against the newest *full-scope* scan, which is
|
||||
# what this variable has always been named after but did not use to be:
|
||||
# the query filtered nothing about scope, so any newer scan that is not
|
||||
# full-scope (an imported one, for instance) made the full-scope scan
|
||||
# skip its own cleanup and leave ephemeral resources with a stale
|
||||
# failed_findings_count permanently.
|
||||
#
|
||||
# `is_full_scope()` reads `trigger` plus the scoping keys inside
|
||||
# `scanner_args`, which is not expressible as a WHERE clause, so the
|
||||
# candidates are walked newest-first in Python until the first full-scope
|
||||
# one. The walk needs no cap: `scan` is itself a full-scope candidate, so
|
||||
# it stops at `scan` at the latest, after reading only the scans newer than
|
||||
# it. A fixed window would return None once more newer scoped scans had
|
||||
# landed than it inspected, and skip the cleanup exactly like the bug above.
|
||||
#
|
||||
# NULL `completed_at` no longer needs an explicit filter here: the shared
|
||||
# ordering in `ScanQuerySet.LATEST_ORDER_BY` sorts NULLs last
|
||||
# rather than excluding them, which also fixes the case where a provider
|
||||
# whose completed scans all have a NULL `completed_at` resolved to None and
|
||||
# therefore never ran the reset at all.
|
||||
with rls_transaction(tenant_id):
|
||||
latest_full_scope_scan_id = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=scan.provider_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("-completed_at", "-inserted_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
candidates = (
|
||||
Scan.objects.filter(tenant_id=tenant_id, provider_id=scan.provider_id)
|
||||
.latest_first()
|
||||
.only("id", "trigger", "scanner_args")
|
||||
.iterator(chunk_size=100)
|
||||
)
|
||||
latest_full_scope_scan_id = next(
|
||||
(candidate.id for candidate in candidates if candidate.is_full_scope()),
|
||||
None,
|
||||
)
|
||||
if latest_full_scope_scan_id != scan.id:
|
||||
logger.info(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -42,6 +43,7 @@ from tasks.jobs.attack_paths import (
|
||||
)
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
from tasks.jobs.backfill import (
|
||||
aggregate_scan_category_summaries,
|
||||
aggregate_scan_resource_group_summaries,
|
||||
@@ -163,13 +165,28 @@ def create_scan_task_record(
|
||||
task_id: str,
|
||||
task_name: str = "scan-perform",
|
||||
task_status: str | None = states.PENDING,
|
||||
task_kwargs: dict | None = None,
|
||||
) -> Task:
|
||||
"""Pre-create the TaskResult + Task rows for a pre-generated task id.
|
||||
|
||||
Pass ``task_kwargs`` when the response built from this record is serialized
|
||||
before the broker publish. ``task_kwargs`` is otherwise only written by the
|
||||
``before_task_publish`` signal (``api/signals.py``), and the scan publish is
|
||||
deferred to ``on_commit``, so the 202 would carry an empty ``task_args`` and
|
||||
the caller would have no way to learn the scan id it was just handed a task
|
||||
for. The publish later overwrites the field with the same kwargs as a Python
|
||||
repr; both forms decode to the same dict (``decode_celery_field``).
|
||||
"""
|
||||
if task_status is None:
|
||||
task_status = states.PENDING
|
||||
|
||||
defaults = {"status": task_status, "task_name": task_name}
|
||||
if task_kwargs is not None:
|
||||
defaults["task_kwargs"] = json.dumps(task_kwargs)
|
||||
|
||||
task_result, _ = TaskResult.objects.update_or_create(
|
||||
task_id=str(task_id),
|
||||
defaults={"status": task_status, "task_name": task_name},
|
||||
defaults=defaults,
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=str(task_id),
|
||||
@@ -711,6 +728,13 @@ def cleanup_stale_attack_paths_scans_task():
|
||||
return cleanup_stale_attack_paths_scans()
|
||||
|
||||
|
||||
@shared_task(
|
||||
name="attack-paths-reap-orphaned-tmp-databases", queue="attack-paths-scans"
|
||||
)
|
||||
def reap_orphaned_attack_paths_tmp_databases_task():
|
||||
return reap_orphaned_tmp_databases()
|
||||
|
||||
|
||||
@shared_task(name="reconcile-orphan-tasks", queue="celery")
|
||||
def reconcile_orphan_tasks_task():
|
||||
"""Periodic watchdog: recover tasks whose worker is gone (deploys, crashes)."""
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.database import TEMP_DB_PREFIX
|
||||
from api.models import AttackPathsScan, StateChoices
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from config.django.base import ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS
|
||||
|
||||
MARGIN = timedelta(hours=ATTACK_PATHS_TMP_DB_REAP_SAFETY_MARGIN_HOURS)
|
||||
|
||||
|
||||
def _tmp_db_name(scan_uuid) -> str:
|
||||
return f"{TEMP_DB_PREFIX}{scan_uuid}"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestReapOrphanedTmpDatabases:
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_ignores_databases_without_the_temp_prefix(self, mock_list, mock_drop):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
mock_list.return_value = ["db-tenant-abc123", "system", "neo4j"]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_temp_db_with_no_scan_row_past_safety_margin(
|
||||
self, mock_list, mock_drop
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
old_scan_id = datetime_to_uuid7(
|
||||
datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
)
|
||||
database = _tmp_db_name(old_scan_id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [database]}
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_temp_db_with_no_scan_row_inside_safety_margin(
|
||||
self, mock_list, mock_drop
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
recent_scan_id = datetime_to_uuid7(datetime.now(tz=UTC) - timedelta(minutes=5))
|
||||
database = _tmp_db_name(recent_scan_id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_temp_db_with_unparseable_scan_id(self, mock_list, mock_drop):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
database = f"{TEMP_DB_PREFIX}not-a-uuid"
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_terminal_scan_past_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old_updated_at = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=old_updated_at)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [database]}
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state",
|
||||
[StateChoices.FAILED, StateChoices.CANCELLED],
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_other_terminal_states_past_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider, state
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old_updated_at = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=state,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=old_updated_at)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result["dropped_count"] == 1
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_terminal_scan_inside_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_margin_counts_from_completed_at_not_updated_at(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(
|
||||
updated_at=old, completed_at=datetime.now(tz=UTC)
|
||||
)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_drops_scan_completed_past_safety_margin(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
old = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.FAILED,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(
|
||||
updated_at=old, completed_at=old
|
||||
)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [database]}
|
||||
mock_drop.assert_called_once_with(database)
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_never_drops_an_executing_scan_regardless_of_age(
|
||||
self, mock_list, mock_drop, tenants_fixture, aws_provider
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
very_old = datetime.now(tz=UTC) - timedelta(days=30)
|
||||
scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=aws_provider,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
AttackPathsScan.objects.filter(id=scan.id).update(updated_at=very_old)
|
||||
|
||||
database = _tmp_db_name(scan.id)
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_one_failed_drop_does_not_stop_the_rest_of_the_sweep(
|
||||
self, mock_list, mock_drop
|
||||
):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
old_time = datetime.now(tz=UTC) - MARGIN - timedelta(hours=1)
|
||||
failing_scan_id = datetime_to_uuid7(old_time)
|
||||
succeeding_scan_id = datetime_to_uuid7(old_time)
|
||||
failing_db = _tmp_db_name(failing_scan_id)
|
||||
succeeding_db = _tmp_db_name(succeeding_scan_id)
|
||||
mock_list.return_value = [failing_db, succeeding_db]
|
||||
mock_drop.side_effect = [Exception("boom"), None]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 1, "databases": [succeeding_db]}
|
||||
assert mock_drop.call_count == 2
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_returns_empty_result_when_listing_databases_fails(self, mock_list):
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
mock_list.side_effect = Exception("neo4j unreachable")
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.drop_database")
|
||||
@patch("tasks.jobs.attack_paths.tmp_db_reaper.graph_database.list_databases")
|
||||
def test_preserves_temp_db_with_random_uuid_and_no_row(self, mock_list, mock_drop):
|
||||
"""A non-UUIDv7 id with no matching row has no reliable timestamp, so it
|
||||
must be left alone rather than guessed at."""
|
||||
from tasks.jobs.attack_paths.tmp_db_reaper import reap_orphaned_tmp_databases
|
||||
|
||||
database = _tmp_db_name(uuid4())
|
||||
mock_list.return_value = [database]
|
||||
|
||||
result = reap_orphaned_tmp_databases()
|
||||
|
||||
assert result == {"dropped_count": 0, "databases": []}
|
||||
mock_drop.assert_not_called()
|
||||
|
||||
|
||||
class TestReapOrphanedTmpDatabasesTask:
|
||||
@patch(
|
||||
"tasks.tasks.reap_orphaned_tmp_databases",
|
||||
return_value={"dropped_count": 2, "databases": ["db-tmp-scan-a"]},
|
||||
)
|
||||
def test_task_invokes_the_reaper(self, mock_reap):
|
||||
from tasks.tasks import reap_orphaned_attack_paths_tmp_databases_task
|
||||
|
||||
result = reap_orphaned_attack_paths_tmp_databases_task.run()
|
||||
|
||||
assert result == {"dropped_count": 2, "databases": ["db-tmp-scan-a"]}
|
||||
mock_reap.assert_called_once_with()
|
||||
@@ -3,9 +3,10 @@ import uuid
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import boto3
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
from django.test import override_settings
|
||||
@@ -63,15 +64,46 @@ class TestOutputs:
|
||||
|
||||
assert mock_boto_client.call_args.kwargs["region_name"] == "us-east-1"
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000")
|
||||
def test_get_s3_client_passes_the_endpoint_when_set(self, mock_boto_client):
|
||||
get_s3_client()
|
||||
|
||||
assert mock_boto_client.call_args.kwargs["endpoint_url"] == "http://minio:9000"
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="")
|
||||
def test_get_s3_client_endpoint_empty_by_default(self, mock_boto_client):
|
||||
"""Empty keeps today's behavior: no endpoint override, real S3 is used."""
|
||||
get_s3_client()
|
||||
|
||||
assert mock_boto_client.call_args.kwargs["endpoint_url"] is None
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@override_settings(DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000")
|
||||
def test_get_s3_client_does_not_fall_back_when_endpoint_set(self, mock_boto_client):
|
||||
"""A configured endpoint means the explicit client failed talking to it. The fallback
|
||||
goes to the default provider chain (e.g. an EC2 instance role) against real AWS, so it
|
||||
must not be used: the original error propagates instead."""
|
||||
error = ClientError({"Error": {"Code": "403"}}, "ListBuckets")
|
||||
mock_boto_client.side_effect = error
|
||||
|
||||
with pytest.raises(ClientError):
|
||||
get_s3_client()
|
||||
|
||||
mock_boto_client.assert_called_once()
|
||||
|
||||
@patch("tasks.jobs.export.boto3.client")
|
||||
@patch("tasks.jobs.export.settings")
|
||||
def test_get_s3_client_fallback(self, mock_settings, mock_boto_client):
|
||||
mock_settings.DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL = ""
|
||||
mock_boto_client.side_effect = [
|
||||
ClientError({"Error": {"Code": "403"}}, "ListBuckets"),
|
||||
MagicMock(),
|
||||
]
|
||||
client = get_s3_client()
|
||||
assert client is not None
|
||||
assert mock_boto_client.call_args_list[1] == call("s3")
|
||||
|
||||
@patch("tasks.jobs.export.get_s3_client")
|
||||
@patch("tasks.jobs.export.base")
|
||||
@@ -278,10 +310,70 @@ def _presign(client):
|
||||
|
||||
|
||||
class TestS3PresignClient:
|
||||
@override_settings(**PRESIGN_SETTINGS, DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="")
|
||||
def test_no_public_endpoint_returns_none(self):
|
||||
@override_settings(
|
||||
**{**PRESIGN_SETTINGS, "DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION": ""},
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
|
||||
)
|
||||
def test_no_public_endpoint_and_no_region_returns_none(self):
|
||||
# Without a region, SigV4 would have to guess one and break other regions.
|
||||
assert get_s3_presign_client() is None
|
||||
|
||||
@override_settings(**PRESIGN_SETTINGS, DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="")
|
||||
def test_region_without_public_endpoint_signs_sigv4_on_the_regional_host(self):
|
||||
# SSE-KMS objects reject the SigV2 URLs boto3 presigns by default, and the
|
||||
# global host redirects for new buckets, which breaks a SigV4 signature.
|
||||
url = urlparse(_presign(get_s3_presign_client()))
|
||||
query = parse_qs(url.query)
|
||||
|
||||
assert url.netloc == "s3.eu-west-1.amazonaws.com"
|
||||
assert url.path == "/output-bucket/tenant/scan/report.zip"
|
||||
assert query["X-Amz-Algorithm"] == ["AWS4-HMAC-SHA256"]
|
||||
assert "/eu-west-1/s3/aws4_request" in query["X-Amz-Credential"][0]
|
||||
|
||||
@override_settings(
|
||||
**{
|
||||
**PRESIGN_SETTINGS,
|
||||
"DJANGO_OUTPUT_S3_AWS_ACCESS_KEY_ID": "",
|
||||
"DJANGO_OUTPUT_S3_AWS_SECRET_ACCESS_KEY": "",
|
||||
},
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
|
||||
)
|
||||
def test_region_without_static_keys_signs_with_the_default_chain(self, monkeypatch):
|
||||
# An ECS task role reaches boto3 through the default chain, like the env here.
|
||||
# A fresh default session keeps these keys from being cached for later tests.
|
||||
monkeypatch.setattr(boto3, "DEFAULT_SESSION", None)
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "role-access-key")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "role-secret-key")
|
||||
monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
|
||||
|
||||
query = parse_qs(urlparse(_presign(get_s3_presign_client())).query)
|
||||
|
||||
assert query["X-Amz-Credential"][0].startswith("role-access-key/")
|
||||
assert "/eu-west-1/s3/aws4_request" in query["X-Amz-Credential"][0]
|
||||
|
||||
@override_settings(
|
||||
**{**PRESIGN_SETTINGS, "DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION": ""},
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="",
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000",
|
||||
)
|
||||
def test_internal_endpoint_without_public_endpoint_signs_against_it(self):
|
||||
# No browser-reachable host was configured, so the internal one is the best
|
||||
# available target instead of falling through to the real AWS host.
|
||||
url = urlparse(_presign(get_s3_presign_client()))
|
||||
|
||||
assert url.netloc == "minio:9000"
|
||||
assert url.path == "/output-bucket/tenant/scan/report.zip"
|
||||
|
||||
@override_settings(
|
||||
**PRESIGN_SETTINGS,
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="https://storage.example.com",
|
||||
DJANGO_OUTPUT_S3_AWS_ENDPOINT_URL="http://minio:9000",
|
||||
)
|
||||
def test_public_endpoint_wins_over_the_internal_endpoint(self):
|
||||
url = urlparse(_presign(get_s3_presign_client()))
|
||||
|
||||
assert url.netloc == "storage.example.com"
|
||||
|
||||
@override_settings(
|
||||
**PRESIGN_SETTINGS,
|
||||
DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL="https://storage.example.com",
|
||||
|
||||
@@ -5967,6 +5967,112 @@ class TestResetEphemeralResourceFindingsCount:
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 5
|
||||
|
||||
def test_runs_when_newer_scan_is_not_full_scope(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
"""A newer scoped scan must not block the full-scope scan's cleanup.
|
||||
|
||||
The race guard used to pick the newest COMPLETED scan of any scope
|
||||
despite being named after full-scope ones, so a single scoped scan
|
||||
landing after a complete one made the complete scan skip its cleanup
|
||||
and leave ephemeral resources with a stale count permanently.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
newer_completed_at = scan1.completed_at + timedelta(minutes=5)
|
||||
Scan.objects.create(
|
||||
name="Newer scoped scan",
|
||||
provider=aws_provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
started_at=newer_completed_at,
|
||||
completed_at=newer_completed_at,
|
||||
scanner_args={"checks": ["check1"]},
|
||||
)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_runs_when_many_newer_scans_are_not_full_scope(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
"""The walk must not give up before it reaches the full-scope scan.
|
||||
|
||||
A fixed look-back window returned None once more newer scoped scans
|
||||
had landed than it inspected, and skipped the cleanup exactly like the
|
||||
original bug did with one.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
for minutes in range(1, 41):
|
||||
newer_completed_at = scan1.completed_at + timedelta(minutes=minutes)
|
||||
Scan.objects.create(
|
||||
name=f"Newer scoped scan {minutes}",
|
||||
provider=aws_provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
started_at=newer_completed_at,
|
||||
completed_at=newer_completed_at,
|
||||
scanner_args={"checks": ["check1"]},
|
||||
)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_runs_when_completed_at_is_null(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
"""NULL `completed_at` used to make the reset never run at all.
|
||||
|
||||
The old guard filtered `completed_at__isnull=False`, so a provider
|
||||
whose completed scans all had a NULL `completed_at` resolved the
|
||||
"latest" scan to None, which never equals `scan.id`.
|
||||
"""
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Scan.all_objects.filter(id=scan1.id).update(completed_at=None)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_does_not_touch_other_providers_resources(
|
||||
self, tenants_fixture, scans_fixture, aws_provider, resources_fixture
|
||||
):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
@@ -14,6 +15,7 @@ from api.models import (
|
||||
StateChoices,
|
||||
Task,
|
||||
)
|
||||
from api.v1.serializers import TaskSerializer
|
||||
from botocore.exceptions import ClientError
|
||||
from celery import states
|
||||
from django_celery_beat.models import IntervalSchedule, PeriodicTask
|
||||
@@ -32,6 +34,7 @@ from tasks.tasks import (
|
||||
_scan_tmp_output_directory,
|
||||
check_integrations_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
create_scan_task_record,
|
||||
generate_outputs_task,
|
||||
mute_findings_in_latest_scans_task,
|
||||
perform_attack_paths_scan_task,
|
||||
@@ -3384,3 +3387,79 @@ class TestTaskTimeLimits:
|
||||
"lighthouse-provider-connection-check",
|
||||
):
|
||||
assert celery_app.tasks[name].time_limit < default
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCreateScanTaskRecord:
|
||||
"""`task_kwargs` is what a response built before the publish can report."""
|
||||
|
||||
def _scan(self, tenant, provider):
|
||||
"""A manual scan, like the one `POST /api/v1/scans` creates."""
|
||||
return Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
|
||||
def _publish_kwargs(self, tenant, scan):
|
||||
"""What `enqueue_scan_execution_on_commit` publishes for this scan."""
|
||||
return {
|
||||
"tenant_id": str(tenant.id),
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
}
|
||||
|
||||
def _task_args(self, task):
|
||||
"""Read the record back the way `TaskSerializer` does."""
|
||||
return TaskSerializer(task).data["task_args"]
|
||||
|
||||
def test_the_stored_kwargs_are_the_ones_the_publish_would_send(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""The 202 reports what is stored here, so it has to be the dispatch kwargs."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = self._scan(tenant, aws_provider)
|
||||
|
||||
task = create_scan_task_record(
|
||||
tenant_id=str(tenant.id),
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_kwargs=self._publish_kwargs(tenant, scan),
|
||||
)
|
||||
|
||||
assert self._task_args(task) == {
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(aws_provider.id),
|
||||
}
|
||||
|
||||
def test_a_record_created_without_kwargs_reports_none(self, tenants_fixture):
|
||||
"""The argument is optional, so the other callers keep their behaviour."""
|
||||
task = create_scan_task_record(
|
||||
tenant_id=str(tenants_fixture[0].id),
|
||||
task_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
assert self._task_args(task) == {}
|
||||
|
||||
def test_the_publish_can_overwrite_the_stored_kwargs(
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""django-celery-results stores a Python repr; both must decode alike."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = self._scan(tenant, aws_provider)
|
||||
task_id = str(uuid.uuid4())
|
||||
kwargs = self._publish_kwargs(tenant, scan)
|
||||
|
||||
task = create_scan_task_record(
|
||||
tenant_id=str(tenant.id), task_id=task_id, task_kwargs=kwargs
|
||||
)
|
||||
before = self._task_args(task)
|
||||
|
||||
# What `before_task_publish` writes once the task reaches the broker.
|
||||
task_result = TaskResult.objects.get(task_id=task_id)
|
||||
task_result.task_kwargs = json.dumps(repr(kwargs))
|
||||
task_result.save(update_fields=["task_kwargs"])
|
||||
task.refresh_from_db()
|
||||
|
||||
assert self._task_args(task) == before
|
||||
|
||||
Generated
+1
-1
@@ -4938,7 +4938,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.44.0"
|
||||
version = "1.45.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
api:
|
||||
hostname: "prowler-api"
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
@@ -44,6 +45,7 @@ services:
|
||||
|
||||
ui:
|
||||
image: prowlercloud/prowler-ui:${PROWLER_UI_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
@@ -61,6 +63,7 @@ services:
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
|
||||
restart: unless-stopped
|
||||
hostname: "postgres-db"
|
||||
volumes:
|
||||
- ./_data/postgres:/var/lib/postgresql/data
|
||||
@@ -81,6 +84,7 @@ services:
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8-alpine@sha256:a038175878d66b9d274fbf8be73c0305e93798b83917647f167e18cef3c71eec
|
||||
restart: unless-stopped
|
||||
hostname: "valkey"
|
||||
volumes:
|
||||
- ./_data/valkey:/data
|
||||
@@ -97,6 +101,7 @@ services:
|
||||
|
||||
neo4j:
|
||||
image: graphstack/dozerdb:5.26.27.0@sha256:9b54d6b3a98a76c00bd23e8e78d8c82081ff168162aebd47b25c234e092cb0a0
|
||||
restart: unless-stopped
|
||||
hostname: "neo4j"
|
||||
volumes:
|
||||
- ./_data/neo4j:/data
|
||||
@@ -129,6 +134,7 @@ services:
|
||||
|
||||
worker:
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
|
||||
stop_grace_period: 120s
|
||||
env_file:
|
||||
@@ -149,6 +155,7 @@ services:
|
||||
|
||||
worker-beat:
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- path: ./.env
|
||||
required: false
|
||||
@@ -165,6 +172,7 @@ services:
|
||||
|
||||
mcp-server:
|
||||
image: prowlercloud/prowler-mcp:${PROWLER_MCP_VERSION:-stable}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PROWLER_MCP_TRANSPORT_MODE=http
|
||||
env_file:
|
||||
|
||||
@@ -4,6 +4,73 @@ description: "New features and improvements in each Prowler release"
|
||||
rss: true
|
||||
---
|
||||
|
||||
<Update label="v5.43.0" description="September 21, 2026">
|
||||
### 🏛️ Compliance — FedRAMP 20x Consolidated Rules 2026
|
||||
|
||||
The FedRAMP 20x Phase One pilot frameworks (`fedramp_20x_ksi_low_aws`, `fedramp_20x_ksi_low_azure` and `fedramp_20x_ksi_low_gcp`) are replaced by two universal frameworks built from the FedRAMP Consolidated Rules for 2026, each covering AWS, Azure, GCP, Kubernetes and Microsoft 365 from a single definition:
|
||||
|
||||
- **FedRAMP 20x KSI** (`fedramp_20x_ksi_2026`): the 46 Key Security Indicators across 10 themes. There is one indicator catalog for every class instead of a separate Low fork; each indicator carries its class applicability and NIST SP 800-53 controls.
|
||||
- **FedRAMP 20x Class C FRR** (`fedramp_20x_frr_class_c_2026`): the 158 FedRAMP Rules of the Class C ruleset that bind cloud service providers. Most are program obligations (reports, notifications, certification package) and stay manual; checks are mapped only where they evidence part of the rule text. Configurable checks carry configuration requirements, so a relaxed `audit_config` cannot turn a requirement green.
|
||||
|
||||
Automation or stored results that reference the pilot framework IDs need to move to `fedramp_20x_ksi_2026`. In Prowler App, both frameworks show the per-provider breakdown in the cross-provider compliance view and can be downloaded as OCSF.
|
||||
|
||||
Read more in the [Compliance documentation](https://docs.prowler.com/user-guide/compliance/tutorials/compliance).
|
||||
|
||||
### 🔎 AWS — Inspector Coverage, CISA KEV and FIPS Checks
|
||||
|
||||
Seven new AWS checks back the vulnerability detection and cryptography rules of FedRAMP 20x Class C:
|
||||
|
||||
- `inspector2_coverage_scan_status_active` and `inspector2_coverage_recently_scanned` report resources Amazon Inspector is not scanning, or last scanned more than `inspector2_max_days_since_last_scan` days ago (default 3).
|
||||
- `inspector2_active_findings_no_known_exploited_vulnerabilities` and `inspector2_active_findings_kev_within_due_date` report active findings whose CVE is in the CISA Known Exploited Vulnerabilities catalog, and those still open past the CISA due date. The KEV data comes from Inspector itself through `inspector2:BatchGetFindingDetails`, so no external feed is needed.
|
||||
- `inspector2_active_findings_within_max_age` reports active findings first observed more than `inspector2_active_finding_max_age_days` days ago (default 192).
|
||||
- `elbv2_listener_fips_tls_enabled` and `transfer_server_fips_security_policy_enabled` report HTTPS/TLS load balancer listeners and Transfer Family servers without a FIPS security policy.
|
||||
|
||||
`inspector2:BatchGetFindingDetails` is not part of `SecurityAudit`, so it is now included in the Prowler additions policy and the CloudFormation scan role. Without it, the KEV checks report `MANUAL` naming the missing permission instead of a false `FAIL`.
|
||||
|
||||
Explore all AWS checks at [Prowler Hub](https://hub.prowler.com/check?provider=aws).
|
||||
|
||||
### ☁️ AWS — Partition Bootstrap Failover
|
||||
|
||||
When `PROWLER_AWS_PARTITION` is set, the bootstrap STS calls (validating credentials, assuming a role and getting an MFA session token) now try up to two more regions of the partition if the first one cannot be reached. A GovCloud host whose configured region belongs to another partition was still sent to `us-gov-east-1`, and on a network that routes only to `us-gov-west-1` the connection check and the scan failed on perfectly valid credentials. Only connection errors and timeouts move on to the next region; credential errors are reported from the first one as before. Later STS calls reuse the region that answered, and nothing changes when `PROWLER_AWS_PARTITION` is unset.
|
||||
|
||||
Read more in the [AWS Regions and Partitions documentation](https://docs.prowler.com/user-guide/providers/aws/regions-and-partitions).
|
||||
|
||||
### 🌐 Azure — Sovereign Cloud Endpoints for Defender and Key Vault
|
||||
|
||||
Defender security contacts and Key Vault key rotation policies now use the endpoints of the cloud selected with `--azure-region` instead of the hardcoded `management.azure.com` and `vault.azure.net` hosts, so both work on `AzureUSGovernment` and `AzureChinaCloud`. Key Vault clients are built from the vault URI that Azure returns for each vault.
|
||||
|
||||
Read more in the [Azure non-default cloud documentation](https://docs.prowler.com/user-guide/providers/azure/use-non-default-cloud).
|
||||
|
||||
### ✉️ Invitations — Expired Invitations No Longer Block Re-Invites
|
||||
|
||||
A pending invitation past its expiry date is now reported as expired, and inviting the same email again marks it as expired and creates the new invitation instead of returning a generic error. The Invitations table disables Edit and Revoke on expired and revoked invitations, and `filter[state__in]` on the invitations endpoint no longer returns a server error.
|
||||
|
||||
In Prowler Cloud, new organizations are offered an **Invite your team** step once the first provider is connected, and Prowler Private Cloud deployments can set `UI_SELF_REGISTRATION_ENABLED=false` to make sign-up invitation-only.
|
||||
|
||||
Read more in the [Invitations documentation](https://docs.prowler.com/user-guide/tutorials/prowler-app-rbac#invitations).
|
||||
|
||||
### 📄 Reports — Downloads on Self-Hosted Storage
|
||||
|
||||
Report downloads no longer depend on the storage host being the same inside and outside the container network. `DJANGO_OUTPUT_S3_AWS_PUBLIC_ENDPOINT_URL` signs the download URL against a browser-reachable host, so a deployment whose object storage answers only on an internal address serves the file instead of a link the browser cannot open. Leaving `DJANGO_OUTPUT_S3_AWS_DEFAULT_REGION` unset, common on S3-compatible storage with no meaningful region, no longer makes the download fail with a server error.
|
||||
|
||||
### 🔍 Checks
|
||||
|
||||
- **Huawei Cloud:** new `smn_topic_subscriptions` check reports SMN topics without any subscription.
|
||||
- **Cloudflare:** the API token links in the provider wizard request the SSL and Certificates, Bot Management and Zone WAF read permissions the checks need, so a token created from the wizard no longer produces failures on permissions it was never granted.
|
||||
- **Microsoft 365:** five Defender malware, anti-phishing and inbound anti-spam checks no longer fail with `KeyError` on tenants that use the Standard or Strict preset security policies, which dropped every finding of those checks. Preset policies are covered by `defender_strict_preset_security_policy_enabled`.
|
||||
- **Google Workspace:** `security_2sv_enforced` reports domain-wide 2-Step Verification failures as `FAIL` even when every failing setting is overridden for a group or organizational unit.
|
||||
|
||||
Explore all checks at [Prowler Hub](https://hub.prowler.com/check).
|
||||
|
||||
### 🔐 Security Updates
|
||||
|
||||
- `libsqlite3-0`, `gzip`, `perl-base`, `libssh2-1t64` and `libpcre2-8-0` upgraded in the SDK and API container images, patching high-severity Debian CVEs.
|
||||
- PowerShell upgraded to 7.5.11 in the SDK and API container images, bundling .NET runtime 9.0.20 and patching CVE-2026-62901.
|
||||
- `anyio` upgraded to 4.14.2 in the SDK, the API and the MCP Server, patching CVE-2026-63374.
|
||||
|
||||
See the [full release notes on GitHub](https://github.com/prowler-cloud/prowler/releases/tag/5.43.0) for the complete list of changes.
|
||||
</Update>
|
||||
|
||||
<Update label="v5.42.0" description="September 11, 2026">
|
||||
### ☁️ AWS — ISO Partitions
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ The generic service pattern is described in [service page](/developer-guide/serv
|
||||
- Directly in the code, in location [`prowler/providers/alibabacloud/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/alibabacloud/services)
|
||||
- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view.
|
||||
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used across all Alibaba Cloud services.
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In the next subsection you can find a list of common patterns that are used across all Alibaba Cloud services.
|
||||
|
||||
### Alibaba Cloud Service Common Patterns
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ By default, Prowler will audit just one account and organization settings per sc
|
||||
|
||||
## AWS Provider Classes Architecture
|
||||
|
||||
The AWS provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the AWS-specific implementation, highlighting how the generic provider concepts are realized for AWS in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). In next subsection you can find a list of the main classes of the AWS provider.
|
||||
The AWS provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the AWS-specific implementation, highlighting how the generic provider concepts are realized for AWS in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). In the next subsection you can find a list of the main classes of the AWS provider.
|
||||
|
||||
### `AwsProvider` (Main Class)
|
||||
|
||||
@@ -59,7 +59,7 @@ The generic service pattern is described in [service page](/developer-guide/serv
|
||||
- Directly in the code, in location [`prowler/providers/aws/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/aws/services)
|
||||
- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view.
|
||||
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used across all AWS services.
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In the next subsection you can find a list of common patterns that are used across all AWS services.
|
||||
|
||||
### AWS Service Common Patterns
|
||||
|
||||
@@ -67,8 +67,8 @@ The best reference to understand how to implement a new service is following the
|
||||
- Every AWS service class inherits from `AWSService`, ensuring access to session, identity, configuration, and threading utilities.
|
||||
- The constructor (`__init__`) always calls `super().__init__` with the service name and provider (e.g. `super().__init__(__class__.__name__, provider))`). Ensure that the service name in boto3 is the same that you use in the constructor. Usually is used the `__class__.__name__` to get the service name because it is the same as the class name.
|
||||
- Resource containers **must** be initialized in the constructor. They should be dictionaries, with the key being the resource ARN or equivalent unique identifier and the value being the resource object.
|
||||
- Resource discovery and attribute collection are parallelized using `self.__threading_call__`, typically by region or resource, for performance. The first parameter of the method is the iterator, if not provided, it will be the region; but if present indicate an array of the resources to be processed.
|
||||
- Resource filtering is consistently enforced using `self.audit_resources` attribute and `is_resource_filtered` function, it is used to see if user has provided some resource that is not in the audit scope, so we can skip it in the service logic. Normally it is used before storing the resource in the service container as follows: `if not self.audit_resources or (is_resource_filtered(resource["arn"], self.audit_resources)):`.
|
||||
- Resource discovery and attribute collection are parallelized using `self.__threading_call__`, typically by region or resource, for performance. The first parameter of the method is the iterator, if not provided, it will be the region; but if present it indicates an array of the resources to be processed.
|
||||
- Resource filtering is consistently enforced using `self.audit_resources` attribute and `is_resource_filtered` function, it is used to see if the user has provided some resource that is not in the audit scope, so we can skip it in the service logic. Normally it is used before storing the resource in the service container as follows: `if not self.audit_resources or (is_resource_filtered(resource["arn"], self.audit_resources)):`.
|
||||
- All AWS resources are represented as Pydantic `BaseModel` classes, providing type safety and structured access to resource attributes.
|
||||
- AWS API calls are wrapped in try/except blocks, with specific handling for `ClientError` and generic exceptions, always logging errors.
|
||||
- If ARN is not present for some resource, it can be constructed using string interpolation, always including partition, service, region, account, and resource ID.
|
||||
@@ -162,7 +162,7 @@ When you instantiate `Check_Report_AWS`, you must provide the check metadata and
|
||||
|
||||
If the resource object does not contain the required attributes, you must set them manually in the check logic.
|
||||
|
||||
Other attributes are inherited from the `Check_Report` class, from those you **always** have to set the `status` and `status_extended` attributes in the check logic.
|
||||
Other attributes are inherited from the `Check_Report` class, from which you **always** have to set the `status` and `status_extended` attributes in the check logic.
|
||||
|
||||
#### Example Usage
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ By default, Prowler will audit all the subscriptions that it is able to list in
|
||||
|
||||
## Azure Provider Classes Architecture
|
||||
|
||||
The Azure provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the Azure-specific implementation, highlighting how the generic provider concepts are realized for Azure in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). In next subsection you can find a list of the main classes of the Azure provider.
|
||||
The Azure provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the Azure-specific implementation, highlighting how the generic provider concepts are realized for Azure in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). In the next subsection you can find a list of the main classes of the Azure provider.
|
||||
|
||||
### `AzureProvider` (Main Class)
|
||||
|
||||
@@ -57,13 +57,13 @@ The generic service pattern is described in [service page](/developer-guide/serv
|
||||
- Directly in the code, in location [`prowler/providers/azure/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/azure/services)
|
||||
- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view.
|
||||
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used across all Azure services.
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In the next subsection you can find a list of common patterns that are used across all Azure services.
|
||||
|
||||
### Azure Service Common Patterns
|
||||
|
||||
- Services communicate with Azure using the Azure Python SDK, mainly using the Azure Management Client (except for the Microsoft Entra ID service, that is using the Microsoft Graph API), you can find the documentation with all the management services [here](https://learn.microsoft.com/en-us/python/api/overview/azure/?view=azure-python).
|
||||
- Every Azure service class inherits from `AzureService`, ensuring access to session, identity, configuration, and client utilities.
|
||||
- The constructor (`__init__`) always calls `super().__init__` with the service Azure Management Client and Prowler provider object (e.g `super().__init__(WebSiteManagementClient, provider)`).
|
||||
- The constructor (`__init__`) always calls `super().__init__` with the service Azure Management Client and Prowler provider object (e.g. `super().__init__(WebSiteManagementClient, provider)`).
|
||||
- Resource containers **must** be initialized in the constructor, and they should be dictionaries, with the key being the subscription ID, the value being a dictionary with the resource ID as key and the resource object as value.
|
||||
- All Azure resources are represented as Pydantic `BaseModel` classes, providing type safety and structured access to resource attributes. Some are represented as dataclasses due to legacy reasons, but new resources should be represented as Pydantic `BaseModel` classes.
|
||||
- Azure SDK functions are wrapped in try/except blocks, with specific handling for errors, always logging errors. It is a best practice to create a custom function for every Azure SDK call, in that way we can handle the errors in a more specific way.
|
||||
|
||||
@@ -103,7 +103,7 @@ class <check_name>(Check):
|
||||
# Set required fields and implement check logic
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"<Description about why the resource is compliant>"
|
||||
# If some of the information needed for the report is not inside the resource, it can be set it manually here.
|
||||
# If some of the information needed for the report is not inside the resource, it can be set manually here.
|
||||
# This depends on the provider and the resource that is being audited.
|
||||
# report.region = resource.region
|
||||
# report.resource_tags = getattr(resource, "tags", [])
|
||||
@@ -437,7 +437,7 @@ For the complete list of available categories, see [Categories Guidelines](/deve
|
||||
|
||||
#### DependsOn
|
||||
|
||||
List of check IDs of checks that if are compliant, this check will be a compliant too or it is not going to give any finding.
|
||||
List of check IDs of checks that if they are compliant, this check will be compliant too or it is not going to give any finding.
|
||||
|
||||
#### RelatedTo
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ The former build-time variables map to the new runtime variables as follows:
|
||||
`UI_CLOUD_ENABLED` is a plain runtime boolean flag that enables Prowler Cloud behavior when set to the exact string `"true"` and defaults to off; unlike the other renamed variables it has no legacy fallback, so `NEXT_PUBLIC_IS_CLOUD_ENV` is no longer read.
|
||||
|
||||
`UI_SELF_REGISTRATION_ENABLED` is a runtime opt-out flag that Prowler Local Server reads only when `UI_CLOUD_ENABLED` is `"true"`. It defaults to on and turns off when set to `"false"`, matched case-insensitively so the same value can be shared with a backend setting written `False`. When it is off, the sign-up page only opens with an invitation token, the sign-in page drops its "Sign up" link, and the profile hides "Create organization"; invited users can still complete their registration. Outside a Prowler Cloud deployment the flag is ignored and account creation stays open.
|
||||
|
||||
## Registry UI Rollout and Rollback
|
||||
|
||||
`UI_REGISTRY_ENABLED` is an optional runtime flag for Prowler Cloud and Private Cloud. Registry is eligible only when both `UI_CLOUD_ENABLED` and `UI_REGISTRY_ENABLED` are the exact string `"true"` and the current user has the backend-authorized `manage_registry` permission. Unset, `"false"`, or malformed values fail closed. The flag defaults to off and is not a replacement for backend authorization. Registry access is independent of billing; Private Cloud can use it with `CLOUD_BILLING_ENABLED=false`.
|
||||
|
||||
@@ -102,7 +102,7 @@ The generic service pattern is described in [service page](/developer-guide/serv
|
||||
- Directly in the code, in location [`prowler/providers/gcp/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/gcp/services)
|
||||
- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view.
|
||||
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used across all GCP services.
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In the next subsection you can find a list of common patterns that are used across all GCP services.
|
||||
|
||||
### GCP Service Common Patterns
|
||||
|
||||
@@ -161,7 +161,7 @@ When you instantiate `Check_Report_GCP`, you must provide the check metadata and
|
||||
- Defaults to "global" if none are available.
|
||||
|
||||
All these attributes can be overridden by passing the corresponding argument to the constructor. If the resource object does not contain the required attributes, you must set them manually.
|
||||
Other attributes are inherited from the `Check_Report` class, from those you **always** have to set the `status` and `status_extended` attributes in the check logic.
|
||||
Other attributes are inherited from the `Check_Report` class, from which you **always** have to set the `status` and `status_extended` attributes in the check logic.
|
||||
|
||||
#### Example Usage
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ A provider is any platform or service that offers resources, data, or functional
|
||||
For providers supported by Prowler, refer to [Prowler Hub](https://hub.prowler.com/).
|
||||
|
||||
<Warning>
|
||||
There are some custom providers added by the community, like [NHN Cloud](https://www.nhncloud.com/), that are not maintained by the Prowler team, but can be used in the Prowler CLI. The main purpose of this documentation is to guide you through creating a new provider and integrating it not only in the CLI, but also in the API and UI. Non official providers can be checked directly at the [Prowler GitHub repository](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers).
|
||||
There are some custom providers added by the community, like [NHN Cloud](https://www.nhncloud.com/), that are not maintained by the Prowler team, but can be used in the Prowler CLI. The main purpose of this documentation is to guide you through creating a new provider and integrating it not only in the CLI, but also in the API and UI. Non-official providers can be checked directly at the [Prowler GitHub repository](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers).
|
||||
</Warning>
|
||||
---
|
||||
|
||||
@@ -105,8 +105,8 @@ Once you have decided the provider you want or need to add to Prowler, the next
|
||||
#### Implementation Complexity
|
||||
|
||||
- **SDK Providers**: Low complexity. You have mature examples like AWS, Azure, GCP, Kubernetes, etc. that you can leverage to implement your provider.
|
||||
- **API Providers**: Medium complexity. You need to implement the authentication and session management, and the API calls to the provider. You now have NHN and MongoDB Atlas as example to follow.
|
||||
- **Tool/Wrapper Providers**: High complexity. You need to implement the argument/output mapping to the provider and handle problems that the tool/wrapper may have. You now have IAC and the PowerShell wrapper as example to follow.
|
||||
- **API Providers**: Medium complexity. You need to implement the authentication and session management, and the API calls to the provider. You now have NHN and MongoDB Atlas as examples to follow.
|
||||
- **Tool/Wrapper Providers**: High complexity. You need to implement the argument/output mapping to the provider and handle problems that the tool/wrapper may have. You now have IAC and the PowerShell wrapper as examples to follow.
|
||||
- **Hybrid Providers**: High complexity. You need to "customize" your provider, mixing the other types of providers to achieve the desired result. You have M365 (msgraph SDK + PowerShell wrapper) and GitHub (PyGithub SDK + graphql API requests) as examples.
|
||||
|
||||
### Determining Regional vs Non-Regional Architecture
|
||||
@@ -1756,7 +1756,7 @@ Argument validation ensures that the API provider receives valid configuration p
|
||||
|
||||
**File:** `prowler/providers/<provider_name>/lib/arguments/arguments.py`
|
||||
|
||||
Arguments depends on the provider and not the type, so the pattern for this step is the same as the [SDK providers](#step-4-implement-arguments).
|
||||
Arguments depend on the provider and not the type, so the pattern for this step is the same as the [SDK providers](#step-4-implement-arguments).
|
||||
|
||||
#### Step 5: Implement Mutelist
|
||||
|
||||
@@ -2397,7 +2397,7 @@ class ToolProvider(Provider):
|
||||
# Build the tool command
|
||||
tool_command = [
|
||||
"your_tool_command",
|
||||
# Add your tool-specific arguments here, this are just examples
|
||||
# Add your tool-specific arguments here, these are just examples
|
||||
"-d",
|
||||
directory,
|
||||
"-o",
|
||||
|
||||
@@ -5,7 +5,7 @@ title: 'Prowler Services'
|
||||
Here you can find how to create a new service, or to complement an existing one, for a [Prowler Provider](/developer-guide/provider).
|
||||
|
||||
<Note>
|
||||
First ensure that the provider you want to add the service is already created. It can be checked [here](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). If the provider is not present, please refer to the [Provider](./provider.md) documentation to create it from scratch.
|
||||
First ensure that the provider you want to add the service to is already created. It can be checked [here](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). If the provider is not present, please refer to the [Provider](/developer-guide/provider) documentation to create it from scratch.
|
||||
</Note>
|
||||
## Introduction
|
||||
|
||||
@@ -102,7 +102,7 @@ class <Service>(ServiceParentClass):
|
||||
|
||||
# A try-except block must be created in each function.
|
||||
try:
|
||||
# If pagination is supported by the provider, is always better to use it, call to the provider API to retrieve the desired data.
|
||||
# If pagination is supported by the provider, it is always better to use it, call to the provider API to retrieve the desired data.
|
||||
describe_<items>_paginator = regional_client.get_paginator("describe_<items>")
|
||||
|
||||
# Paginator to get every item.
|
||||
@@ -133,7 +133,7 @@ class <Service>(ServiceParentClass):
|
||||
|
||||
# When handling exceptions, use the following approach to log errors appropriately based on the cloud provider being used:
|
||||
except Exception as error:
|
||||
# Depending on each provider we can must use different fields in the logger, e.g.: AWS: regional_client.region or self.region, GCP: project_id and location, Azure: subscription
|
||||
# Depending on each provider we must use different fields in the logger, e.g.: AWS: regional_client.region or self.region, GCP: project_id and location, Azure: subscription
|
||||
logger.error(
|
||||
f"{<provider_specific_field>} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
@@ -284,7 +284,7 @@ class Test_iam_password_policy_uppercase:
|
||||
|
||||
assert len(results) == 1
|
||||
assert result[0].status == "FAIL"
|
||||
assert result[0].status_extended == "IAM password policy does not srequire at least one uppercase letter."
|
||||
assert result[0].status_extended == "IAM password policy does not require at least one uppercase letter."
|
||||
assert result[0].resource_arn == f"arn:aws:iam::{AWS_ACCOUNT_NUMBER}:root"
|
||||
assert result[0].resource_id == AWS_ACCOUNT_NUMBER
|
||||
assert result[0].resource_tags == []
|
||||
@@ -882,7 +882,7 @@ from unittest import mock
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
# Import some constans values needed in almost every check
|
||||
# Import some constant values needed in almost every check
|
||||
|
||||
from tests.providers.azure.azure_fixtures import (
|
||||
AZURE_SUBSCRIPTION_ID,
|
||||
@@ -1024,7 +1024,7 @@ from prowler.providers.azure.services.appinsights.appinsights_service import (
|
||||
Component,
|
||||
)
|
||||
|
||||
# Import some constans values needed in almost every check
|
||||
# Import some constant values needed in almost every check
|
||||
|
||||
from tests.providers.azure.azure_fixtures import (
|
||||
AZURE_SUBSCRIPTION_ID,
|
||||
|
||||
@@ -128,8 +128,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.42.0"
|
||||
PROWLER_API_VERSION="5.42.0"
|
||||
PROWLER_UI_VERSION="5.43.0"
|
||||
PROWLER_API_VERSION="5.43.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -6,7 +6,7 @@ This section contains the instructions to subscribe to **Prowler Cloud** through
|
||||
|
||||
## How to Subscribe
|
||||
|
||||
To get to the **Prowler Cloud** product listing in the AWS Marketplace, and click the `View purchase options` button:
|
||||
Go to the **Prowler Cloud** product listing in the AWS Marketplace, and click the `View purchase options` button:
|
||||
|
||||
1. Use this link to be taken directly to the [Prowler Cloud Marketplace Listing](https://aws.amazon.com/marketplace/pp/prodview-6ochhig5kxpok):
|
||||
|
||||
@@ -24,7 +24,7 @@ After you have subscribed to the **Prowler Cloud** product, you will need to set
|
||||
|
||||

|
||||
|
||||
2. You will be redirected to **Prowler Cloud Sign In** page. You can sign in with an existing account or sign up with a new account:
|
||||
2. You will be redirected to the **Prowler Cloud Sign In** page. You can sign in with an existing account or sign up with a new account:
|
||||
|
||||

|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ title: 'Overview'
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
Prowler Cloud runs an enhanced version of Lighthouse AI in Open Source repository, the Agentic Cloud Defender that helps teams understand, prioritize, and remediate security findings across cloud environments.
|
||||
Prowler Cloud runs an enhanced version of Lighthouse AI in the Open Source repository, the Agentic Cloud Defender that helps teams understand, prioritize, and remediate security findings across cloud environments.
|
||||
|
||||
<SubscriptionBanner />
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ Prowler Cloud automates scanning single or multiple accounts and has all of the
|
||||
|
||||
<Card title="Create your account here to see Prowler Cloud in action" href="https://cloud.prowler.com/sign-up" />
|
||||
|
||||
With 100% consistency across our open source policies and APIs. Prowler Cloud provides the following added benefits:
|
||||
With 100% consistency across our open source policies and APIs, Prowler Cloud provides the following added benefits:
|
||||
|
||||
<ul>
|
||||
<li> Immediate sign-up and account provisioning, including a trial period with zero billing details needed at registration. </li>
|
||||
@@ -20,5 +20,5 @@ With 100% consistency across our open source policies and APIs. Prowler Cloud pr
|
||||
<li> Zero touch third party notifications to Slack, Jira, and more. </li>
|
||||
</ul>
|
||||
|
||||
The team who built [Prowler](https://github.com/prowler-cloud/prowler), has helped thousands of companies get Cloud Security under control, is now making it easier by taking
|
||||
The team who built [Prowler](https://github.com/prowler-cloud/prowler), which has helped thousands of companies get Cloud Security under control, is now making it easier by taking
|
||||
[Prowler](https://github.com/prowler-cloud/prowler) to the [Cloud](https://prowler.com)!
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
title: "Overview"
|
||||
---
|
||||
|
||||
**Prowler Hub** is our growing public library of versioned checks, cloud service artifacts, and compliance frameworks with its mappings. It’s searchable, explainable, and built to serve the community.
|
||||
**Prowler Hub** is our growing public library of versioned checks, cloud service artifacts, and compliance frameworks with their mappings. It’s searchable, explainable, and built to serve the community.
|
||||
|
||||
**Why this matters**: Every engineer has asked, “What does this check actually do?” Prowler Hub answers that question in one place, lets you pin to a specific version, and pulls definitions into your own tools or dashboards.
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ Prowler Lighthouse AI is powerful, but there are limitations:
|
||||
- **Continuous improvement**: Please report any issues, as the feature may make mistakes or encounter errors, despite extensive testing.
|
||||
- **Access limitations**: Lighthouse AI can only access data the logged-in user can view. If you can't see certain information, Lighthouse AI can't see it either.
|
||||
- **NextJS session dependence**: If your Prowler application session expires or logs out, Lighthouse AI will error out. Refresh and log back in to continue.
|
||||
- **Response quality**: The response quality depends on the selected LLM provider and model. Choose models with strong tool-calling capabilities for best results. We recommend `gpt-5` model from OpenAI.
|
||||
- **Response quality**: The response quality depends on the selected LLM provider and model. Choose models with strong tool-calling capabilities for best results. We recommend the `gpt-5` model from OpenAI.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -82,7 +82,7 @@ If you encounter issues with Prowler Lighthouse AI or have suggestions for impro
|
||||
|
||||
### What Data Is Shared to LLM Providers?
|
||||
|
||||
The following API endpoints are accessible to Prowler Lighthouse AI. Data from the following API endpoints could be shared with LLM provider depending on the scope of user's query:
|
||||
The following API endpoints are accessible to Prowler Lighthouse AI. Data from the following API endpoints could be shared with the LLM provider depending on the scope of the user's query:
|
||||
|
||||
## FAQs
|
||||
|
||||
@@ -110,7 +110,7 @@ Lighthouse AI [automatically filters](https://github.com/prowler-cloud/prowler/b
|
||||
|
||||
**3. Is my security data shared with LLM providers?**
|
||||
|
||||
Minimal data is shared to generate useful responses. Agent can access security findings and remediation details when needed. Provider secrets are protected by design and cannot be read. The LLM provider credentials configured with Lighthouse AI are only accessible to the Next.js server and are never sent to the LLM providers. Resource metadata (names, tags, account/project IDs, etc.) may be shared with the configured LLM provider based on query requirements.
|
||||
Minimal data is shared to generate useful responses. The agent can access security findings and remediation details when needed. Provider secrets are protected by design and cannot be read. The LLM provider credentials configured with Lighthouse AI are only accessible to the Next.js server and are never sent to the LLM providers. Resource metadata (names, tags, account/project IDs, etc.) may be shared with the configured LLM provider based on query requirements.
|
||||
|
||||
**4. Can the Lighthouse AI change my cloud environment?**
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ Prowler supports a wide range of providers organized by category:
|
||||
| [LLM](/user-guide/providers/llm/getting-started-llm) | Official | Models | CLI |
|
||||
| [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | Tenants | UI, API, CLI |
|
||||
| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | Organizations | UI, API, CLI |
|
||||
| [Okta](/user-guide/providers/okta/getting-started-okta) | Official | Organizations | CLI |
|
||||
| [Okta](/user-guide/providers/okta/getting-started-okta) | Official | Organizations | UI, API, CLI |
|
||||
| [Vercel](/user-guide/providers/vercel/getting-started-vercel) | Official | Teams / Projects | UI, API, CLI |
|
||||
|
||||
### Kubernetes
|
||||
|
||||
@@ -14,7 +14,7 @@ In macOS Ventura, the default value for the `file descriptors` is `256`. With th
|
||||
|
||||
If you have a different OS and you are experiencing the same, please increase the value of your `file descriptors`. You can check it running `ulimit -a | grep "file descriptors"`.
|
||||
|
||||
This error is also related with a lack of system requirements. To improve performance, Prowler stores information in memory so it may need to be run in a system with more than 1GB of memory.
|
||||
This error is also related to a lack of system requirements. To improve performance, Prowler stores information in memory so it may need to be run in a system with more than 1GB of memory.
|
||||
|
||||
See section [Logging](/user-guide/cli/tutorials/logging) for further information or [contact us](/contact).
|
||||
|
||||
|
||||
@@ -239,7 +239,7 @@ The **Code** tab in the Claude desktop app runs the same Claude Code as the CLI,
|
||||
</Step>
|
||||
|
||||
<Step title="Verify in the Code tab">
|
||||
Open a **Code** tab session and ask for a Prowler tool: "Do you have access to the Prowler MCP tools?", it should respond with a list of available tools or confirming that it has access.
|
||||
Open a **Code** tab session and ask for a Prowler tool: "Do you have access to the Prowler MCP tools?", it should respond with a list of available tools or confirm that it has access.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ For Prowler, the **global** scope is usually the right choice — your findings
|
||||
|
||||
<Steps>
|
||||
<Step title="Open the MCP settings">
|
||||
From Agent Window open **Customize** in the Cursor sidebar, then select the MCP section.
|
||||
From the Agent Window open **Customize** in the Cursor sidebar, then select the MCP section.
|
||||
|
||||
On earlier versions, press `Cmd + Shift + J` (macOS) or `Ctrl + Shift + J` (Windows/Linux) to open Cursor Settings, then click **Tools & MCP** in the sidebar.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ Pick your agent below. Each guide is a full walkthrough with screenshots, verifi
|
||||
The Chat tab, via a local bridge
|
||||
</Card>
|
||||
<Card title="Codex / ChatGPT Desktop App" icon="code" href="/user-guide/ai-agents/codex">
|
||||
ChatGPT Desktop App, Codex CLI, the VS Code extension through same config file
|
||||
ChatGPT Desktop App, Codex CLI, the VS Code extension through the same config file
|
||||
</Card>
|
||||
<Card title="Cursor" icon="arrow-pointer" href="/user-guide/ai-agents/cursor">
|
||||
Agentic code editor. Global and project scopes
|
||||
|
||||
@@ -4,7 +4,7 @@ title: "Configuration File"
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
Several Prowler's checks have user configurable variables that can be modified in a common **configuration file**. This file can be found in the following [path](https://github.com/prowler-cloud/prowler/blob/master/prowler/config/config.yaml):
|
||||
Several Prowler checks have user configurable variables that can be modified in a common **configuration file**. This file can be found in the following [path](https://github.com/prowler-cloud/prowler/blob/master/prowler/config/config.yaml):
|
||||
|
||||
```
|
||||
prowler/config/config.yaml
|
||||
@@ -353,7 +353,7 @@ This is the new Prowler configuration file format. The old one without provider
|
||||
# AWS Configuration
|
||||
aws:
|
||||
# AWS Global Configuration
|
||||
# aws.mute_non_default_regions --> Set to True to muted failed findings in non-default regions for AccessAnalyzer, GuardDuty, SecurityHub, DRS and Config
|
||||
# aws.mute_non_default_regions --> Set to True to mute failed findings in non-default regions for AccessAnalyzer, GuardDuty, SecurityHub, DRS and Config
|
||||
mute_non_default_regions: False
|
||||
|
||||
# AWS Resource Scan Limit Configuration
|
||||
|
||||
@@ -10,7 +10,7 @@ You can use `--custom-checks-metadata-file` followed by the path to your custom
|
||||
|
||||
## Available Fields
|
||||
|
||||
The list of supported check's metadata fields that can be overridden are listed as follows:
|
||||
The list of supported check's metadata fields that can be overridden is listed as follows:
|
||||
|
||||
- Severity
|
||||
- CheckTitle
|
||||
|
||||
@@ -65,7 +65,7 @@ This page shows all the info related to the compliance selected. Multiple filter
|
||||
|
||||
<img src="/images/cli/dashboard/dashboard-compliance.png" />
|
||||
|
||||
To add your own compliance to compliance page, add a file with the compliance name (using `_` instead of `.`) to the path `/dashboard/compliance`.
|
||||
To add your own compliance to the compliance page, add a file with the compliance name (using `_` instead of `.`) to the path `/dashboard/compliance`.
|
||||
|
||||
In this file use the format present in the other compliance files to create the table. Example for CIS 2.0:
|
||||
|
||||
|
||||
@@ -163,7 +163,7 @@ Mutelist:
|
||||
Resources:
|
||||
- "test"
|
||||
Tags:
|
||||
- "environment=prod" # Will mute every resource except in account 123456789012 except the ones containing the string "test" and tag environment=prod
|
||||
- "environment=prod" # Will mute every resource in account 123456789012 except the ones containing the string "test" and tag environment=prod
|
||||
|
||||
"*":
|
||||
Checks:
|
||||
|
||||
@@ -26,4 +26,4 @@ By default, it extracts resources from all the regions, you could use `-f`/`--fi
|
||||
|
||||
## Objections
|
||||
|
||||
The inventorying process is carried out with `resourcegroupstaggingapi` calls, which means that only resources they have or have had tags will appear (except for the IAM and S3 resources which are done with Boto3 API calls).
|
||||
The inventorying process is carried out with `resourcegroupstaggingapi` calls, which means that only resources that have or have had tags will appear (except for the IAM and S3 resources which are done with Boto3 API calls).
|
||||
|
||||
@@ -114,7 +114,7 @@ The CSV format follows a standardized structure across all providers. The follow
|
||||
|
||||
#### CSV Headers Mapping
|
||||
|
||||
The following table shows the mapping between the CSV headers and the providers fields:
|
||||
The following table shows the mapping between the CSV headers and the providers' fields:
|
||||
|
||||
| Open Source Consolidated| AWS| GCP| AZURE| KUBERNETES
|
||||
|----------|----------|----------|----------|----------
|
||||
@@ -134,7 +134,7 @@ The following table shows the mapping between the CSV headers and the providers
|
||||
|
||||
### JSON-OCSF
|
||||
|
||||
The JSON-OCSF output format implements the [Detection Finding](https://schema.ocsf.io/classes/detection_finding) from the [OCSF](https://schema.ocsf.io)
|
||||
The JSON-OCSF output format implements the [Detection Finding](https://schema.ocsf.io/classes/detection_finding) from the [OCSF](https://schema.ocsf.io).
|
||||
|
||||
```json
|
||||
[{
|
||||
|
||||
@@ -253,7 +253,7 @@ Standard results plus the framework breakdown are printed to the terminal. A ded
|
||||
<img src="/images/cli/compliance/compliance-cis-sample1.png" />
|
||||
|
||||
<Note>
|
||||
If Prowler cannot find a resource related with a check from a compliance requirement, that requirement is omitted from the output.
|
||||
If Prowler cannot find a resource related to a check from a compliance requirement, that requirement is omitted from the output.
|
||||
</Note>
|
||||
|
||||
### List Available Compliance Frameworks
|
||||
|
||||
@@ -167,7 +167,7 @@ The same `External ID` entered in the Prowler UI must match the `ExternalId` par
|
||||
```
|
||||
|
||||
<Note>
|
||||
Check the aws documentation [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/sts_example_sts_GetSessionToken_section.html)
|
||||
Check the AWS documentation [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/sts_example_sts_GetSessionToken_section.html)
|
||||
</Note>
|
||||
|
||||
2. Copy the output containing:
|
||||
|
||||
@@ -29,6 +29,22 @@ Boto3 defaults both timeouts to 60 seconds. In networks with restricted egress (
|
||||
|
||||
</Note>
|
||||
|
||||
## Retries Configuration
|
||||
|
||||
<VersionBadge version="5.44.0" />
|
||||
|
||||
The number of retries is set with `--aws-retries-max-attempts`, where `0` disables retries. It can also be set through an environment variable, which is the way to tune it in Prowler Cloud and other deployments without a CLI:
|
||||
|
||||
```console
|
||||
export PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS=0
|
||||
```
|
||||
|
||||
The CLI flag takes precedence over the environment variable. The value must be a non-negative integer; when neither is set, Prowler uses 3 retries.
|
||||
|
||||
<Warning>
|
||||
The environment variable is process-wide: it applies to every AWS provider built in the process where it is set, not only to a connection check. A scan started in that same process picks it up too. Boto3's Standard retry mode, which Prowler uses, also retries service-side throttling responses (see the errors listed below), so `0` disables retries for those as well. On a large account a scan can hit throttling under normal load, and with retries disabled that throttling becomes a hard failure instead of a retried call. Set the variable only on the processes that run connection checks; leave scan workers on the default, or raise their retry count instead of lowering it.
|
||||
</Warning>
|
||||
|
||||
## Retry Behavior Overview
|
||||
|
||||
Boto3's Standard retry mode includes the following mechanisms:
|
||||
@@ -65,7 +81,7 @@ Boto3's Standard retry mode includes the following mechanisms:
|
||||
|
||||
- Nondescriptive Transient Error Codes: The retrier applies retry logic to standard HTTP status codes signaling transient errors: 500, 502, 503, 504.
|
||||
|
||||
- Exponential Backoff Strategy: Each retry attempt follows exponential backoff with a base factor of 2, ensuring progressive delay between retries. Maximum backoff time: 20 seconds
|
||||
- Exponential Backoff Strategy: Each retry attempt follows exponential backoff with a base factor of 2, ensuring progressive delay between retries. Maximum backoff time: 20 seconds.
|
||||
|
||||
## Validating Retry Attempts
|
||||
|
||||
@@ -76,4 +92,4 @@ For testing or modifying Prowler's behavior, use the following steps to confirm
|
||||
|
||||
This approach follows the [AWS documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#checking-retry-attempts-in-your-client-logs), which states that if a retry is performed, a message starting with "Retry needed" will be prompted.
|
||||
|
||||
It is possible to determine the total number of calls made using `grep -i 'Sending http request' debuglogs.txt | wc -l`
|
||||
It is possible to determine the total number of calls made using `grep -i 'Sending http request' debuglogs.txt | wc -l`.
|
||||
|
||||
@@ -27,6 +27,7 @@ When scanning the China (`aws-cn`), European Sovereign Cloud (`aws-eusc`) or Gov
|
||||
Refer to: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials for more information about the AWS credential configuration.
|
||||
|
||||
</Note>
|
||||
|
||||
### Declaring the Partition
|
||||
|
||||
`PROWLER_AWS_PARTITION` tells Prowler which partition the scan runs against, without relying on a region being configured:
|
||||
@@ -35,7 +36,7 @@ Refer to: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credenti
|
||||
export PROWLER_AWS_PARTITION="aws-us-gov"
|
||||
```
|
||||
|
||||
It matters most where nothing else says. Resolving an identity means calling STS before anything is known about the credentials, and with no region configured Prowler would otherwise start from the commercial endpoints. Declaring the partition makes that first call go to the right place, which is the difference between a scan that starts and one that fails on an endpoint the credentials cannot use.
|
||||
It matters most where nothing else does. Resolving an identity means calling STS before anything is known about the credentials, and with no region configured Prowler would otherwise start from the commercial endpoints. Declaring the partition makes that first call go to the right place, which is the difference between a scan that starts and one that fails on an endpoint the credentials cannot use.
|
||||
|
||||
A region configured for the session still wins when it belongs to the declared partition, so a deployment in `us-gov-west-1` is not sent to `us-gov-east-1`. A region belonging to a different partition is ignored, since a partition that has been declared explicitly is the more deliberate statement of the two.
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ If AWS Security Hub is already enabled, you can proceed to the [next section](#e
|
||||
|
||||
If the Prowler integration is already enabled in AWS Security Hub, you can proceed to the [next section](#sending-findings-to-aws-security-hub) and begin sending findings.
|
||||
|
||||
Once **AWS Security Hub** is activated, **Prowler** must be enabled as partner integration to allow security findings to be sent to it.
|
||||
Once **AWS Security Hub** is activated, **Prowler** must be enabled as a partner integration to allow security findings to be sent to it.
|
||||
|
||||
1. Enabling AWS Security Hub via Console
|
||||
Open the **AWS Security Hub** console: https://console.aws.amazon.com/securityhub/.
|
||||
@@ -51,7 +51,7 @@ Open the **AWS Security Hub** console: https://console.aws.amazon.com/securityhu
|
||||
|
||||
5. A new modal will appear to confirm that the integration with **Prowler** is being enabled. 
|
||||
|
||||
6. Click “**Accept Findings**”, to authorize **AWS Security Hub** to receive findings from Prowler. 
|
||||
6. Click “**Accept Findings**” to authorize **AWS Security Hub** to receive findings from Prowler. 
|
||||
|
||||
### Using AWS CLI
|
||||
|
||||
@@ -71,7 +71,7 @@ This command requires the `securityhub:EnableSecurityHub` permission. Ensure you
|
||||
</Note>
|
||||
**Step 2: Enable Prowler Integration**
|
||||
|
||||
Once **AWS Security Hub** is activated, **Prowler** must be enabled as partner integration to allow security findings to be sent to it. Run the following AWS CLI commands:
|
||||
Once **AWS Security Hub** is activated, **Prowler** must be enabled as a partner integration to allow security findings to be sent to it. Run the following AWS CLI commands:
|
||||
|
||||
```shell
|
||||
aws securityhub enable-import-findings-for-product --region eu-west-1 --product-arn arn:aws:securityhub:<region>::product/prowler/prowler
|
||||
@@ -151,7 +151,7 @@ prowler --security-hub --status FAIL
|
||||
|
||||
**Configuring Findings Output**
|
||||
|
||||
Instead of using `--status FAIL`, the `--send-sh-only-fails` argument to store all findings in Prowler outputs while sending only FAIL findings to AWS Security:
|
||||
Instead of using `--status FAIL`, use the `--send-sh-only-fails` argument to store all findings in Prowler outputs while sending only FAIL findings to AWS Security Hub:
|
||||
|
||||
```sh
|
||||
prowler --security-hub --send-sh-only-fails
|
||||
|
||||
@@ -55,7 +55,7 @@ For Google Cloud, first enter your `GCP Project ID` and then select the authenti
|
||||
- [Generate a key for a service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)
|
||||
|
||||
<img src="/images/prowler-app/gcp-service-account-creds.png" alt="GCP Service Account Credentials" width="700" />
|
||||
For detailed instructions on how to setup Service Account authentication, see the [Authentication](/user-guide/providers/gcp/authentication#service-account-authentication) page.
|
||||
For detailed instructions on how to set up Service Account authentication, see the [Authentication](/user-guide/providers/gcp/authentication#service-account-authentication) page.
|
||||
</Tab>
|
||||
<Tab title="Application Default Credentials">
|
||||
1. Run the following command in your terminal to authenticate with GCP:
|
||||
|
||||
@@ -396,7 +396,7 @@ Installing PowerShell is different depending on your OS:
|
||||
|
||||
</Tab>
|
||||
<Tab title="Linux (Ubuntu)">
|
||||
[Ubuntu](https://learn.microsoft.com/es-es/powershell/scripting/install/install-ubuntu?view=powershell-7.5#installation-via-package-repository-the-package-repository): The required version for installing PowerShell +7.4 on Ubuntu are Ubuntu 22.04 and Ubuntu 24.04.
|
||||
[Ubuntu](https://learn.microsoft.com/es-es/powershell/scripting/install/install-ubuntu?view=powershell-7.5#installation-via-package-repository-the-package-repository): The required versions for installing PowerShell +7.4 on Ubuntu are Ubuntu 22.04 and Ubuntu 24.04.
|
||||
The recommended way to install it is downloading the package available on PMC.
|
||||
|
||||
Follow these steps:
|
||||
@@ -482,7 +482,7 @@ Installing PowerShell is different depending on your OS:
|
||||
|
||||
</Tab>
|
||||
<Tab title="Linux (Debian)">
|
||||
[Debian](https://learn.microsoft.com/es-es/powershell/scripting/install/install-debian?view=powershell-7.5#installation-on-debian-11-or-12-via-the-package-repository): The required version for installing PowerShell +7.4 on Debian are Debian 11 and Debian 12. The recommended way to install it is downloading the package available on PMC.
|
||||
[Debian](https://learn.microsoft.com/es-es/powershell/scripting/install/install-debian?view=powershell-7.5#installation-on-debian-11-or-12-via-the-package-repository): The required versions for installing PowerShell +7.4 on Debian are Debian 11 and Debian 12. The recommended way to install it is downloading the package available on PMC.
|
||||
|
||||
Follow these steps:
|
||||
|
||||
@@ -521,7 +521,7 @@ Installing PowerShell is different depending on your OS:
|
||||
|
||||
</Tab>
|
||||
<Tab title="Linux (RHEL)">
|
||||
[Rhel](https://learn.microsoft.com/es-es/powershell/scripting/install/install-rhel?view=powershell-7.5#installation-via-the-package-repository): The required version for installing PowerShell +7.4 on Red Hat are RHEL 8 and RHEL 9. The recommended way to install it is downloading the package available on PMC.
|
||||
[Rhel](https://learn.microsoft.com/es-es/powershell/scripting/install/install-rhel?view=powershell-7.5#installation-via-the-package-repository): The required versions for installing PowerShell +7.4 on Red Hat are RHEL 8 and RHEL 9. The recommended way to install it is downloading the package available on PMC.
|
||||
|
||||
Follow these steps:
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
Prowler supports OpenStack both from the CLI and from Prowler Cloud. This guide walks you through the requirements, how to connect the provider in the UI, and how to run scans from the command line.
|
||||
|
||||
<Note>
|
||||
Prowler currently supports **public cloud OpenStack providers** (OVH, Infomaniak, Vexxhost, etc.). Support for self-deployed OpenStack environments is not yet available, if you are interested in this feature, please [open an issue](https://github.com/prowler-cloud/prowler/issues/new) or [contact us](https://prowler.com/contact).
|
||||
Prowler currently supports **public cloud OpenStack providers** (OVH, Infomaniak, Vexxhost, etc.). Support for self-deployed OpenStack environments is not yet available. If you are interested in this feature, please [open an issue](https://github.com/prowler-cloud/prowler/issues/new) or [contact us](https://prowler.com/contact).
|
||||
</Note>
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -154,7 +154,7 @@ Each Jira integration provides management actions through dedicated buttons:
|
||||
|--------|---------|------------------|-------|
|
||||
| **Test** | Verify integration connectivity | • Test Jira API access<br/>• Validate credentials<br/>• Check project permissions<br/>• Verify work item creation capability | Results displayed in notification message |
|
||||
| **Credentials** | Update authentication settings | • Change API token<br/>• Update email<br/>• Update Jira domain | Click "Update Credentials" to save changes |
|
||||
| **Enable/Disable** | Toggle integration status | • Enable or disable integration<br/>| Status change takes effect immediately |
|
||||
| **Enable/Disable** | Toggle integration status | • Enable or disable integration | Status change takes effect immediately |
|
||||
| **Delete** | Remove integration permanently | • Permanently delete integration<br/>• Remove all configuration data | ⚠️ **Cannot be undone** - confirm before deleting |
|
||||
|
||||
## Known Limitations
|
||||
|
||||
@@ -330,8 +330,8 @@ Once the required permissions are set up, proceed to configure the S3 integratio
|
||||
|
||||

|
||||
|
||||
6. Click "Next" to configure credentials
|
||||
7. Configure AWS authentication using one of the supported methods:
|
||||
5. Click "Next" to configure credentials
|
||||
6. Configure AWS authentication using one of the supported methods:
|
||||
|
||||
- **AWS SDK Default:** Use default AWS credentials from the environment. For Prowler Cloud users, this is the recommended option as the service has AWS credentials to assume IAM roles with ARNs matching `arn:aws:iam::*:role/Prowler*` or `arn:aws:iam::*:role/prowler*`
|
||||
- **Access Keys:** Provide AWS access key ID and secret access key
|
||||
@@ -339,14 +339,14 @@ Once the required permissions are set up, proceed to configure the S3 integratio
|
||||
|
||||

|
||||
|
||||
8. Optional - For IAM role authentication, complete the required fields:
|
||||
7. Optional - For IAM role authentication, complete the required fields:
|
||||
|
||||
- **Role ARN:** The Amazon Resource Name of the IAM role
|
||||
- **External ID:** Unique identifier for additional security (defaults to Tenant/Organization ID) - mandatory and automatically filled
|
||||
- **Role Session Name:** Optional - name for the assumed role session
|
||||
- **Session Duration:** Optional - duration in seconds for the session
|
||||
|
||||
9. Click "Create Integration" to verify the connection and complete the setup
|
||||
8. Click "Create Integration" to verify the connection and complete the setup
|
||||
|
||||
<Check>
|
||||
Once credentials are configured and the connection test passes, the S3 integration will be active. Scan results will automatically be exported to the specified bucket after each scan completes. Run a new scan and check the S3 bucket to verify the integration is working.
|
||||
|
||||
@@ -89,6 +89,10 @@ After adding your cloud account credentials, click the `Check connection` button
|
||||
|
||||
<img src="/images/test-connection-button.png" alt="Test Connection" width="700" />
|
||||
|
||||
<Note>
|
||||
For a single AWS account, Prowler tests the connection as part of the `Connect account` step, so the wizard moves straight to launching the scan.
|
||||
</Note>
|
||||
|
||||
## Step 6: Scan Started
|
||||
After the connection check succeeds, save the provider and start your first scan with the `Launch Scan` button. The `Scans` section shows the scan in progress:
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
`PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS` environment variable to set the Boto3 retries for deployments without CLI flags
|
||||
@@ -0,0 +1 @@
|
||||
STS calls after role assumption use the answering region, avoiding a second wait for an unreachable partition region
|
||||
@@ -0,0 +1 @@
|
||||
Pass the E2E AWS credentials to the UI E2E workflow through environment variables instead of template expansion
|
||||
@@ -52,7 +52,7 @@ class _MutableTimestamp:
|
||||
|
||||
timestamp = _MutableTimestamp(datetime.today())
|
||||
timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc))
|
||||
prowler_version = "5.43.0"
|
||||
prowler_version = "5.44.0"
|
||||
html_logo_url = "https://github.com/prowler-cloud/prowler/"
|
||||
square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png"
|
||||
aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png"
|
||||
|
||||
@@ -112,7 +112,7 @@ class AwsProvider(Provider):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
role_arn: str = None,
|
||||
session_duration: int = 3600,
|
||||
external_id: str = None,
|
||||
@@ -141,6 +141,7 @@ class AwsProvider(Provider):
|
||||
|
||||
Args:
|
||||
- retries_max_attempts: The maximum number of retries for the AWS client.
|
||||
Defaults to the PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS environment variable or, if unset, to 3.
|
||||
- role_arn: The ARN of the IAM role to assume.
|
||||
- session_duration: The duration of the session in seconds, between 900 and 43200.
|
||||
- external_id: The external ID to use when assuming the IAM role.
|
||||
@@ -1230,7 +1231,8 @@ class AwsProvider(Provider):
|
||||
|
||||
Args:
|
||||
- session: The AWS session object
|
||||
- assumed_role_info: The AWSAssumeRoleInfo object
|
||||
- assumed_role_info: The AWSAssumeRoleInfo object. Its sts_region is
|
||||
updated to the region that answered, so later calls go straight there
|
||||
|
||||
Returns:
|
||||
- AWSCredentials: The AWS credentials for the assumed role
|
||||
@@ -1256,11 +1258,14 @@ class AwsProvider(Provider):
|
||||
mfa_info = AwsProvider.input_role_mfa_token_and_code()
|
||||
assume_role_arguments["SerialNumber"] = mfa_info.arn
|
||||
assume_role_arguments["TokenCode"] = mfa_info.totp
|
||||
_, assumed_credentials = AwsProvider.sts_call_with_partition_failover(
|
||||
session,
|
||||
assumed_role_info.sts_region,
|
||||
lambda sts_client: sts_client.assume_role(**assume_role_arguments),
|
||||
sts_region, assumed_credentials = (
|
||||
AwsProvider.sts_call_with_partition_failover(
|
||||
session,
|
||||
assumed_role_info.sts_region,
|
||||
lambda sts_client: sts_client.assume_role(**assume_role_arguments),
|
||||
)
|
||||
)
|
||||
assumed_role_info.sts_region = sts_region
|
||||
# Convert the UTC datetime object to your local timezone
|
||||
credentials_expiration_local_time = (
|
||||
assumed_credentials["Credentials"]["Expiration"]
|
||||
@@ -1558,6 +1563,8 @@ class AwsProvider(Provider):
|
||||
session,
|
||||
assumed_role_information,
|
||||
)
|
||||
# Validate where the role was assumed, not where it timed out
|
||||
aws_region = assumed_role_information.sts_region
|
||||
session = Session(
|
||||
aws_access_key_id=assumed_role_credentials.aws_access_key_id,
|
||||
aws_secret_access_key=assumed_role_credentials.aws_secret_access_key,
|
||||
|
||||
@@ -2,7 +2,10 @@ import os
|
||||
|
||||
from botocore.config import Config
|
||||
|
||||
from prowler.providers.aws.exceptions.exceptions import AWSInvalidBoto3TimeoutError
|
||||
from prowler.providers.aws.exceptions.exceptions import (
|
||||
AWSInvalidBoto3RetriesError,
|
||||
AWSInvalidBoto3TimeoutError,
|
||||
)
|
||||
|
||||
AWS_STS_GLOBAL_ENDPOINT_REGION = "us-east-1"
|
||||
AWS_REGION_US_EAST_1 = "us-east-1"
|
||||
@@ -27,10 +30,28 @@ def get_boto3_timeout_from_env(name: str, default: int) -> int:
|
||||
return int(raw)
|
||||
|
||||
|
||||
def get_boto3_retries_from_env(name: str, default: int) -> int:
|
||||
"""Non-negative integer retries read from the environment, or default when unset."""
|
||||
raw = os.getenv(name, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
if not raw.isdecimal():
|
||||
raise AWSInvalidBoto3RetriesError(
|
||||
file=os.path.basename(__file__),
|
||||
message=f"{name} must be a non-negative integer number of retries, got {raw!r}",
|
||||
)
|
||||
return int(raw)
|
||||
|
||||
|
||||
def get_default_session_config() -> Config:
|
||||
return Config(
|
||||
user_agent_extra=BOTO3_USER_AGENT_EXTRA,
|
||||
retries={"max_attempts": BOTO3_RETRIES_MAX_ATTEMPTS, "mode": "standard"},
|
||||
retries={
|
||||
"max_attempts": get_boto3_retries_from_env(
|
||||
"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", BOTO3_RETRIES_MAX_ATTEMPTS
|
||||
),
|
||||
"mode": "standard",
|
||||
},
|
||||
connect_timeout=get_boto3_timeout_from_env(
|
||||
"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", BOTO3_CONNECT_TIMEOUT
|
||||
),
|
||||
|
||||
@@ -82,6 +82,10 @@ class AWSBaseException(ProwlerException):
|
||||
"message": "The Boto3 timeout configured through the environment is invalid",
|
||||
"remediation": "Set PROWLER_AWS_BOTO3_CONNECT_TIMEOUT and PROWLER_AWS_BOTO3_READ_TIMEOUT to a positive integer number of seconds.",
|
||||
},
|
||||
(1919, "AWSInvalidBoto3RetriesError"): {
|
||||
"message": "The Boto3 retries configured through the environment are invalid",
|
||||
"remediation": "Set PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS to a non-negative integer, 0 disables retries.",
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, code, file=None, original_exception=None, message=None):
|
||||
@@ -244,3 +248,12 @@ class AWSInvalidBoto3TimeoutError(AWSBaseException):
|
||||
super().__init__(
|
||||
1918, file=file, original_exception=original_exception, message=message
|
||||
)
|
||||
|
||||
|
||||
class AWSInvalidBoto3RetriesError(AWSBaseException):
|
||||
"""Boto3 retries configured through the environment are not a non-negative integer."""
|
||||
|
||||
def __init__(self, file=None, original_exception=None, message=None):
|
||||
super().__init__(
|
||||
1919, file=file, original_exception=original_exception, message=message
|
||||
)
|
||||
|
||||
@@ -85,7 +85,7 @@ class S3:
|
||||
aws_access_key_id: str = None,
|
||||
aws_secret_access_key: str = None,
|
||||
aws_session_token: Optional[str] = None,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
regions: set = set(),
|
||||
) -> None:
|
||||
"""
|
||||
|
||||
@@ -106,7 +106,7 @@ class SecurityHub:
|
||||
aws_access_key_id: str = None,
|
||||
aws_secret_access_key: str = None,
|
||||
aws_session_token: Optional[str] = None,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
regions: set = set(),
|
||||
) -> "SecurityHub":
|
||||
"""
|
||||
|
||||
@@ -40,7 +40,7 @@ class AwsSetUpSession:
|
||||
aws_access_key_id: str = None,
|
||||
aws_secret_access_key: str = None,
|
||||
aws_session_token: Optional[str] = None,
|
||||
retries_max_attempts: int = 3,
|
||||
retries_max_attempts: Optional[int] = None,
|
||||
regions: set = set(),
|
||||
connect_timeout: Optional[int] = None,
|
||||
read_timeout: Optional[int] = None,
|
||||
@@ -106,6 +106,8 @@ class AwsSetUpSession:
|
||||
session=self._session.current_session,
|
||||
aws_region=sts_region,
|
||||
)
|
||||
# Later STS calls go where validation got an answer, not where it timed out
|
||||
sts_region = caller_identity.region
|
||||
|
||||
logger.info("Credentials validated")
|
||||
########
|
||||
|
||||
+1
-1
@@ -144,7 +144,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"}
|
||||
name = "prowler"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.14"
|
||||
version = "5.43.0"
|
||||
version = "5.44.0"
|
||||
|
||||
[project.scripts]
|
||||
prowler = "prowler.__main__:prowler"
|
||||
|
||||
@@ -28,15 +28,19 @@ from prowler.providers.aws.config import (
|
||||
AWS_STS_GLOBAL_ENDPOINT_REGION,
|
||||
BOTO3_CONNECT_TIMEOUT,
|
||||
BOTO3_READ_TIMEOUT,
|
||||
BOTO3_RETRIES_MAX_ATTEMPTS,
|
||||
BOTO3_USER_AGENT_EXTRA,
|
||||
ROLE_SESSION_NAME,
|
||||
get_boto3_retries_from_env,
|
||||
get_boto3_timeout_from_env,
|
||||
get_default_session_config,
|
||||
)
|
||||
from prowler.providers.aws.exceptions.exceptions import (
|
||||
AWSAccessKeyIDInvalidError,
|
||||
AWSArgumentTypeValidationError,
|
||||
AWSAssumeRoleError,
|
||||
AWSIAMRoleARNInvalidResourceTypeError,
|
||||
AWSInvalidBoto3RetriesError,
|
||||
AWSInvalidBoto3TimeoutError,
|
||||
AWSInvalidPartitionError,
|
||||
AWSInvalidProviderIdError,
|
||||
@@ -1845,6 +1849,143 @@ aws:
|
||||
]
|
||||
assert isinstance(credentials, AWSCredentials)
|
||||
assert credentials.aws_access_key_id == "AKIAIOSFODNN7EXAMPLE"
|
||||
# Refreshing the credentials later goes straight to the region that answered
|
||||
assert assumed_role_info.sts_region == AWS_REGION_GOV_CLOUD_US_WEST_1
|
||||
|
||||
def test_assume_role_does_not_retry_a_credential_error(self, monkeypatch):
|
||||
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
|
||||
current_session = session.Session(region_name=AWS_REGION_US_EAST_1)
|
||||
attempted_regions = []
|
||||
|
||||
def create_sts_session(session, aws_region):
|
||||
attempted_regions.append(aws_region)
|
||||
sts_client = mock.MagicMock()
|
||||
sts_client.assume_role.side_effect = botocore.exceptions.ClientError(
|
||||
{"Error": {"Code": "AccessDenied", "Message": "denied"}},
|
||||
"AssumeRole",
|
||||
)
|
||||
return sts_client
|
||||
|
||||
assumed_role_info = AWSAssumeRoleInfo(
|
||||
role_arn=ARN(
|
||||
arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role"
|
||||
),
|
||||
session_duration=3600,
|
||||
external_id=None,
|
||||
mfa_enabled=False,
|
||||
role_session_name=ROLE_SESSION_NAME,
|
||||
sts_region=AWS_REGION_GOV_CLOUD_US_EAST_1,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.create_sts_session",
|
||||
new=create_sts_session,
|
||||
):
|
||||
with raises(AWSAssumeRoleError):
|
||||
AwsProvider.assume_role(current_session, assumed_role_info)
|
||||
|
||||
assert attempted_regions == [AWS_REGION_GOV_CLOUD_US_EAST_1]
|
||||
assert assumed_role_info.sts_region == AWS_REGION_GOV_CLOUD_US_EAST_1
|
||||
|
||||
def test_test_connection_role_validates_where_the_role_was_assumed(
|
||||
self, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
|
||||
monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
|
||||
attempted_calls = []
|
||||
|
||||
def create_sts_session(session, aws_region):
|
||||
if aws_region == AWS_REGION_GOV_CLOUD_US_EAST_1:
|
||||
attempted_calls.append(aws_region)
|
||||
raise botocore.exceptions.ConnectTimeoutError(
|
||||
endpoint_url=f"https://sts.{aws_region}.amazonaws.com"
|
||||
)
|
||||
sts_client = mock.MagicMock()
|
||||
|
||||
def assume_role(**_):
|
||||
attempted_calls.append(("AssumeRole", aws_region))
|
||||
return {
|
||||
"Credentials": {
|
||||
"AccessKeyId": "AKIAIOSFODNN7EXAMPLE",
|
||||
"SecretAccessKey": "secret",
|
||||
"SessionToken": "token",
|
||||
"Expiration": datetime.now() + timedelta(seconds=3600),
|
||||
}
|
||||
}
|
||||
|
||||
def get_caller_identity():
|
||||
attempted_calls.append(("GetCallerIdentity", aws_region))
|
||||
return {
|
||||
"UserId": "test-user-id",
|
||||
"Account": AWS_ACCOUNT_NUMBER,
|
||||
"Arn": AWS_GOV_CLOUD_ACCOUNT_ARN,
|
||||
}
|
||||
|
||||
sts_client.assume_role.side_effect = assume_role
|
||||
sts_client.get_caller_identity.side_effect = get_caller_identity
|
||||
return sts_client
|
||||
|
||||
with patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.create_sts_session",
|
||||
new=create_sts_session,
|
||||
):
|
||||
connection = AwsProvider.test_connection(
|
||||
role_arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role",
|
||||
aws_access_key_id="test-access-key",
|
||||
aws_secret_access_key="test-secret-key",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
assert connection.is_connected
|
||||
# The unreachable region is paid for once, not again for the validation
|
||||
assert attempted_calls == [
|
||||
AWS_REGION_GOV_CLOUD_US_EAST_1,
|
||||
("AssumeRole", AWS_REGION_GOV_CLOUD_US_WEST_1),
|
||||
("GetCallerIdentity", AWS_REGION_GOV_CLOUD_US_WEST_1),
|
||||
]
|
||||
|
||||
@mock_aws
|
||||
def test_aws_set_up_session_assumes_the_role_where_validation_got_an_answer(
|
||||
self, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("PROWLER_AWS_PARTITION", AWS_GOV_CLOUD_PARTITION)
|
||||
monkeypatch.setenv("AWS_DEFAULT_REGION", AWS_REGION_US_EAST_1)
|
||||
answered = AWSCallerIdentity(
|
||||
user_id="test-user-id",
|
||||
account=AWS_ACCOUNT_NUMBER,
|
||||
arn=ARN(AWS_GOV_CLOUD_ACCOUNT_ARN),
|
||||
region=AWS_REGION_GOV_CLOUD_US_WEST_1,
|
||||
)
|
||||
sts_regions = []
|
||||
|
||||
class RoleAssumed(Exception):
|
||||
pass
|
||||
|
||||
def assume_role(session, assumed_role_info):
|
||||
sts_regions.append(assumed_role_info.sts_region)
|
||||
raise RoleAssumed
|
||||
|
||||
with (
|
||||
patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.validate_credentials",
|
||||
return_value=answered,
|
||||
),
|
||||
patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.assume_role",
|
||||
side_effect=assume_role,
|
||||
),
|
||||
):
|
||||
with raises(RoleAssumed):
|
||||
AwsSetUpSession(
|
||||
role_arn=f"arn:{AWS_GOV_CLOUD_PARTITION}:iam::{AWS_ACCOUNT_NUMBER}:role/test-role",
|
||||
session_duration=900,
|
||||
external_id="test-external-id",
|
||||
role_session_name=ROLE_SESSION_NAME,
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
)
|
||||
|
||||
assert sts_regions == [AWS_REGION_GOV_CLOUD_US_WEST_1]
|
||||
|
||||
def test_setup_session_mfa_falls_back_to_the_next_partition_region(
|
||||
self, monkeypatch
|
||||
@@ -3352,6 +3493,123 @@ aws:
|
||||
):
|
||||
get_boto3_timeout_from_env("PROWLER_AWS_BOTO3_CONNECT_TIMEOUT", 10)
|
||||
|
||||
def test_get_default_session_config_retries_from_env(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
|
||||
):
|
||||
config = get_default_session_config()
|
||||
|
||||
assert config.retries == {"max_attempts": 1, "mode": "standard"}
|
||||
|
||||
def test_get_default_session_config_retries_from_env_0_disables_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
):
|
||||
config = get_default_session_config()
|
||||
|
||||
assert config.retries == {"max_attempts": 0, "mode": "standard"}
|
||||
|
||||
def test_set_session_config_argument_overrides_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
|
||||
):
|
||||
config = AwsProvider.set_session_config(5)
|
||||
|
||||
assert config.retries == {"max_attempts": 5, "mode": "standard"}
|
||||
|
||||
@mock_aws
|
||||
def test_aws_provider_without_retries_argument_uses_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
):
|
||||
aws_provider = AwsProvider()
|
||||
client = aws_provider.session.current_session.client(
|
||||
"ec2", region_name=AWS_REGION_US_EAST_1
|
||||
)
|
||||
|
||||
# botocore rewrites max_attempts into total_max_attempts (retries + 1)
|
||||
assert client.meta.config.retries["total_max_attempts"] == 1
|
||||
|
||||
@mock_aws
|
||||
def test_aws_provider_retries_argument_overrides_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
):
|
||||
aws_provider = AwsProvider(retries_max_attempts=7)
|
||||
client = aws_provider.session.current_session.client(
|
||||
"ec2", region_name=AWS_REGION_US_EAST_1
|
||||
)
|
||||
|
||||
assert client.meta.config.retries["total_max_attempts"] == 8
|
||||
|
||||
@mock_aws
|
||||
def test_aws_set_up_session_without_retries_argument_uses_env_retries(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "1"}
|
||||
):
|
||||
aws_session = AwsSetUpSession(
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
)
|
||||
client = aws_session._session.current_session.client(
|
||||
"ec2", region_name=AWS_REGION_US_EAST_1
|
||||
)
|
||||
|
||||
assert client.meta.config.retries["total_max_attempts"] == 2
|
||||
|
||||
def test_test_connection_session_uses_env_retries(self):
|
||||
with (
|
||||
mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": "0"}
|
||||
),
|
||||
mock.patch.object(
|
||||
AwsProvider,
|
||||
"validate_credentials",
|
||||
return_value=AWSCallerIdentity(
|
||||
user_id="test-user-id",
|
||||
account=AWS_ACCOUNT_NUMBER,
|
||||
arn=ARN(AWS_ACCOUNT_ARN),
|
||||
region=AWS_REGION_US_EAST_1,
|
||||
),
|
||||
) as mock_validate_credentials,
|
||||
):
|
||||
connection = AwsProvider.test_connection(
|
||||
aws_access_key_id="test-access-key",
|
||||
aws_secret_access_key="test-secret-key",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
assert connection.is_connected
|
||||
validated_session = mock_validate_credentials.call_args.args[0]
|
||||
assert validated_session._session.get_default_client_config().retries == {
|
||||
"max_attempts": 0,
|
||||
"mode": "standard",
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize("raw", ["-1", "three", "1.5"])
|
||||
def test_get_boto3_retries_from_env_rejects_anything_but_non_negative_integers(
|
||||
self, raw
|
||||
):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": raw}
|
||||
):
|
||||
with raises(
|
||||
AWSInvalidBoto3RetriesError,
|
||||
match="PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS",
|
||||
):
|
||||
get_boto3_retries_from_env("PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", 3)
|
||||
|
||||
def test_get_boto3_retries_from_env_blank_falls_back_to_default(self):
|
||||
with mock.patch.dict(
|
||||
os.environ, {"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS": " "}
|
||||
):
|
||||
assert (
|
||||
get_boto3_retries_from_env(
|
||||
"PROWLER_AWS_BOTO3_RETRIES_MAX_ATTEMPTS", BOTO3_RETRIES_MAX_ATTEMPTS
|
||||
)
|
||||
== BOTO3_RETRIES_MAX_ATTEMPTS
|
||||
)
|
||||
|
||||
def test_get_boto3_timeout_from_env_blank_falls_back_to_default(self):
|
||||
with mock.patch.dict(os.environ, {"PROWLER_AWS_BOTO3_CONNECT_TIMEOUT": " "}):
|
||||
assert (
|
||||
|
||||
@@ -302,6 +302,7 @@ export const handlersForOrganizations = (
|
||||
organizations.map((o) => o.secretId).filter((id): id is string => !!id),
|
||||
);
|
||||
let orgSeq = 0;
|
||||
let providerSeq = 0;
|
||||
let secretSeq = 0;
|
||||
/** Reads per connection task, so `executingPolls` can hold one task running. */
|
||||
const connectionTaskReads = new Map<string, number>();
|
||||
@@ -565,6 +566,37 @@ export const handlersForOrganizations = (
|
||||
HttpResponse.json({ data: [], meta: collectionMeta(0) }),
|
||||
),
|
||||
|
||||
// --- single-account connect (AWS one-step form) -----------------------
|
||||
http.post(`${API}/providers`, async ({ request }) => {
|
||||
const body = (await request.json()) as {
|
||||
data: { attributes: { provider: string; uid: string; alias?: string } };
|
||||
};
|
||||
providerSeq += 1;
|
||||
return HttpResponse.json(
|
||||
{
|
||||
data: {
|
||||
id: `provider-created-${providerSeq}`,
|
||||
type: "providers",
|
||||
attributes: {
|
||||
...body.data.attributes,
|
||||
connection: { connected: false, last_checked_at: null },
|
||||
},
|
||||
},
|
||||
},
|
||||
{ status: 201 },
|
||||
);
|
||||
}),
|
||||
|
||||
http.post(`${API}/providers/secrets`, () => {
|
||||
secretSeq += 1;
|
||||
return HttpResponse.json(
|
||||
{
|
||||
data: { id: `secret-created-${secretSeq}`, type: "provider-secrets" },
|
||||
},
|
||||
{ status: 201 },
|
||||
);
|
||||
}),
|
||||
|
||||
// --- providers (uid resolution) + connection testing -----------------
|
||||
http.get<{ id: string }>(`${API}/providers/:id`, ({ params }) => {
|
||||
const provider = fx.providers.find((p) => p.id === params.id);
|
||||
|
||||
@@ -215,3 +215,72 @@ describe("adaptFindingsByResourceResponse — malformed input", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("adaptFindingsByResourceResponse — provider id", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("should carry the provider id resolved through the scan include", () => {
|
||||
// Given — scan.provider include path, as the drawer requests it
|
||||
createDictMock.mockImplementation((type: string) => {
|
||||
if (type === "scans") {
|
||||
return {
|
||||
"scan-1": {
|
||||
id: "scan-1",
|
||||
attributes: {},
|
||||
relationships: { provider: { data: { id: "provider-1" } } },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (type === "providers") {
|
||||
return {
|
||||
"provider-1": {
|
||||
id: "provider-1",
|
||||
attributes: { provider: "aws", alias: "prod", uid: "123" },
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const input = {
|
||||
data: {
|
||||
id: "finding-1",
|
||||
attributes: {
|
||||
uid: "uid-1",
|
||||
check_id: "s3_check",
|
||||
status: "FAIL",
|
||||
severity: "high",
|
||||
check_metadata: {},
|
||||
},
|
||||
relationships: {
|
||||
resources: { data: [] },
|
||||
scan: { data: { id: "scan-1" } },
|
||||
},
|
||||
},
|
||||
included: [],
|
||||
};
|
||||
|
||||
// When
|
||||
const [finding] = adaptFindingsByResourceResponse(input);
|
||||
|
||||
// Then — the partial-scan request needs the id, not only the uid
|
||||
expect(finding.providerId).toBe("provider-1");
|
||||
expect(finding.providerUid).toBe("123");
|
||||
});
|
||||
|
||||
it("should leave the provider id empty when the scan is not included", () => {
|
||||
createDictMock.mockReturnValue({});
|
||||
|
||||
const [finding] = adaptFindingsByResourceResponse({
|
||||
data: {
|
||||
id: "finding-1",
|
||||
attributes: { uid: "uid-1", check_id: "s3_check", status: "FAIL" },
|
||||
relationships: { resources: { data: [] }, scan: { data: null } },
|
||||
},
|
||||
});
|
||||
|
||||
expect(finding.providerId).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,6 +64,7 @@ export interface ResourceDrawerFinding {
|
||||
resourceDetails: string | null;
|
||||
resourceMetadata: Record<string, unknown> | string | null;
|
||||
// Provider
|
||||
providerId: string;
|
||||
providerType: ProviderType;
|
||||
providerAlias: string;
|
||||
providerUid: string;
|
||||
@@ -280,6 +281,7 @@ export function adaptFindingsByResourceResponse(
|
||||
| null
|
||||
| undefined) ?? null,
|
||||
// Provider
|
||||
providerId: providerRelId ?? "",
|
||||
providerType: ((providerAttrs.provider as string | undefined) ||
|
||||
"aws") as ProviderType,
|
||||
providerAlias: (providerAttrs.alias as string | undefined) || "",
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { fetchMock, getAuthHeadersMock, revalidatePathMock } = vi.hoisted(
|
||||
() => ({
|
||||
fetchMock: vi.fn(),
|
||||
getAuthHeadersMock: vi.fn(),
|
||||
revalidatePathMock: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
vi.mock("@/lib/helper", () => ({
|
||||
apiBaseUrl: "https://api.test/api/v1",
|
||||
getAuthHeaders: getAuthHeadersMock,
|
||||
}));
|
||||
|
||||
vi.mock("next/cache", () => ({
|
||||
revalidatePath: revalidatePathMock,
|
||||
}));
|
||||
|
||||
import { createMuteRule } from "./mute-rules";
|
||||
|
||||
const NAME_CONFLICT_DETAIL = "A mute rule with this name already exists.";
|
||||
|
||||
const errorResponse = (contentType: string, body: string, status = 400) =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "Content-Type": contentType },
|
||||
});
|
||||
|
||||
const nameConflictBody = JSON.stringify({
|
||||
errors: [
|
||||
{
|
||||
detail: NAME_CONFLICT_DETAIL,
|
||||
status: "400",
|
||||
source: { pointer: "/data/attributes/name" },
|
||||
code: "invalid",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const muteRuleFormData = () => {
|
||||
const formData = new FormData();
|
||||
formData.set("name", "Root account has a hardware MFA device enabled");
|
||||
formData.set("reason", "Not our approach here with SSO");
|
||||
formData.set("finding_ids", JSON.stringify(["finding-1"]));
|
||||
return formData;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
});
|
||||
|
||||
describe("createMuteRule", () => {
|
||||
it("should return only the error detail for a JSON:API error response", async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
errorResponse("application/vnd.api+json", nameConflictBody),
|
||||
);
|
||||
|
||||
const result = await createMuteRule(null, muteRuleFormData());
|
||||
|
||||
expect(result?.errors?.general).toBe(NAME_CONFLICT_DETAIL);
|
||||
expect(revalidatePathMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should return only the error detail for a plain JSON error response", async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
errorResponse("application/json", nameConflictBody),
|
||||
);
|
||||
|
||||
const result = await createMuteRule(null, muteRuleFormData());
|
||||
|
||||
expect(result?.errors?.general).toBe(NAME_CONFLICT_DETAIL);
|
||||
});
|
||||
|
||||
it("should return the response text for a non-JSON error response", async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
errorResponse("text/plain", "Bad gateway", 502),
|
||||
);
|
||||
|
||||
const result = await createMuteRule(null, muteRuleFormData());
|
||||
|
||||
expect(result?.errors?.general).toBe("Bad gateway");
|
||||
});
|
||||
});
|
||||
@@ -156,7 +156,8 @@ export const createMuteRule = async (
|
||||
let errorMessage = `Failed to create mute rule: ${response.statusText}`;
|
||||
const responseContentType = response.headers.get("content-type");
|
||||
try {
|
||||
if (responseContentType?.includes("application/json")) {
|
||||
// The API answers with application/vnd.api+json
|
||||
if (responseContentType?.includes("json")) {
|
||||
const errorData = await response.json();
|
||||
const jsonApiError = (
|
||||
errorData as {
|
||||
|
||||
@@ -546,7 +546,7 @@ export const applyDiscovery = async (
|
||||
);
|
||||
// No `include`: the apply view rejects the parameter outright and fails the
|
||||
// whole request. The created providers' uids are read afterwards instead, with
|
||||
// `getProviderUidsByIds`.
|
||||
// `getProviderUidsAndConnectionBaselines`.
|
||||
|
||||
const attributes = buildApplyAttributes(payload);
|
||||
|
||||
|
||||
@@ -151,28 +151,28 @@ export const getProvider = async (formData: FormData) => {
|
||||
const PROVIDERS_PAGE_MAX = 100;
|
||||
|
||||
/**
|
||||
* Uids of the given providers, keyed by provider id. A provider's `uid` is the
|
||||
* candidate it was created for (AWS account id / GCP project id), so this is what
|
||||
* matches an apply's created providers back to the selection. Batched with
|
||||
* `filter[id__in]` rather than one `GET /providers/{id}` per id.
|
||||
* Providers matching the given ids, batched with `filter[id__in]` (page size
|
||||
* `PROVIDERS_PAGE_MAX`, the server max, which also bounds the id batch size)
|
||||
* rather than one `GET /providers/{id}` per id. Shared by every action below
|
||||
* that resolves providers by id; a batch that fails to fetch leaves its
|
||||
* providers out of the result rather than failing the rest, and it is on each
|
||||
* caller to say what "missing" means for its own map. Not exported: an
|
||||
* exported function in this `"use server"` module becomes a callable server
|
||||
* action.
|
||||
*/
|
||||
export const getProviderUidsByIds = async (
|
||||
const fetchProvidersByIds = async (
|
||||
providerIds: string[],
|
||||
): Promise<Record<string, string>> => {
|
||||
): Promise<ProvidersApiResponse["data"]> => {
|
||||
const uniqueIds = Array.from(new Set(providerIds.filter(Boolean)));
|
||||
if (uniqueIds.length === 0) {
|
||||
return {};
|
||||
return [];
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
const batches: string[][] = [];
|
||||
const providers: ProvidersApiResponse["data"] = [];
|
||||
|
||||
for (let start = 0; start < uniqueIds.length; start += PROVIDERS_PAGE_MAX) {
|
||||
batches.push(uniqueIds.slice(start, start + PROVIDERS_PAGE_MAX));
|
||||
}
|
||||
|
||||
const uidById: Record<string, string> = {};
|
||||
|
||||
for (const batch of batches) {
|
||||
const batch = uniqueIds.slice(start, start + PROVIDERS_PAGE_MAX);
|
||||
const url = new URL(`${apiBaseUrl}/providers`);
|
||||
url.searchParams.set("filter[id__in]", batch.join(","));
|
||||
url.searchParams.set("page[size]", String(PROVIDERS_PAGE_MAX));
|
||||
@@ -183,20 +183,93 @@ export const getProviderUidsByIds = async (
|
||||
| ProvidersApiResponse
|
||||
| undefined;
|
||||
|
||||
for (const provider of result?.data ?? []) {
|
||||
const uid = provider?.attributes?.uid;
|
||||
if (typeof provider?.id === "string" && typeof uid === "string") {
|
||||
uidById[provider.id] = uid;
|
||||
}
|
||||
}
|
||||
providers.push(...(result?.data ?? []));
|
||||
} catch {
|
||||
// A failed batch leaves its providers unmapped rather than failing the rest.
|
||||
// A failed batch leaves its providers out of the result rather than
|
||||
// failing the rest.
|
||||
}
|
||||
}
|
||||
|
||||
return providers;
|
||||
};
|
||||
|
||||
/**
|
||||
* Uids of the given providers, keyed by provider id. A provider's `uid` is the
|
||||
* candidate it was created for (AWS account id / GCP project id), so this is what
|
||||
* matches an apply's created providers back to the selection.
|
||||
*/
|
||||
export const getProviderUidsByIds = async (
|
||||
providerIds: string[],
|
||||
): Promise<Record<string, string>> => {
|
||||
const uidById: Record<string, string> = {};
|
||||
|
||||
for (const provider of await fetchProvidersByIds(providerIds)) {
|
||||
const uid = provider?.attributes?.uid;
|
||||
if (typeof provider?.id === "string" && typeof uid === "string") {
|
||||
uidById[provider.id] = uid;
|
||||
}
|
||||
}
|
||||
|
||||
return uidById;
|
||||
};
|
||||
|
||||
/**
|
||||
* `connection.last_checked_at` for each given provider, keyed by id. Read before
|
||||
* a batch of connection checks is dispatched, so `resolveProviderConnectionState`
|
||||
* can tell a check's own result apart from an older one already on record by
|
||||
* comparing values, never by comparing the browser's clock against the server's
|
||||
* (see that function for why). A provider missing from the response -- the batch
|
||||
* read failed, or it was deleted mid-flight -- is left out of the map rather than
|
||||
* defaulted, so callers can tell "no prior check" (`null`) from "unknown".
|
||||
*/
|
||||
export const getProviderConnectionBaselines = async (
|
||||
providerIds: string[],
|
||||
): Promise<Record<string, string | null>> => {
|
||||
const baselineById: Record<string, string | null> = {};
|
||||
|
||||
for (const provider of await fetchProvidersByIds(providerIds)) {
|
||||
if (typeof provider?.id === "string") {
|
||||
baselineById[provider.id] =
|
||||
provider.attributes?.connection?.last_checked_at ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
return baselineById;
|
||||
};
|
||||
|
||||
/**
|
||||
* Uid and `connection.last_checked_at` for each given provider, keyed by id, read
|
||||
* with a single batched `filter[id__in]` request. The organization onboarding
|
||||
* apply step needs both right after creating providers: the uid to match each one
|
||||
* back to the candidate it was created for (see `getProviderUidsByIds`), and the
|
||||
* baseline to compare a dispatched check's result against (see
|
||||
* `getProviderConnectionBaselines`). Reading them together avoids fetching the
|
||||
* same set of providers twice.
|
||||
*/
|
||||
export const getProviderUidsAndConnectionBaselines = async (
|
||||
providerIds: string[],
|
||||
): Promise<{
|
||||
uidById: Record<string, string>;
|
||||
baselineById: Record<string, string | null>;
|
||||
}> => {
|
||||
const uidById: Record<string, string> = {};
|
||||
const baselineById: Record<string, string | null> = {};
|
||||
|
||||
for (const provider of await fetchProvidersByIds(providerIds)) {
|
||||
if (typeof provider?.id !== "string") {
|
||||
continue;
|
||||
}
|
||||
const uid = provider.attributes?.uid;
|
||||
if (typeof uid === "string") {
|
||||
uidById[provider.id] = uid;
|
||||
}
|
||||
baselineById[provider.id] =
|
||||
provider.attributes?.connection?.last_checked_at ?? null;
|
||||
}
|
||||
|
||||
return { uidById, baselineById };
|
||||
};
|
||||
|
||||
export const updateProvider = async (formData: FormData) => {
|
||||
const headers = await getAuthHeaders({ contentType: true });
|
||||
const providerId = formData.get(ProviderCredentialFields.PROVIDER_ID);
|
||||
|
||||
@@ -6,12 +6,14 @@ const {
|
||||
getAuthHeadersMock,
|
||||
handleApiErrorMock,
|
||||
handleApiResponseMock,
|
||||
isReportDownloadLockedMock,
|
||||
} = vi.hoisted(() => ({
|
||||
addScanOperationMock: vi.fn(),
|
||||
fetchMock: vi.fn(),
|
||||
getAuthHeadersMock: vi.fn(),
|
||||
handleApiErrorMock: vi.fn(),
|
||||
handleApiResponseMock: vi.fn(),
|
||||
isReportDownloadLockedMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib", () => ({
|
||||
@@ -23,6 +25,10 @@ vi.mock("@/lib", () => ({
|
||||
error instanceof Error ? error.message : String(error),
|
||||
}));
|
||||
|
||||
vi.mock("next/cache", () => ({
|
||||
revalidatePath: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/server-actions-helper", () => ({
|
||||
handleApiError: handleApiErrorMock,
|
||||
handleApiResponse: handleApiResponseMock,
|
||||
@@ -32,7 +38,17 @@ vi.mock("@/lib/sentry-breadcrumbs", () => ({
|
||||
addScanOperation: addScanOperationMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/report-download-access", () => ({
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR:
|
||||
"Report downloads require an active subscription.",
|
||||
isReportDownloadLocked: isReportDownloadLockedMock,
|
||||
}));
|
||||
|
||||
import {
|
||||
createPartialScan,
|
||||
getComplianceCsv,
|
||||
getComplianceOcsf,
|
||||
getCompliancePdfReport,
|
||||
getExportsZip,
|
||||
launchOrganizationScans,
|
||||
scheduleOrganizationDailyScans,
|
||||
@@ -156,6 +172,7 @@ describe("getExportsZip", () => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
isReportDownloadLockedMock.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it("returns a generic server error when the report endpoint returns HTML", async () => {
|
||||
@@ -181,3 +198,98 @@ describe("getExportsZip", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("report downloads for subscription-only tenants", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
isReportDownloadLockedMock.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "scan ZIP", download: () => getExportsZip("scan-123") },
|
||||
{
|
||||
name: "compliance CSV",
|
||||
download: () => getComplianceCsv("scan-123", "cis_2.0_aws"),
|
||||
},
|
||||
{
|
||||
name: "compliance OCSF",
|
||||
download: () => getComplianceOcsf("scan-123", "dora_aws"),
|
||||
},
|
||||
{
|
||||
name: "compliance PDF",
|
||||
download: () => getCompliancePdfReport("scan-123", "threatscore"),
|
||||
},
|
||||
])("rejects the $name without calling the API", async ({ download }) => {
|
||||
// When
|
||||
const result = await download();
|
||||
|
||||
// Then
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
error: "Report downloads require an active subscription.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("createPartialScan", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
getAuthHeadersMock.mockResolvedValue({ Authorization: "Bearer token" });
|
||||
fetchMock.mockResolvedValue(new Response(null, { status: 202 }));
|
||||
handleApiResponseMock.mockResolvedValue({ data: { id: "scan-1" } });
|
||||
});
|
||||
|
||||
it("posts the resource uids as a scan of one provider", async () => {
|
||||
// When
|
||||
const result = await createPartialScan({
|
||||
providerId: "provider-1",
|
||||
resourceUids: ["arn:aws:s3:::bucket"],
|
||||
});
|
||||
|
||||
// Then
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.example.com/api/v1/scans",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
type: "scans",
|
||||
attributes: { resource_uids: ["arn:aws:s3:::bucket"] },
|
||||
relationships: {
|
||||
provider: { data: { type: "providers", id: "provider-1" } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(handleApiResponseMock).toHaveBeenCalledWith(
|
||||
expect.any(Response),
|
||||
"/scans",
|
||||
);
|
||||
expect(result).toEqual({ data: { id: "scan-1" } });
|
||||
expect(addScanOperationMock).toHaveBeenCalledWith("start", "scan-1");
|
||||
});
|
||||
|
||||
it("refuses more resources than the API accepts without calling it", async () => {
|
||||
// Given — the API caps a partial scan at 10 resources.
|
||||
const resourceUids = Array.from(
|
||||
{ length: 11 },
|
||||
(_, index) => `arn:aws:s3:::bucket-${index}`,
|
||||
);
|
||||
|
||||
// When
|
||||
const result = await createPartialScan({
|
||||
providerId: "provider-1",
|
||||
resourceUids,
|
||||
});
|
||||
|
||||
// Then
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
error: "Select between 1 and 10 resources to re-check",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,9 +15,14 @@ import {
|
||||
} from "@/lib/compliance/compliance-report-types";
|
||||
import { runWithConcurrencyLimit } from "@/lib/concurrency";
|
||||
import { appendSanitizedProviderTypeFilters } from "@/lib/provider-filters";
|
||||
import {
|
||||
isReportDownloadLocked,
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR,
|
||||
} from "@/lib/report-download-access";
|
||||
import { addScanOperation } from "@/lib/sentry-breadcrumbs";
|
||||
import { handleApiError, handleApiResponse } from "@/lib/server-actions-helper";
|
||||
import { SCAN_STATES } from "@/types/attack-paths";
|
||||
import { PARTIAL_SCAN_MAX_RESOURCES } from "@/types/partial-scans";
|
||||
|
||||
const ORGANIZATION_SCAN_CONCURRENCY_LIMIT = 5;
|
||||
|
||||
@@ -182,6 +187,70 @@ export const scanOnDemand = async (formData: FormData) => {
|
||||
}
|
||||
};
|
||||
|
||||
/** Prowler Cloud only: re-check up to PARTIAL_SCAN_MAX_RESOURCES resources of one provider. */
|
||||
export const createPartialScan = async ({
|
||||
providerId,
|
||||
resourceUids,
|
||||
}: {
|
||||
providerId: string;
|
||||
resourceUids: string[];
|
||||
}) => {
|
||||
if (!providerId) {
|
||||
return { error: "Provider ID is required" };
|
||||
}
|
||||
if (
|
||||
resourceUids.length === 0 ||
|
||||
resourceUids.length > PARTIAL_SCAN_MAX_RESOURCES
|
||||
) {
|
||||
return {
|
||||
error: `Select between 1 and ${PARTIAL_SCAN_MAX_RESOURCES} resources to re-check`,
|
||||
};
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: true });
|
||||
|
||||
addScanOperation("create", undefined, {
|
||||
provider_id: providerId,
|
||||
partial: true,
|
||||
resource_count: resourceUids.length,
|
||||
});
|
||||
|
||||
const url = new URL(`${apiBaseUrl}/scans`);
|
||||
|
||||
try {
|
||||
const requestBody = {
|
||||
data: {
|
||||
type: "scans",
|
||||
attributes: { resource_uids: resourceUids },
|
||||
relationships: {
|
||||
provider: {
|
||||
data: {
|
||||
type: "providers",
|
||||
id: providerId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
const result = await handleApiResponse(response, "/scans");
|
||||
if (result?.data?.id) {
|
||||
addScanOperation("start", result.data.id);
|
||||
revalidatePath("/scans");
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
addScanOperation("create");
|
||||
return handleApiError(error);
|
||||
}
|
||||
};
|
||||
|
||||
export const scheduleDaily = async (formData: FormData) => {
|
||||
const headers = await getAuthHeaders({ contentType: true });
|
||||
|
||||
@@ -377,6 +446,10 @@ export const updateScan = async (formData: FormData) => {
|
||||
};
|
||||
|
||||
export const getExportsZip = async (scanId: string) => {
|
||||
if (await isReportDownloadLocked()) {
|
||||
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
|
||||
const url = new URL(`${apiBaseUrl}/scans/${scanId}/report`);
|
||||
@@ -457,6 +530,10 @@ const _fetchScanBinary = async (
|
||||
filename: string,
|
||||
errorLabel: string,
|
||||
): Promise<ScanBinaryResult> => {
|
||||
if (await isReportDownloadLocked()) {
|
||||
return { error: REPORT_DOWNLOAD_LOCKED_ERROR };
|
||||
}
|
||||
|
||||
const headers = await getAuthHeaders({ contentType: false });
|
||||
const url = new URL(`${apiBaseUrl}/scans/${scanId}/${urlPath}`);
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
} from "@/lib/compliance/compliance-report-types";
|
||||
import { LIGHTHOUSE_COMPLIANCE_CONTEXT_MODE } from "@/lib/lighthouse/context/constants";
|
||||
import { buildComplianceContext } from "@/lib/lighthouse/context/contributions";
|
||||
import { isReportDownloadLocked } from "@/lib/report-download-access";
|
||||
import { isCloud } from "@/lib/shared/env";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { SearchParamsProps } from "@/types";
|
||||
@@ -77,6 +78,8 @@ export default async function ComplianceDetail({
|
||||
notFound();
|
||||
}
|
||||
|
||||
const subscriptionOnlyPromise = isReportDownloadLocked();
|
||||
|
||||
// Cross-provider mode replaces the per-scan pipeline with the universal
|
||||
// roll-up view. Prowler Cloud-only: the OSS API has no such endpoint, so
|
||||
// the route is blocked in OSS the same way the compliance tab is.
|
||||
@@ -85,6 +88,7 @@ export default async function ComplianceDetail({
|
||||
redirect("/compliance");
|
||||
}
|
||||
|
||||
const subscriptionOnly = await subscriptionOnlyPromise;
|
||||
return (
|
||||
<Suspense
|
||||
key={buildSearchParamsKey(resolvedSearchParams)}
|
||||
@@ -105,6 +109,7 @@ export default async function ComplianceDetail({
|
||||
complianceId={complianceId}
|
||||
searchParams={resolvedSearchParams}
|
||||
targetSection={section}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</Suspense>
|
||||
);
|
||||
@@ -124,6 +129,7 @@ export default async function ComplianceDetail({
|
||||
}
|
||||
|
||||
const crossAccountTitle = compliancetitle.split("-").join(" ");
|
||||
const subscriptionOnly = await subscriptionOnlyPromise;
|
||||
return (
|
||||
<ContentLayout
|
||||
title={
|
||||
@@ -148,6 +154,7 @@ export default async function ComplianceDetail({
|
||||
providerType={providerType}
|
||||
searchParams={resolvedSearchParams}
|
||||
targetSection={section}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</Suspense>
|
||||
</ContentLayout>
|
||||
@@ -172,18 +179,23 @@ export default async function ComplianceDetail({
|
||||
let selectedScan: ScanEntity | null = null;
|
||||
const selectedScanId = scanId || null;
|
||||
|
||||
const [metadataInfoData, attributesData, selectedScanResponse] =
|
||||
await Promise.all([
|
||||
getComplianceOverviewMetadataInfo({
|
||||
filters: {
|
||||
"filter[scan_id]": selectedScanId ?? undefined,
|
||||
},
|
||||
}),
|
||||
getComplianceAttributes(complianceId, selectedScanId ?? undefined),
|
||||
selectedScanId
|
||||
? getScan(selectedScanId, { include: "provider" })
|
||||
: Promise.resolve(null),
|
||||
]);
|
||||
const [
|
||||
metadataInfoData,
|
||||
attributesData,
|
||||
selectedScanResponse,
|
||||
subscriptionOnly,
|
||||
] = await Promise.all([
|
||||
getComplianceOverviewMetadataInfo({
|
||||
filters: {
|
||||
"filter[scan_id]": selectedScanId ?? undefined,
|
||||
},
|
||||
}),
|
||||
getComplianceAttributes(complianceId, selectedScanId ?? undefined),
|
||||
selectedScanId
|
||||
? getScan(selectedScanId, { include: "provider" })
|
||||
: Promise.resolve(null),
|
||||
subscriptionOnlyPromise,
|
||||
]);
|
||||
|
||||
// The compliance catalog is still warming after a deploy/restart. Show the
|
||||
// "still loading" state with a Try Again instead of rendering an empty page.
|
||||
@@ -309,6 +321,7 @@ export default async function ComplianceDetail({
|
||||
complianceId,
|
||||
latestCisIds.has(complianceId),
|
||||
)}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -5,11 +5,13 @@ const {
|
||||
getAuthHeadersMock,
|
||||
handleApiResponseMock,
|
||||
captureExceptionMock,
|
||||
isReportDownloadLockedMock,
|
||||
} = vi.hoisted(() => ({
|
||||
fetchMock: vi.fn(),
|
||||
getAuthHeadersMock: vi.fn(),
|
||||
handleApiResponseMock: vi.fn(),
|
||||
captureExceptionMock: vi.fn(),
|
||||
isReportDownloadLockedMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib", () => ({
|
||||
@@ -28,6 +30,12 @@ vi.mock("@sentry/nextjs", () => ({
|
||||
captureException: captureExceptionMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/report-download-access", () => ({
|
||||
REPORT_DOWNLOAD_LOCKED_ERROR:
|
||||
"Report downloads require an active subscription.",
|
||||
isReportDownloadLocked: isReportDownloadLockedMock,
|
||||
}));
|
||||
|
||||
import {
|
||||
generateCrossProviderPdf,
|
||||
getCrossProviderComplianceOverview,
|
||||
@@ -56,6 +64,29 @@ beforeEach(() => {
|
||||
Authorization: "Bearer test-token",
|
||||
});
|
||||
handleApiResponseMock.mockResolvedValue({ data: null });
|
||||
isReportDownloadLockedMock.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
describe("cross-provider PDF reports for subscription-only tenants", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "generation",
|
||||
run: () => generateCrossProviderPdf({ complianceId: "csa_ccm_4.0" }),
|
||||
},
|
||||
{ name: "download", run: () => getCrossProviderPdfBinary("task-1") },
|
||||
])("rejects the $name without calling the API", async ({ run }) => {
|
||||
// Given
|
||||
isReportDownloadLockedMock.mockResolvedValue(true);
|
||||
|
||||
// When
|
||||
const result = await run();
|
||||
|
||||
// Then
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
error: "Report downloads require an active subscription.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("getCrossProviderComplianceOverview", () => {
|
||||
|
||||
@@ -48,6 +48,7 @@ interface CrossAccountDetailProps {
|
||||
providerType: KnownProviderType;
|
||||
searchParams: Record<string, string | string[] | undefined>;
|
||||
targetSection?: string;
|
||||
subscriptionOnly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -63,6 +64,7 @@ export const CrossAccountDetail = async ({
|
||||
providerType,
|
||||
searchParams,
|
||||
targetSection,
|
||||
subscriptionOnly = false,
|
||||
}: CrossAccountDetailProps) => {
|
||||
const filters = parseCrossAccountFilters(searchParams);
|
||||
|
||||
@@ -205,6 +207,7 @@ export const CrossAccountDetail = async ({
|
||||
providerType={providerType}
|
||||
filters={{ ...filters, scanIds: attrs.scan_ids }}
|
||||
latestPdf={latestPdf}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
}
|
||||
filters={
|
||||
|
||||
@@ -44,6 +44,7 @@ interface CrossProviderDetailProps {
|
||||
complianceId: string;
|
||||
searchParams: Record<string, string | string[] | undefined>;
|
||||
targetSection?: string;
|
||||
subscriptionOnly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -57,6 +58,7 @@ export const CrossProviderDetail = async ({
|
||||
complianceId,
|
||||
searchParams,
|
||||
targetSection,
|
||||
subscriptionOnly = false,
|
||||
}: CrossProviderDetailProps) => {
|
||||
const filters = parseCrossProviderFilters(searchParams);
|
||||
|
||||
@@ -206,6 +208,7 @@ export const CrossProviderDetail = async ({
|
||||
complianceId={complianceId}
|
||||
filters={{ ...filters, scanIds: attrs.scan_ids }}
|
||||
latestPdf={latestPdf}
|
||||
subscriptionOnly={subscriptionOnly}
|
||||
/>
|
||||
}
|
||||
filters={
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user