diff --git a/.env b/.env index 8294fac91c..3a666b2e9c 100644 --- a/.env +++ b/.env @@ -72,8 +72,8 @@ NEO4J_APOC_IMPORT_FILE_ENABLED=false NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true NEO4J_APOC_TRIGGER_ENABLED=false NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687 -# Neo4j Prowler settings -ATTACK_PATHS_BATCH_SIZE=1000 +# Attack Paths graph settings +ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE=1000 ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES=3 ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS=30 ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES=250 @@ -158,7 +158,7 @@ SENTRY_RELEASE=local # REO_DEV_CLIENT_ID= #### Prowler release version #### -NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.35.0 +NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.36.0 # Social login credentials SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google" diff --git a/AGENTS.md b/AGENTS.md index 763b3f10e5..997c4bbaff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST: | Action | Skill | |--------|-------| | Add changelog entry for a PR or feature | `prowler-changelog` | +| Adding ConfigRequirements guardrails to compliance requirements | `prowler-compliance` | | Adding DRF pagination or permissions | `django-drf` | | Adding a compliance output formatter (per-provider class + table dispatcher) | `prowler-compliance` | | Adding indexes or constraints to database tables | `django-migration-psql` | @@ -84,6 +85,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST: | Creating ViewSets, serializers, or filters in api/ | `django-drf` | | Creating Zod schemas | `zod-4` | | Creating a git commit | `prowler-commit` | +| Creating a universal (multi-provider) compliance framework | `prowler-compliance` | | Creating new checks | `prowler-sdk-check` | | Creating new skills | `skill-creator` | | Creating or reviewing Django migrations | `django-migration-psql` | diff --git a/README.md b/README.md index c58d903c08..76e6534551 100644 --- a/README.md +++ b/README.md @@ -123,12 +123,12 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically | Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/user-guide/compliance/tutorials/compliance) | [Categories](https://docs.prowler.com/user-guide/cli/tutorials/misc#categories) | Support | Interface | |---|---|---|---|---|---|---| -| AWS | 615 | 86 | 47 | 19 | Official | UI, API, CLI | -| Azure | 190 | 22 | 21 | 16 | Official | UI, API, CLI | +| AWS | 621 | 86 | 47 | 19 | Official | UI, API, CLI | +| Azure | 191 | 22 | 21 | 16 | Official | UI, API, CLI | | GCP | 109 | 20 | 19 | 12 | Official | UI, API, CLI | -| Kubernetes | 90 | 7 | 8 | 11 | Official | UI, API, CLI | +| Kubernetes | 92 | 7 | 8 | 11 | Official | UI, API, CLI | | GitHub | 24 | 3 | 2 | 5 | Official | UI, API, CLI | -| M365 | 109 | 10 | 6 | 10 | Official | UI, API, CLI | +| M365 | 111 | 10 | 6 | 10 | Official | UI, API, CLI | | OCI | 52 | 14 | 5 | 10 | Official | UI, API, CLI | | Alibaba Cloud | 63 | 9 | 6 | 9 | Official | UI, API, CLI | | Cloudflare | 29 | 3 | 2 | 5 | Official | UI, API, CLI | diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index a311614ca9..1b26c2b014 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to the **Prowler API** are documented in this file. +## [1.36.0] (Prowler v5.35.0) + +### 🐞 Fixed + +- `attack-paths-scan-perform` Celery tasks now use the configurable long-task time limits instead of the six-hour defaults [(#12009)](https://github.com/prowler-cloud/prowler/pull/12009) +- Attack Paths scans handle provider deletion races cleanly, detect stale tasks after 16 hours, use backend-specific graph synchronization batches, and report exhausted Neptune write retries with the original database error [(#12019)](https://github.com/prowler-cloud/prowler/pull/12019) + +### 🔐 Security + +- Jira integration credentials only accept bare Atlassian site names containing letters, numbers, and hyphens [(#12012)](https://github.com/prowler-cloud/prowler/pull/12012) +- Social account linking requires a verified matching email from both the identity provider and the existing user account without sending account connection notifications [(#12013)](https://github.com/prowler-cloud/prowler/pull/12013) + +--- + ## [1.35.0] (Prowler v5.34.0) ### 🐞 Fixed diff --git a/api/changelog.d/attack-paths-scan-time-limit.fixed.md b/api/changelog.d/attack-paths-scan-time-limit.fixed.md deleted file mode 100644 index 3726b52668..0000000000 --- a/api/changelog.d/attack-paths-scan-time-limit.fixed.md +++ /dev/null @@ -1 +0,0 @@ -`attack-paths-scan-perform` Celery tasks now use the configurable long-task time limits instead of the six-hour defaults diff --git a/api/changelog.d/compliance-overview-single-transaction.changed.md b/api/changelog.d/compliance-overview-single-transaction.changed.md new file mode 100644 index 0000000000..0e7a6714b3 --- /dev/null +++ b/api/changelog.d/compliance-overview-single-transaction.changed.md @@ -0,0 +1 @@ +Compliance overview ingest now runs in a single transaction per scan with a configurable `COPY` batch size (`DJANGO_COMPLIANCE_COPY_BATCH_SIZE`, default 2000), reducing write pressure on the database diff --git a/api/changelog.d/jira-site-name-validation.security.md b/api/changelog.d/jira-site-name-validation.security.md deleted file mode 100644 index d06244297a..0000000000 --- a/api/changelog.d/jira-site-name-validation.security.md +++ /dev/null @@ -1 +0,0 @@ -Jira integration credentials only accept bare Atlassian site names containing letters, numbers, and hyphens diff --git a/api/changelog.d/oci-regionless-api-legacy-region.changed.md b/api/changelog.d/oci-regionless-api-legacy-region.changed.md new file mode 100644 index 0000000000..087b027c88 --- /dev/null +++ b/api/changelog.d/oci-regionless-api-legacy-region.changed.md @@ -0,0 +1 @@ +OCI provider secrets no longer require `region`; legacy `region` input is accepted for backwards compatibility but ignored before storing or scanning diff --git a/api/changelog.d/social-account-linking.security.md b/api/changelog.d/social-account-linking.security.md deleted file mode 100644 index ab468dbefb..0000000000 --- a/api/changelog.d/social-account-linking.security.md +++ /dev/null @@ -1 +0,0 @@ -Social account linking now requires a verified matching email from both the identity provider and the existing user account diff --git a/api/pyproject.toml b/api/pyproject.toml index 8061632cb8..607a8c7348 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -71,7 +71,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.36.0" +version = "1.37.0" # Shared ruff baseline (kept in sync with mcp_server/pyproject.toml). # target-version tracks this project's lowest supported Python. diff --git a/api/src/backend/api/adapters.py b/api/src/backend/api/adapters.py index d8c5b1b386..99ab88bf81 100644 --- a/api/src/backend/api/adapters.py +++ b/api/src/backend/api/adapters.py @@ -1,5 +1,3 @@ -import logging - from allauth.account.models import EmailAddress from allauth.core.exceptions import ImmediateHttpResponse from allauth.socialaccount.adapter import DefaultSocialAccountAdapter @@ -17,8 +15,6 @@ from api.utils import accept_invitation_for_user from django.db import transaction from django.http import HttpResponseForbidden -logger = logging.getLogger(__name__) - class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter): @staticmethod @@ -105,12 +101,6 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter): raise ImmediateHttpResponse(HttpResponseForbidden()) sociallogin.connect(request, existing_user) - def send_notification_mail(self, *args, **kwargs): - try: - return super().send_notification_mail(*args, **kwargs) - except OSError: - logger.exception("Failed to send social account connection notification") - def save_user(self, request, sociallogin, form=None): """ Called after the user data is fully populated from the provider diff --git a/api/src/backend/api/attack_paths/database.py b/api/src/backend/api/attack_paths/database.py index 3a33b964b7..3ef55b7eca 100644 --- a/api/src/backend/api/attack_paths/database.py +++ b/api/src/backend/api/attack_paths/database.py @@ -27,6 +27,7 @@ from django.conf import ( MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250) TEMP_DB_PREFIX = "db-tmp-scan-" +DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound" # Exceptions @@ -44,6 +45,10 @@ class GraphDatabaseQueryException(Exception): return self.message +class NeptuneWriteRetryExhaustedException(GraphDatabaseQueryException): + pass + + class WriteQueryNotAllowedException(GraphDatabaseQueryException): pass diff --git a/api/src/backend/api/attack_paths/retryable_session.py b/api/src/backend/api/attack_paths/retryable_session.py index 2cd32f54ee..e7e78e9798 100644 --- a/api/src/backend/api/attack_paths/retryable_session.py +++ b/api/src/backend/api/attack_paths/retryable_session.py @@ -10,6 +10,28 @@ import neo4j.exceptions logger = logging.getLogger(__name__) +class RetryExhaustedError(Exception): + def __init__( + self, + *, + retry_context: str, + method_name: str, + attempts: int, + elapsed_seconds: float, + last_error: Exception, + ) -> None: + self.retry_context = retry_context + self.method_name = method_name + self.attempts = attempts + self.elapsed_seconds = elapsed_seconds + self.last_error = last_error + last_message = getattr(last_error, "message", None) or str(last_error) + super().__init__( + f"{retry_context} {method_name} failed after {attempts} attempts over " + f"{elapsed_seconds:.3f}s. Last error: {last_message}" + ) + + class RetryableSession: """Wrapper around ``neo4j.Session`` with a refreshable retry policy.""" @@ -19,11 +41,13 @@ class RetryableSession: max_retries: int, retry_if: Callable[[Exception], bool] | None = None, initial_retry_delay_seconds: float = 0, + retry_context: str | None = None, ) -> None: self._session_factory = session_factory self._max_retries = max(0, max_retries) self._retry_if = retry_if self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds) + self._retry_context = retry_context self._session = self._session_factory() def close(self) -> None: @@ -54,6 +78,7 @@ class RetryableSession: def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any: attempt = 0 last_exc: Exception | None = None + started_at = time.monotonic() while attempt <= self._max_retries: try: @@ -68,17 +93,38 @@ class RetryableSession: attempt += 1 if attempt > self._max_retries: + if self._retry_context is not None: + raise RetryExhaustedError( + retry_context=self._retry_context, + method_name=method_name, + attempts=attempt, + elapsed_seconds=time.monotonic() - started_at, + last_error=exc, + ) from exc raise delay = self._retry_delay(attempt) - logger.warning( - "Graph session %s failed with %s; retry %s/%s in %.3fs", - method_name, - type(exc).__name__, - attempt, - self._max_retries, - delay, - ) + if self._retry_context is not None: + error_message = getattr(exc, "message", None) or str(exc) + logger.warning( + "%s %s failed with %s: %s; retry %s/%s in %.3fs", + self._retry_context, + method_name, + type(exc).__name__, + error_message, + attempt, + self._max_retries, + delay, + ) + else: + logger.warning( + "Graph session %s failed with %s; retry %s/%s in %.3fs", + method_name, + type(exc).__name__, + attempt, + self._max_retries, + delay, + ) self._refresh_session() if delay: time.sleep(delay) diff --git a/api/src/backend/api/attack_paths/sink/base.py b/api/src/backend/api/attack_paths/sink/base.py index 0ba4737f5e..0134e0a41f 100644 --- a/api/src/backend/api/attack_paths/sink/base.py +++ b/api/src/backend/api/attack_paths/sink/base.py @@ -15,6 +15,8 @@ class SinkDatabase(Protocol): has a single graph, and isolation is label-based). """ + sync_batch_size: int + def init(self) -> None: ... def close(self) -> None: ... diff --git a/api/src/backend/api/attack_paths/sink/neo4j.py b/api/src/backend/api/attack_paths/sink/neo4j.py index cb7d4889b0..c820ed9d93 100644 --- a/api/src/backend/api/attack_paths/sink/neo4j.py +++ b/api/src/backend/api/attack_paths/sink/neo4j.py @@ -54,6 +54,8 @@ DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound" class Neo4jSink(SinkDatabase): """Neo4j-backed sink. Multi-database cluster; tenant isolation is physical.""" + sync_batch_size = env.int("ATTACK_PATHS_NEO4J_SYNC_BATCH_SIZE", default=1000) + def __init__(self) -> None: self._driver: neo4j.Driver | None = None self._lock = threading.Lock() @@ -203,7 +205,7 @@ class Neo4jSink(SinkDatabase): """ from api.attack_paths.database import GraphDatabaseQueryException from tasks.jobs.attack_paths.config import ( - BATCH_SIZE, + GRAPH_MUTATION_BATCH_SIZE, PROVIDER_RESOURCE_LABEL, get_provider_label, ) @@ -251,7 +253,7 @@ class Neo4jSink(SinkDatabase): total_key="rels", deleted_key="deleted_rels", initial_total=deleted_relationships, - batch_size=BATCH_SIZE, + batch_size=GRAPH_MUTATION_BATCH_SIZE, drop_t0=drop_t0, ) relationship_batches += phase_batches @@ -270,7 +272,7 @@ class Neo4jSink(SinkDatabase): total_key="nodes", deleted_key="deleted_nodes", initial_total=0, - batch_size=BATCH_SIZE, + batch_size=GRAPH_MUTATION_BATCH_SIZE, drop_t0=drop_t0, ) diff --git a/api/src/backend/api/attack_paths/sink/neptune.py b/api/src/backend/api/attack_paths/sink/neptune.py index c68b6f8277..022e3c8669 100644 --- a/api/src/backend/api/attack_paths/sink/neptune.py +++ b/api/src/backend/api/attack_paths/sink/neptune.py @@ -25,7 +25,7 @@ from urllib.parse import urlsplit import neo4j import neo4j.exceptions -from api.attack_paths.retryable_session import RetryableSession +from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError from api.attack_paths.sink.base import SinkDatabase from api.attack_paths.sink.drop import ( NODE_DELETE_QUERY_TEMPLATE, @@ -85,6 +85,8 @@ def _is_retryable_write_error(exc: Exception) -> bool: class NeptuneSink(SinkDatabase): """Neptune-backed sink. Single database; isolation is label-based.""" + sync_batch_size = env.int("ATTACK_PATHS_NEPTUNE_SYNC_BATCH_SIZE", default=500) + def __init__(self) -> None: self._writer: neo4j.Driver | None = None self._reader: neo4j.Driver | None = None @@ -206,6 +208,7 @@ class NeptuneSink(SinkDatabase): from api.attack_paths.database import ( ClientStatementException, GraphDatabaseQueryException, + NeptuneWriteRetryExhaustedException, WriteQueryNotAllowedException, ) @@ -227,9 +230,17 @@ class NeptuneSink(SinkDatabase): initial_retry_delay_seconds=( NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0 ), + retry_context="Neptune write" if is_write_session else None, ) yield session_wrapper + except RetryExhaustedError as exc: + last_error = exc.last_error + raise NeptuneWriteRetryExhaustedException( + message=str(exc), + code=getattr(last_error, "code", None), + ) from last_error + except neo4j.exceptions.Neo4jError as exc: if ( default_access_mode == neo4j.READ_ACCESS @@ -291,7 +302,7 @@ class NeptuneSink(SinkDatabase): graph's branching factor. """ from tasks.jobs.attack_paths.config import ( - BATCH_SIZE, + GRAPH_MUTATION_BATCH_SIZE, PROVIDER_RESOURCE_LABEL, get_provider_label, ) @@ -330,7 +341,7 @@ class NeptuneSink(SinkDatabase): total_key="rels", deleted_key="deleted_rels", initial_total=deleted_relationships, - batch_size=BATCH_SIZE, + batch_size=GRAPH_MUTATION_BATCH_SIZE, drop_t0=drop_t0, ) relationship_batches += phase_batches @@ -349,7 +360,7 @@ class NeptuneSink(SinkDatabase): total_key="nodes", deleted_key="deleted_nodes", initial_total=0, - batch_size=BATCH_SIZE, + batch_size=GRAPH_MUTATION_BATCH_SIZE, drop_t0=drop_t0, ) diff --git a/api/src/backend/api/decorators.py b/api/src/backend/api/decorators.py index a055b2252f..2dd2d5fea4 100644 --- a/api/src/backend/api/decorators.py +++ b/api/src/backend/api/decorators.py @@ -1,12 +1,13 @@ import uuid from functools import wraps +from api.attack_paths.database import GraphDatabaseQueryException from api.db_router import READ_REPLICA_ALIAS from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction from api.exceptions import ProviderDeletedException -from api.models import Provider, Scan +from api.models import Membership, Provider, Scan, Tenant from django.core.exceptions import ObjectDoesNotExist -from django.db import DatabaseError, connection, transaction +from django.db import DEFAULT_DB_ALIAS, DatabaseError, connection, transaction from rest_framework_json_api.serializers import ValidationError @@ -75,9 +76,11 @@ def handle_provider_deletion(func): """ Decorator that raises `ProviderDeletedException` if provider was deleted during execution. - Catches `ObjectDoesNotExist` and `DatabaseError` (including `IntegrityError`), checks if - provider still exists, and raises `ProviderDeletedException` if not. Otherwise, - re-raises original exception. + Catches `ObjectDoesNotExist`, `DatabaseError` (including `IntegrityError`), and + `GraphDatabaseQueryException`, checks if provider still exists, and raises + `ProviderDeletedException` if not. Graph database errors also check whether the + tenant still exists and has memberships. Otherwise, re-raises the original + exception. Requires `tenant_id` and `provider_id` in kwargs. @@ -92,11 +95,16 @@ def handle_provider_deletion(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except (ObjectDoesNotExist, DatabaseError): + except (ObjectDoesNotExist, DatabaseError, GraphDatabaseQueryException) as exc: tenant_id = kwargs.get("tenant_id") provider_id = kwargs.get("provider_id") + database_alias = ( + DEFAULT_DB_ALIAS + if isinstance(exc, GraphDatabaseQueryException) + else READ_REPLICA_ALIAS + ) - with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + with rls_transaction(tenant_id, using=database_alias): if provider_id is None: scan_id = kwargs.get("scan_id") if scan_id is None: @@ -113,6 +121,13 @@ def handle_provider_deletion(func): raise ProviderDeletedException( f"Provider '{provider_id}' was deleted during the scan" ) from None + if isinstance(exc, GraphDatabaseQueryException) and ( + not Tenant.objects.filter(pk=tenant_id).exists() + or not Membership.objects.filter(tenant_id=tenant_id).exists() + ): + raise ProviderDeletedException( + f"Tenant '{tenant_id}' was deleted during the scan" + ) from None raise return wrapper diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index de2cbfce2f..420ebb27cf 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.36.0 + version: 1.37.0 description: |- Prowler API specification. diff --git a/api/src/backend/api/tests/test_adapters.py b/api/src/backend/api/tests/test_adapters.py index 70cd055205..c86d3f5620 100644 --- a/api/src/backend/api/tests/test_adapters.py +++ b/api/src/backend/api/tests/test_adapters.py @@ -302,7 +302,9 @@ class TestProwlerSocialAccountAdapter: sociallogin.connect.assert_called_once_with(request, create_test_user) - def test_verified_social_account_link_notifies_owner(self, create_test_user, rf): + def test_verified_social_account_link_does_not_send_notification( + self, create_test_user, rf + ): _verify_local_email(create_test_user) sociallogin = _real_oauth_sociallogin( create_test_user, @@ -318,34 +320,7 @@ class TestProwlerSocialAccountAdapter: uid="verified-google-account", user=create_test_user, ).exists() - assert len(mail.outbox) == 1 - assert mail.outbox[0].to == [create_test_user.email] - - def test_notification_delivery_failure_does_not_break_verified_link( - self, create_test_user, rf - ): - _verify_local_email(create_test_user) - sociallogin = _real_oauth_sociallogin( - create_test_user, - uid="verified-google-account-without-smtp", - ) - request = rf.get("/") - - with ( - context.request_context(request), - patch( - "allauth.socialaccount.adapter." - "DefaultSocialAccountAdapter.send_notification_mail", - side_effect=ConnectionRefusedError, - ), - ): - ProwlerSocialAccountAdapter().pre_social_login(request, sociallogin) - - assert SocialAccount.objects.filter( - provider="google", - uid="verified-google-account-without-smtp", - user=create_test_user, - ).exists() + assert mail.outbox == [] def test_pre_social_login_uses_verified_email_missing_from_extra_data( self, create_test_user, rf @@ -367,7 +342,7 @@ class TestProwlerSocialAccountAdapter: def test_social_account_linking_settings_are_fail_closed(self): assert not socialaccount_app_settings.EMAIL_AUTHENTICATION assert not socialaccount_app_settings.EMAIL_AUTHENTICATION_AUTO_CONNECT - assert account_app_settings.EMAIL_NOTIFICATIONS + assert not account_app_settings.EMAIL_NOTIFICATIONS def test_save_user_social_with_invitation_joins_invited_tenant( self, rf, create_test_user, tenants_fixture diff --git a/api/src/backend/api/tests/test_decorators.py b/api/src/backend/api/tests/test_decorators.py index 5c2897730f..0bf58340a1 100644 --- a/api/src/backend/api/tests/test_decorators.py +++ b/api/src/backend/api/tests/test_decorators.py @@ -2,11 +2,12 @@ import uuid from unittest.mock import call, patch import pytest +from api.attack_paths.database import GraphDatabaseQueryException from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY from api.decorators import handle_provider_deletion, set_tenant from api.exceptions import ProviderDeletedException from django.core.exceptions import ObjectDoesNotExist -from django.db import DatabaseError, IntegrityError +from django.db import DEFAULT_DB_ALIAS, DatabaseError, IntegrityError @pytest.mark.django_db @@ -204,6 +205,106 @@ class TestHandleProviderDeletionDecorator: with pytest.raises(DatabaseError): task_func(tenant_id=str(tenant.id), provider_id=str(provider.id)) + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Provider.objects.filter") + def test_graph_database_error_provider_missing_or_soft_deleted( + self, mock_provider_filter, mock_rls, tenants_fixture + ): + tenant = tenants_fixture[0] + provider_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_provider_filter.return_value.exists.return_value = False + + @handle_provider_deletion + def task_func(**kwargs): + raise GraphDatabaseQueryException("Temporary database not found") + + with pytest.raises(ProviderDeletedException): + task_func(tenant_id=str(tenant.id), provider_id=provider_id) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Tenant.objects.filter") + @patch("api.decorators.Provider.objects.filter") + def test_graph_database_error_tenant_missing( + self, mock_provider_filter, mock_tenant_filter, mock_rls, tenants_fixture + ): + tenant = tenants_fixture[0] + provider_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_provider_filter.return_value.exists.return_value = True + mock_tenant_filter.return_value.exists.return_value = False + + @handle_provider_deletion + def task_func(**kwargs): + raise GraphDatabaseQueryException("Temporary database not found") + + with pytest.raises(ProviderDeletedException): + task_func(tenant_id=str(tenant.id), provider_id=provider_id) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Membership.objects.filter") + @patch("api.decorators.Tenant.objects.filter") + @patch("api.decorators.Provider.objects.filter") + def test_graph_database_error_tenant_without_memberships( + self, + mock_provider_filter, + mock_tenant_filter, + mock_membership_filter, + mock_rls, + tenants_fixture, + ): + tenant = tenants_fixture[0] + provider_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_provider_filter.return_value.exists.return_value = True + mock_tenant_filter.return_value.exists.return_value = True + mock_membership_filter.return_value.exists.return_value = False + + @handle_provider_deletion + def task_func(**kwargs): + raise GraphDatabaseQueryException("Temporary database not found") + + with pytest.raises(ProviderDeletedException): + task_func(tenant_id=str(tenant.id), provider_id=provider_id) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Membership.objects.filter") + @patch("api.decorators.Tenant.objects.filter") + @patch("api.decorators.Provider.objects.filter") + def test_graph_database_error_active_provider_and_tenant_reraises( + self, + mock_provider_filter, + mock_tenant_filter, + mock_membership_filter, + mock_rls, + tenants_fixture, + ): + tenant = tenants_fixture[0] + provider_id = str(uuid.uuid4()) + graph_error = GraphDatabaseQueryException("Temporary database not found") + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_provider_filter.return_value.exists.return_value = True + mock_tenant_filter.return_value.exists.return_value = True + mock_membership_filter.return_value.exists.return_value = True + + @handle_provider_deletion + def task_func(**kwargs): + raise graph_error + + with pytest.raises(GraphDatabaseQueryException) as exc_info: + task_func(tenant_id=str(tenant.id), provider_id=provider_id) + + assert exc_info.value is graph_error + mock_rls.assert_called_once_with(str(tenant.id), using=DEFAULT_DB_ALIAS) + def test_missing_provider_and_scan_raises_assertion(self, tenants_fixture): """Raises AssertionError when neither provider_id nor scan_id in kwargs.""" diff --git a/api/src/backend/api/tests/test_retryable_session.py b/api/src/backend/api/tests/test_retryable_session.py index 8bb457b8b6..09b184c223 100644 --- a/api/src/backend/api/tests/test_retryable_session.py +++ b/api/src/backend/api/tests/test_retryable_session.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, patch import pytest -from api.attack_paths.retryable_session import RetryableSession +from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError from neo4j.exceptions import ServiceUnavailable @@ -24,6 +24,7 @@ class TestRetryableSession: max_retries=3, retry_if=lambda exc: exc is retryable_error, initial_retry_delay_seconds=2, + retry_context="Neptune write", ) assert session.execute_write(work) == "success" @@ -54,6 +55,7 @@ class TestRetryableSession: max_retries=3, retry_if=lambda _: False, initial_retry_delay_seconds=2, + retry_context="Neptune write", ) with pytest.raises(RuntimeError) as exc_info: @@ -83,3 +85,81 @@ class TestRetryableSession: driver_sessions[0].close.assert_called_once_with() driver_sessions[1].close.assert_called_once_with() driver_sessions[2].close.assert_not_called() + + def test_retry_exhaustion_with_context_reports_attempts_and_elapsed_time(self): + error = RuntimeError("still retryable") + driver_sessions = [MagicMock() for _ in range(3)] + for driver_session in driver_sessions: + driver_session.execute_write.side_effect = error + session = RetryableSession( + session_factory=MagicMock(side_effect=driver_sessions), + max_retries=2, + retry_if=lambda _: True, + retry_context="Neptune write", + ) + + with ( + patch( + "api.attack_paths.retryable_session.time.monotonic", + side_effect=[100.0, 127.1234], + ), + pytest.raises(RetryExhaustedError) as exc_info, + ): + session.execute_write(MagicMock()) + + assert exc_info.value.method_name == "execute_write" + assert exc_info.value.attempts == 3 + assert exc_info.value.elapsed_seconds == pytest.approx(27.1234) + assert exc_info.value.last_error is error + assert exc_info.value.__cause__ is error + assert str(exc_info.value) == ( + "Neptune write execute_write failed after 3 attempts over 27.123s. " + "Last error: still retryable" + ) + + def test_retry_exhaustion_with_zero_retries_reports_one_attempt(self): + error = ServiceUnavailable("still unavailable") + driver_session = MagicMock() + driver_session.execute_write.side_effect = error + session = RetryableSession( + session_factory=MagicMock(return_value=driver_session), + max_retries=0, + retry_context="Neptune write", + ) + + with pytest.raises(RetryExhaustedError) as exc_info: + session.execute_write(MagicMock()) + + assert exc_info.value.attempts == 1 + + @patch("api.attack_paths.retryable_session.time.sleep") + @patch("api.attack_paths.retryable_session.random.uniform", return_value=3.0) + def test_contextual_retry_warning_includes_original_error( + self, _mock_uniform, _mock_sleep + ): + error = RuntimeError("retryable detail") + first_session = MagicMock() + first_session.execute_write.side_effect = error + second_session = MagicMock() + second_session.execute_write.return_value = "success" + session = RetryableSession( + session_factory=MagicMock(side_effect=[first_session, second_session]), + max_retries=1, + retry_if=lambda _: True, + initial_retry_delay_seconds=2, + retry_context="Neptune write", + ) + + with patch("api.attack_paths.retryable_session.logger.warning") as mock_warning: + assert session.execute_write(MagicMock()) == "success" + + mock_warning.assert_called_once_with( + "%s %s failed with %s: %s; retry %s/%s in %.3fs", + "Neptune write", + "execute_write", + "RuntimeError", + "retryable detail", + 1, + 1, + 3.0, + ) diff --git a/api/src/backend/api/tests/test_sentry.py b/api/src/backend/api/tests/test_sentry.py index 082f563808..67ba1ee01e 100644 --- a/api/src/backend/api/tests/test_sentry.py +++ b/api/src/backend/api/tests/test_sentry.py @@ -1,6 +1,7 @@ import logging from unittest.mock import MagicMock, patch +import pytest from config.settings import sentry as sentry_settings from config.settings.sentry import before_send @@ -82,6 +83,45 @@ def test_before_send_passes_through_non_ignored_log(): assert result == event +def test_before_send_ignores_cartography_missing_temporary_database_log(): + log_record = _make_log_record( + msg="Cartography job failed with %s for database %s", + name="cartography.graph.job", + args=( + "Neo.ClientError.Database.DatabaseNotFound", + "db-tmp-scan-12345678", + ), + ) + + event = MagicMock() + + assert before_send(event, {"log_record": log_record}) is None + + +@pytest.mark.parametrize( + ("logger_name", "message"), + [ + ( + "cartography.graph.job.worker", + "Neo.ClientError.Database.DatabaseNotFound for db-tmp-scan-12345678", + ), + ( + "cartography.graph.job", + "DatabaseNotFound for db-tmp-scan-12345678", + ), + ( + "cartography.graph.job", + "Neo.ClientError.Database.DatabaseNotFound for db-tenant-12345678", + ), + ], +) +def test_before_send_passes_through_similar_cartography_logs(logger_name, message): + log_record = _make_log_record(msg=message, name=logger_name) + event = MagicMock() + + assert before_send(event, {"log_record": log_record}) is event + + def test_before_send_passes_through_non_ignored_exception(): """Test that before_send passes through exceptions that don't contain ignored exceptions.""" exc_info = (Exception, Exception("Some other error message"), None) diff --git a/api/src/backend/api/tests/test_serializers.py b/api/src/backend/api/tests/test_serializers.py index 0ee674d22c..8e77d63604 100644 --- a/api/src/backend/api/tests/test_serializers.py +++ b/api/src/backend/api/tests/test_serializers.py @@ -3,7 +3,12 @@ from api.v1.serializer_utils.integrations import ( JiraCredentialSerializer, S3ConfigSerializer, ) -from api.v1.serializers import ImageProviderSecret, KubernetesProviderSecret +from api.v1.serializer_utils.providers import ProviderSecretField +from api.v1.serializers import ( + ImageProviderSecret, + KubernetesProviderSecret, + OracleCloudProviderSecret, +) from rest_framework.exceptions import ValidationError @@ -190,6 +195,64 @@ class TestImageProviderSecret: assert "non_field_errors" in serializer.errors +class TestOracleCloudProviderSecret: + def valid_secret(self, **overrides): + secret = { + "user": "ocid1.user.oc1..aaaaaaaexample", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_content": "fake-base64-key-content", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaexample", + } + secret.update(overrides) + return secret + + def test_accepts_regionless_secret(self): + serializer = OracleCloudProviderSecret(data=self.valid_secret()) + + assert serializer.is_valid(), serializer.errors + assert "region" not in serializer.validated_data + + def test_accepts_and_ignores_region_field(self): + secret = self.valid_secret(region="us-phoenix-1") + serializer = OracleCloudProviderSecret(data=secret) + + assert serializer.is_valid(), serializer.errors + + assert "region" not in serializer.validated_data + + @pytest.mark.parametrize( + "legacy_field, legacy_value", + [ + ("region", None), + ("region", ""), + ("region", {"name": "us-ashburn-1"}), + ], + ) + def test_accepts_and_ignores_any_legacy_region_value( + self, legacy_field, legacy_value + ): + serializer = OracleCloudProviderSecret( + data=self.valid_secret(**{legacy_field: legacy_value}) + ) + + assert serializer.is_valid(), serializer.errors + + assert legacy_field not in serializer.validated_data + + +class TestProviderSecretFieldSchema: + def test_oraclecloud_schema_includes_legacy_region_field(self): + schema = ProviderSecretField._spectacular_annotation["field"] + oraclecloud_schema = next( + credential_schema + for credential_schema in schema["oneOf"] + if credential_schema["title"] + == "Oracle Cloud Infrastructure (OCI) API Key Credentials" + ) + + assert oraclecloud_schema["properties"]["region"]["deprecated"] is True + + class TestKubernetesProviderSecret: def test_valid_static_kubeconfig_is_accepted(self): kubeconfig_content = """ diff --git a/api/src/backend/api/tests/test_sink.py b/api/src/backend/api/tests/test_sink.py index 487519923e..c778626b62 100644 --- a/api/src/backend/api/tests/test_sink.py +++ b/api/src/backend/api/tests/test_sink.py @@ -11,7 +11,11 @@ from unittest.mock import MagicMock, patch import neo4j import pytest from api.attack_paths import sink as sink_module -from api.attack_paths.database import GraphDatabaseQueryException +from api.attack_paths.database import ( + GraphDatabaseQueryException, + NeptuneWriteRetryExhaustedException, +) +from api.attack_paths.retryable_session import RetryExhaustedError from api.attack_paths.sink import factory from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink from api.attack_paths.sink.neptune import ( @@ -123,6 +127,14 @@ class TestSinkFactory: assert mock_driver.call_count == 1 +def test_neo4j_sync_batch_size_defaults_to_1000(): + assert Neo4jSink.sync_batch_size == 1000 + + +def test_neptune_sync_batch_size_defaults_to_500(): + assert NeptuneSink.sync_batch_size == 500 + + class TestGetBackendForScan: """``get_backend_for_scan`` routes by the row's recorded sink backend.""" @@ -372,6 +384,7 @@ class TestNeptuneRetryPolicy: assert ( kwargs["initial_retry_delay_seconds"] == NEPTUNE_WRITE_RETRY_DELAY_SECONDS ) + assert kwargs["retry_context"] == "Neptune write" @patch("api.attack_paths.sink.neptune.RetryableSession") def test_reader_session_does_not_enable_write_retry_policy(self, retryable_session): @@ -384,6 +397,48 @@ class TestNeptuneRetryPolicy: kwargs = retryable_session.call_args.kwargs assert kwargs["retry_if"] is None assert kwargs["initial_retry_delay_seconds"] == 0 + assert kwargs["retry_context"] is None + + def test_writer_retry_exhaustion_preserves_neptune_error_details(self): + message = ( + "Unexpected server exception 'Operation failed due to conflicting " + "concurrent operations (please retry), 0 transactions are currently " + "rolling back.'" + ) + error = neo4j.exceptions.Neo4jError._hydrate_neo4j( + code="BoltProtocol.unexpectedException", + message=message, + ) + retry_error = RetryExhaustedError( + retry_context="Neptune write", + method_name="execute_write", + attempts=4, + elapsed_seconds=27.1234, + last_error=error, + ) + sink = NeptuneSink() + driver = MagicMock() + retryable_session = MagicMock() + retryable_session.execute_write.side_effect = retry_error + + with ( + patch.object(sink, "_get_writer", return_value=driver), + patch( + "api.attack_paths.sink.neptune.RetryableSession", + return_value=retryable_session, + ), + pytest.raises(NeptuneWriteRetryExhaustedException) as exc_info, + ): + with sink.get_session() as session: + session.execute_write(MagicMock()) + + assert exc_info.value.code == "BoltProtocol.unexpectedException" + assert str(exc_info.value) == ( + "BoltProtocol.unexpectedException: Neptune write execute_write failed " + "after 4 attempts over 27.123s. Last error: " + f"{message}" + ) + assert exc_info.value.__cause__ is error class TestNeptuneSinkDropSubgraph: diff --git a/api/src/backend/api/tests/test_utils.py b/api/src/backend/api/tests/test_utils.py index 9e96685477..4b5e8e1694 100644 --- a/api/src/backend/api/tests/test_utils.py +++ b/api/src/backend/api/tests/test_utils.py @@ -171,6 +171,53 @@ class TestInitializeProwlerProvider: key="value", mutelist_content={"key": "value"} ) + @patch("api.utils.return_prowler_provider") + def test_initialize_oraclecloud_provider_removes_region_string( + self, mock_return_prowler_provider + ): + provider = MagicMock() + provider.provider = Provider.ProviderChoices.ORACLECLOUD.value + provider.secret.secret = { + "user": "ocid1.user.oc1..fake", + "fingerprint": "00:11:22:33:44:55:66:77", + "key_content": "fake-base64-key-content", + "tenancy": "ocid1.tenancy.oc1..fake", + "region": "us-ashburn-1", + } + mock_return_prowler_provider.return_value = MagicMock() + + initialize_prowler_provider(provider) + + mock_return_prowler_provider.return_value.assert_called_once_with( + user="ocid1.user.oc1..fake", + fingerprint="00:11:22:33:44:55:66:77", + key_content="fake-base64-key-content", + tenancy="ocid1.tenancy.oc1..fake", + ) + + @patch("api.utils.return_prowler_provider") + def test_initialize_oraclecloud_provider_without_region_omits_scan_filter( + self, mock_return_prowler_provider + ): + provider = MagicMock() + provider.provider = Provider.ProviderChoices.ORACLECLOUD.value + provider.secret.secret = { + "user": "ocid1.user.oc1..fake", + "fingerprint": "00:11:22:33:44:55:66:77", + "key_content": "fake-base64-key-content", + "tenancy": "ocid1.tenancy.oc1..fake", + } + mock_return_prowler_provider.return_value = MagicMock() + + initialize_prowler_provider(provider) + + mock_return_prowler_provider.return_value.assert_called_once_with( + user="ocid1.user.oc1..fake", + fingerprint="00:11:22:33:44:55:66:77", + key_content="fake-base64-key-content", + tenancy="ocid1.tenancy.oc1..fake", + ) + class TestProwlerProviderConnectionTest: @patch("api.utils.return_prowler_provider") @@ -185,6 +232,37 @@ class TestProwlerProviderConnectionTest: key="value", provider_id="1234567890", raise_on_exception=False ) + @patch("api.utils.return_prowler_provider") + def test_oraclecloud_connection_test_uses_direct_credentials_without_region( + self, mock_return_prowler_provider + ): + provider = MagicMock() + provider.uid = "ocid1.tenancy.oc1..aaaaaaaexample" + provider.provider = Provider.ProviderChoices.ORACLECLOUD.value + provider.secret.secret = { + "user": "ocid1.user.oc1..aaaaaaaexample", + "fingerprint": "00:11:22:33:44:55:66:77", + "key_content": "fake-base64-key-content", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaexample", + } + mock_return_prowler_provider.return_value = MagicMock() + + prowler_provider_connection_test(provider) + + mock_return_prowler_provider.return_value.test_connection.assert_called_once_with( + user="ocid1.user.oc1..aaaaaaaexample", + fingerprint="00:11:22:33:44:55:66:77", + key_content="fake-base64-key-content", + tenancy="ocid1.tenancy.oc1..aaaaaaaexample", + region=getattr( + OraclecloudProvider, + "_bootstrap_region", + OraclecloudProvider._home_region, + ), + provider_id="ocid1.tenancy.oc1..aaaaaaaexample", + raise_on_exception=False, + ) + @pytest.mark.django_db @patch("api.utils.return_prowler_provider") def test_prowler_provider_connection_test_without_secret( @@ -356,7 +434,7 @@ class TestGetProwlerProviderKwargs: expected_result = {**secret_dict, **expected_extra_kwargs} assert result == expected_result - def test_get_prowler_provider_kwargs_oraclecloud_converts_region_string_to_set( + def test_get_prowler_provider_kwargs_oraclecloud_removes_region( self, ): secret_dict = { @@ -377,8 +455,13 @@ class TestGetProwlerProviderKwargs: result = get_prowler_provider_kwargs(provider) - expected_result = {**secret_dict, "region": {"us-ashburn-1"}} - assert result == expected_result + assert result == { + "user": "ocid1.user.oc1..fake", + "fingerprint": "00:11:22:33:44:55:66:77", + "key_content": "-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----", + "tenancy": "ocid1.tenancy.oc1..fake", + "pass_phrase": "fake-passphrase", + } def test_get_prowler_provider_kwargs_with_mutelist(self): provider_uid = "provider_uid" diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index bfe8e57bc9..96d366e847 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -2917,6 +2917,48 @@ class TestProviderGroupViewSet: @pytest.mark.django_db class TestProviderSecretViewSet: + @staticmethod + def _oraclecloud_secret(**overrides): + secret = { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_content": "test-key-content", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + } + secret.update(overrides) + return secret + + def _create_oraclecloud_secret( + self, + authenticated_client, + oraclecloud_provider, + secret, + name="OCI Secret", + ): + data = { + "data": { + "type": "provider-secrets", + "attributes": { + "name": name, + "secret_type": ProviderSecret.TypeChoices.STATIC, + "secret": secret, + }, + "relationships": { + "provider": { + "data": { + "type": "providers", + "id": str(oraclecloud_provider.id), + } + } + }, + } + } + return authenticated_client.post( + reverse("providersecret-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + def test_provider_secrets_list(self, authenticated_client, provider_secret_fixture): response = authenticated_client.get(reverse("providersecret-list")) assert response.status_code == status.HTTP_200_OK @@ -3076,7 +3118,6 @@ current-context: test-context "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", "key_content": "-----BEGIN RSA PRIVATE KEY-----\ntest-key-content\n-----END RSA PRIVATE KEY-----", "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", - "region": "us-ashburn-1", }, ), # OCI with API key credentials (with key_file) @@ -3088,7 +3129,6 @@ current-context: test-context "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", "key_file": "/path/to/oci_api_key.pem", "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", - "region": "us-ashburn-1", }, ), # OCI with API key credentials (with passphrase) @@ -3100,7 +3140,6 @@ current-context: test-context "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", "key_content": "-----BEGIN RSA PRIVATE KEY-----\ntest-encrypted-key\n-----END RSA PRIVATE KEY-----", "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", - "region": "us-ashburn-1", "pass_phrase": "my-secure-passphrase", }, ), @@ -3258,6 +3297,103 @@ current-context: test-context == data["data"]["relationships"]["provider"]["data"]["id"] ) + def test_provider_secrets_create_oraclecloud_without_region_stores_no_region( + self, + authenticated_client, + oraclecloud_provider, + ): + response = self._create_oraclecloud_secret( + authenticated_client, + oraclecloud_provider, + self._oraclecloud_secret(), + ) + + assert response.status_code == status.HTTP_201_CREATED + provider_secret = ProviderSecret.objects.get() + assert "region" not in provider_secret.secret + + def test_provider_secrets_create_oraclecloud_accepts_and_ignores_region( + self, + authenticated_client, + oraclecloud_provider, + ): + response = self._create_oraclecloud_secret( + authenticated_client, + oraclecloud_provider, + self._oraclecloud_secret( + key_content=" test-key-content ", region=" us-ashburn-1 " + ), + ) + + assert response.status_code == status.HTTP_201_CREATED + provider_secret = ProviderSecret.objects.get() + assert provider_secret.secret["key_content"] == "test-key-content" + assert "region" not in provider_secret.secret + + def test_provider_secrets_update_oraclecloud_without_region_stores_no_region( + self, + authenticated_client, + oraclecloud_provider, + ): + create_response = self._create_oraclecloud_secret( + authenticated_client, + oraclecloud_provider, + self._oraclecloud_secret(), + ) + provider_secret = ProviderSecret.objects.get( + id=create_response.json()["data"]["id"] + ) + data = { + "data": { + "type": "provider-secrets", + "id": str(provider_secret.id), + "attributes": {"secret": self._oraclecloud_secret()}, + } + } + + response = authenticated_client.patch( + reverse("providersecret-detail", kwargs={"pk": provider_secret.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + + assert response.status_code == status.HTTP_200_OK + provider_secret.refresh_from_db() + assert "region" not in provider_secret.secret + + def test_provider_secrets_update_oraclecloud_accepts_and_ignores_region( + self, + authenticated_client, + oraclecloud_provider, + ): + create_response = self._create_oraclecloud_secret( + authenticated_client, + oraclecloud_provider, + self._oraclecloud_secret(), + ) + provider_secret = ProviderSecret.objects.get( + id=create_response.json()["data"]["id"] + ) + data = { + "data": { + "type": "provider-secrets", + "id": str(provider_secret.id), + "attributes": { + "secret": self._oraclecloud_secret(region=" us-ashburn-1 ") + }, + } + } + + response = authenticated_client.patch( + reverse("providersecret-detail", kwargs={"pk": provider_secret.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + + assert response.status_code == status.HTTP_200_OK + provider_secret.refresh_from_db() + assert "region" not in provider_secret.secret + @pytest.mark.parametrize( "attributes, error_code, error_pointer", ( diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index bb636b1bfa..8e73b96a39 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -252,12 +252,6 @@ def get_prowler_provider_kwargs( **prowler_provider_kwargs, "filter_accounts": [provider.uid], } - elif provider.provider == Provider.ProviderChoices.ORACLECLOUD.value: - if isinstance(prowler_provider_kwargs.get("region"), str): - prowler_provider_kwargs = { - **prowler_provider_kwargs, - "region": {prowler_provider_kwargs["region"]}, - } elif provider.provider == Provider.ProviderChoices.OPENSTACK.value: # clouds_yaml_content, clouds_yaml_cloud and provider_id are validated # in the provider itself, so it's not needed here. @@ -288,6 +282,11 @@ def get_prowler_provider_kwargs( **{k: v for k, v in prowler_provider_kwargs.items() if v}, } + elif provider.provider == Provider.ProviderChoices.ORACLECLOUD.value: + prowler_provider_kwargs = _normalize_oraclecloud_provider_kwargs( + prowler_provider_kwargs + ) + if mutelist_processor: mutelist_content = mutelist_processor.configuration.get("Mutelist", {}) # IaC and Image providers don't support mutelist (both use Trivy's built-in logic) @@ -300,6 +299,40 @@ def get_prowler_provider_kwargs( return prowler_provider_kwargs +def _normalize_oraclecloud_provider_kwargs(secret: dict) -> dict: + """Normalize external OCI secret fields into SDK provider kwargs.""" + prowler_provider_kwargs = secret.copy() + prowler_provider_kwargs.pop("region", None) + + return prowler_provider_kwargs + + +def _normalize_oraclecloud_connection_test_kwargs(secret: dict) -> dict: + """Normalize external OCI secret fields into test_connection kwargs.""" + from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider + + prowler_provider_kwargs = secret.copy() + prowler_provider_kwargs.pop("region", None) + + if ( + prowler_provider_kwargs.get("user") + and prowler_provider_kwargs.get("fingerprint") + and prowler_provider_kwargs.get("tenancy") + and ( + prowler_provider_kwargs.get("key_content") + or prowler_provider_kwargs.get("key_file") + ) + ): + # Connection validation needs one OCI endpoint, but scans remain unfiltered. + prowler_provider_kwargs["region"] = getattr( + OraclecloudProvider, + "_bootstrap_region", + OraclecloudProvider._home_region, + ) + + return prowler_provider_kwargs + + def initialize_prowler_provider( provider: Provider, mutelist_processor: Processor | None = None, @@ -402,6 +435,15 @@ def prowler_provider_connection_test(provider: Provider) -> Connection: if prowler_provider_kwargs.get("registry_token"): image_kwargs["registry_token"] = prowler_provider_kwargs["registry_token"] return prowler_provider.test_connection(**image_kwargs) + elif provider.provider == Provider.ProviderChoices.ORACLECLOUD.value: + oraclecloud_kwargs = _normalize_oraclecloud_connection_test_kwargs( + prowler_provider_kwargs + ) + return prowler_provider.test_connection( + **oraclecloud_kwargs, + provider_id=provider.uid, + raise_on_exception=False, + ) else: return prowler_provider.test_connection( **prowler_provider_kwargs, diff --git a/api/src/backend/api/v1/serializer_utils/providers.py b/api/src/backend/api/v1/serializer_utils/providers.py index 50c1c7376c..49b593049f 100644 --- a/api/src/backend/api/v1/serializer_utils/providers.py +++ b/api/src/backend/api/v1/serializer_utils/providers.py @@ -295,16 +295,21 @@ from rest_framework_json_api import serializers "type": "string", "description": "The OCID of the tenancy.", }, - "region": { - "type": "string", - "description": "The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1).", - }, "pass_phrase": { "type": "string", "description": "The passphrase for the private key, if encrypted.", }, + "region": { + "type": "string", + "deprecated": True, + "description": "Legacy OCI region field accepted for backwards compatibility but ignored; OCI scans all regions.", + }, }, - "required": ["user", "fingerprint", "tenancy", "region"], + "required": ["user", "fingerprint", "tenancy"], + "anyOf": [ + {"required": ["key_file"]}, + {"required": ["key_content"]}, + ], }, { "type": "object", diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 0f08c8f4ba..750174e7a8 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -1672,6 +1672,7 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer): validation_error.detail[f"secret/{key}"] = value del validation_error.detail[key] raise validation_error + return serializer.validated_data class AwsProviderSecret(serializers.Serializer): @@ -1813,14 +1814,32 @@ class IacProviderSecret(serializers.Serializer): resource_name = "provider-secrets" +class LegacyOCIRegionField(serializers.Field): + def to_internal_value(self, data): + return data + + def to_representation(self, value): + return value + + class OracleCloudProviderSecret(serializers.Serializer): user = serializers.CharField() fingerprint = serializers.CharField() key_file = serializers.CharField(required=False) key_content = serializers.CharField(required=False) tenancy = serializers.CharField() - region = serializers.CharField() pass_phrase = serializers.CharField(required=False) + region = LegacyOCIRegionField(required=False, allow_null=True) + + def validate(self, attrs): + attrs.pop("region", None) + + if "key_file" not in attrs and "key_content" not in attrs: + raise serializers.ValidationError( + {"key_file": "Either key_file or key_content must be provided."} + ) + + return attrs class Meta: resource_name = "provider-secrets" @@ -1965,7 +1984,11 @@ class ProviderSecretCreateSerializer(RLSSerializer, BaseWriteProviderSecretSeria secret = attrs.get("secret") validated_attrs = super().validate(attrs) - self.validate_secret_based_on_provider(provider.provider, secret_type, secret) + validated_secret = self.validate_secret_based_on_provider( + provider.provider, secret_type, secret + ) + if provider.provider == Provider.ProviderChoices.ORACLECLOUD.value: + validated_attrs["secret"] = validated_secret return validated_attrs @@ -1997,7 +2020,11 @@ class ProviderSecretUpdateSerializer(BaseWriteProviderSecretSerializer): secret = attrs.get("secret") validated_attrs = super().validate(attrs) - self.validate_secret_based_on_provider(provider.provider, secret_type, secret) + validated_secret = self.validate_secret_based_on_provider( + provider.provider, secret_type, secret + ) + if provider.provider == Provider.ProviderChoices.ORACLECLOUD.value: + validated_attrs["secret"] = validated_secret return validated_attrs diff --git a/api/src/backend/config/django/base.py b/api/src/backend/config/django/base.py index 04251b4479..a079942600 100644 --- a/api/src/backend/config/django/base.py +++ b/api/src/backend/config/django/base.py @@ -312,8 +312,8 @@ ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES = env.int( "ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES", 30 ) ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int( - "ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880 -) # 48h + "ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 960 +) # 16h # Selects where the persistent attack-paths graph is stored. The scan # temporary database is always Neo4j; only the sink is configurable. diff --git a/api/src/backend/config/settings/sentry.py b/api/src/backend/config/settings/sentry.py index d75f9360e5..a3acbe37bb 100644 --- a/api/src/backend/config/settings/sentry.py +++ b/api/src/backend/config/settings/sentry.py @@ -91,6 +91,13 @@ def before_send(event, hint): log_msg = log_record.getMessage() log_lvl = log_record.levelno + if ( + getattr(log_record, "name", "") == "cartography.graph.job" + and "Neo.ClientError.Database.DatabaseNotFound" in log_msg + and "db-tmp-scan-" in log_msg + ): + return None + # The Neo4j driver logs transient connection errors (defunct # connections, resets) at ERROR level via the `neo4j.io` logger. # `RetryableSession` handles these with retries. If all retries diff --git a/api/src/backend/config/settings/social_login.py b/api/src/backend/config/settings/social_login.py index 9ab3e89d73..ffc6a4724e 100644 --- a/api/src/backend/config/settings/social_login.py +++ b/api/src/backend/config/settings/social_login.py @@ -13,7 +13,7 @@ GITHUB_OAUTH_CALLBACK_URL = env("SOCIAL_GITHUB_OAUTH_CALLBACK_URL", default="") ACCOUNT_LOGIN_METHODS = {"email"} # Use Email / Password authentication ACCOUNT_SIGNUP_FIELDS = ["email*", "password1*", "password2*"] ACCOUNT_EMAIL_VERIFICATION = "none" # Do not require email confirmation -ACCOUNT_EMAIL_NOTIFICATIONS = True +ACCOUNT_EMAIL_NOTIFICATIONS = False ACCOUNT_USER_MODEL_USERNAME_FIELD = None REST_AUTH = { "TOKEN_MODEL": None, diff --git a/api/src/backend/tasks/jobs/attack_paths/aws.py b/api/src/backend/tasks/jobs/attack_paths/aws.py index 15ecd86a19..4b2b96dd1b 100644 --- a/api/src/backend/tasks/jobs/attack_paths/aws.py +++ b/api/src/backend/tasks/jobs/attack_paths/aws.py @@ -8,6 +8,8 @@ import aioboto3 import boto3 import botocore import neo4j +import neo4j.exceptions +from api.attack_paths.database import DATABASE_NOT_FOUND_CODE from api.models import ( AttackPathsScan as ProwlerAPIAttackPathsScan, ) @@ -347,6 +349,12 @@ def sync_aws_account( ) except Exception as e: + if ( + isinstance(e, neo4j.exceptions.Neo4jError) + and e.code == DATABASE_NOT_FOUND_CODE + ): + raise + logger.info( f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s (FAILED)" ) diff --git a/api/src/backend/tasks/jobs/attack_paths/config.py b/api/src/backend/tasks/jobs/attack_paths/config.py index d8ed63a8fc..122fc8a9e0 100644 --- a/api/src/backend/tasks/jobs/attack_paths/config.py +++ b/api/src/backend/tasks/jobs/attack_paths/config.py @@ -10,13 +10,10 @@ NormalizedList = _provider_config.NormalizedList PROVIDER_CONFIGS = _provider_config.PROVIDER_CONFIGS ProviderConfig = _provider_config.ProviderConfig -# Batch size for Neo4j write operations (resource labeling, cleanup) -BATCH_SIZE = env.int("ATTACK_PATHS_BATCH_SIZE", 1000) +# Batch size for graph mutation operations (resource labeling and subgraph deletion) +GRAPH_MUTATION_BATCH_SIZE = env.int("ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE", 1000) # Batch size for Postgres findings fetch (keyset pagination page size) FINDINGS_BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 1000) -# Batch size for temp-to-tenant graph sync (nodes and relationships per cursor page) -SYNC_BATCH_SIZE = env.int("ATTACK_PATHS_SYNC_BATCH_SIZE", 1000) - # Neo4j internal labels (Prowler-specific, not provider-specific) # - `Internet`: Singleton node representing external internet access for exposed-resource queries # - `ProwlerFinding`: Label for finding nodes created by Prowler and linked to cloud resources diff --git a/api/src/backend/tasks/jobs/attack_paths/findings.py b/api/src/backend/tasks/jobs/attack_paths/findings.py index 6cc7ddb2e0..c47c9f1149 100644 --- a/api/src/backend/tasks/jobs/attack_paths/findings.py +++ b/api/src/backend/tasks/jobs/attack_paths/findings.py @@ -21,8 +21,8 @@ from cartography.config import Config as CartographyConfig from celery.utils.log import get_task_logger from prowler.config import config as ProwlerConfig from tasks.jobs.attack_paths.config import ( - BATCH_SIZE, FINDINGS_BATCH_SIZE, + GRAPH_MUTATION_BATCH_SIZE, get_node_uid_field, get_provider_resource_label, get_root_node_label, @@ -135,7 +135,7 @@ def add_resource_label( while labeled_count > 0: result = neo4j_session.run( query, - {"provider_uid": provider_uid, "batch_size": BATCH_SIZE}, + {"provider_uid": provider_uid, "batch_size": GRAPH_MUTATION_BATCH_SIZE}, ) labeled_count = result.single().get("labeled_count", 0) total_labeled += labeled_count diff --git a/api/src/backend/tasks/jobs/attack_paths/scan.py b/api/src/backend/tasks/jobs/attack_paths/scan.py index 32337c6832..e161c9eb8d 100644 --- a/api/src/backend/tasks/jobs/attack_paths/scan.py +++ b/api/src/backend/tasks/jobs/attack_paths/scan.py @@ -372,7 +372,19 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: except Exception as e: exception_message = utils.stringify_exception(e, "Attack Paths scan failed") - logger.exception(exception_message) + temporary_database_missing = ( + isinstance(e, graph_database.GraphDatabaseQueryException) + and e.code == graph_database.DATABASE_NOT_FOUND_CODE + and tmp_database_name in str(e) + ) + if temporary_database_missing: + logger.warning(exception_message) + else: + logger.exception(exception_message) + cleanup_log_level = ( + logging.WARNING if temporary_database_missing else logging.ERROR + ) + cleanup_exc_info = not temporary_database_missing ingestion_exceptions["global_error"] = exception_message # Recover `graph_data_ready` based on how far the swap got @@ -387,19 +399,24 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: ) except Exception: - logger.error( - f"Failed to recover `graph_data_ready` for provider {attack_paths_scan.provider_id}", - exc_info=True, + logger.log( + cleanup_log_level, + "Failed to recover `graph_data_ready` for provider " + f"{attack_paths_scan.provider_id}", + exc_info=cleanup_exc_info, ) # Dropping the temporary database if it still exists try: graph_database.drop_database(tmp_cartography_config.neo4j_database) - except Exception as e: - logger.error( - f"Failed to drop temporary Neo4j database `{tmp_cartography_config.neo4j_database}` during cleanup: {e}", - exc_info=True, + except Exception as cleanup_error: + logger.log( + cleanup_log_level, + "Failed to drop temporary Neo4j database " + f"`{tmp_cartography_config.neo4j_database}` during cleanup: " + f"{cleanup_error}", + exc_info=cleanup_exc_info, ) # Set Attack Paths scan state to FAILED @@ -407,10 +424,12 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: db_utils.finish_attack_paths_scan( attack_paths_scan, StateChoices.FAILED, ingestion_exceptions ) - except Exception as e: - logger.error( - f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` (row may have been deleted): {e}", - exc_info=True, + except Exception as cleanup_error: + logger.log( + cleanup_log_level, + f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` " + f"(row may have been deleted): {cleanup_error}", + exc_info=cleanup_exc_info, ) raise diff --git a/api/src/backend/tasks/jobs/attack_paths/sync.py b/api/src/backend/tasks/jobs/attack_paths/sync.py index 00c2c585c7..98a52bd48b 100644 --- a/api/src/backend/tasks/jobs/attack_paths/sync.py +++ b/api/src/backend/tasks/jobs/attack_paths/sync.py @@ -30,7 +30,6 @@ from tasks.jobs.attack_paths.config import ( PROVIDER_CONFIGS, PROVIDER_ISOLATION_PROPERTIES, PROVIDER_RESOURCE_LABEL, - SYNC_BATCH_SIZE, NormalizedList, get_provider_label, get_tenant_label, @@ -116,6 +115,7 @@ def sync_nodes( Source and target sessions are opened sequentially per batch to avoid holding two Bolt connections simultaneously for the entire sync duration. """ + batch_size = sink.sync_batch_size t0 = time.perf_counter() last_id = -1 parents_synced = 0 @@ -137,7 +137,7 @@ def sync_nodes( with graph_database.get_session(source_database) as source_session: result = source_session.run( NODE_FETCH_QUERY, - {"last_id": last_id, "batch_size": SYNC_BATCH_SIZE}, + {"last_id": last_id, "batch_size": batch_size}, ) for record in result: batch_count += 1 @@ -156,17 +156,17 @@ def sync_nodes( for labels, batch in parent_groups.items(): rendered_labels = _render_labels(labels, extra_labels) - for sink_batch in _iter_sink_batches(batch): + for sink_batch in _iter_sink_batches(batch, batch_size): sink.write_nodes(target_database, rendered_labels, sink_batch) for child_label, batch in child_groups.items(): rendered_labels = _render_labels((child_label,), extra_labels) - for sink_batch in _iter_sink_batches(batch): + for sink_batch in _iter_sink_batches(batch, batch_size): sink.write_nodes(target_database, rendered_labels, sink_batch) children_synced += len(batch) for rel_type, batch in rel_groups.items(): - for sink_batch in _iter_sink_batches(batch): + for sink_batch in _iter_sink_batches(batch, batch_size): sink.write_relationships( target_database, rel_type, provider_id, sink_batch ) @@ -205,6 +205,7 @@ def sync_relationships( Source and target sessions are opened sequentially per batch to avoid holding two Bolt connections simultaneously for the entire sync duration. """ + batch_size = sink.sync_batch_size t0 = time.perf_counter() last_id = -1 total_synced = 0 @@ -217,7 +218,7 @@ def sync_relationships( with graph_database.get_session(source_database) as source_session: result = source_session.run( RELATIONSHIPS_FETCH_QUERY, - {"last_id": last_id, "batch_size": SYNC_BATCH_SIZE}, + {"last_id": last_id, "batch_size": batch_size}, ) for record in result: batch_count += 1 @@ -229,7 +230,7 @@ def sync_relationships( break for rel_type, batch in grouped.items(): - for sink_batch in _iter_sink_batches(batch): + for sink_batch in _iter_sink_batches(batch, batch_size): sink.write_relationships( target_database, rel_type, provider_id, sink_batch ) @@ -247,10 +248,9 @@ def sync_relationships( def _iter_sink_batches( rows: list[dict[str, Any]], - batch_size: int | None = None, + batch_size: int, ) -> Iterator[list[dict[str, Any]]]: """Yield final sink write batches after source rows have been transformed.""" - batch_size = SYNC_BATCH_SIZE if batch_size is None else batch_size if batch_size <= 0: raise ValueError("Sink batch size must be greater than zero") diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 39db32abe4..36dc806ff3 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -6,7 +6,7 @@ import re import time import uuid from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Callable, Iterable from datetime import UTC, datetime from typing import Any @@ -49,6 +49,7 @@ from celery.utils.log import get_task_logger from config.django.base import DJANGO_FINDINGS_BATCH_SIZE from config.env import env from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS +from django.core.exceptions import ImproperlyConfigured from django.db import DatabaseError, IntegrityError, OperationalError, transaction from django.db.models import ( Case, @@ -99,6 +100,16 @@ COMPLIANCE_REQUIREMENT_COPY_COLUMNS = ( FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=3000) # Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres. SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=1000) +# Rows per COPY statement when ingesting compliance requirement overviews. All +# batches of a scan share one transaction/commit; the batch size only bounds the +# client-side CSV buffer and how long each individual COPY statement runs on the +# writer (memory footprint, lock time and slow-statement logging under load). +COMPLIANCE_COPY_BATCH_SIZE = env.int("DJANGO_COMPLIANCE_COPY_BATCH_SIZE", default=2000) +if COMPLIANCE_COPY_BATCH_SIZE < 1: + raise ImproperlyConfigured( + "DJANGO_COMPLIANCE_COPY_BATCH_SIZE must be a positive integer, got " + f"{COMPLIANCE_COPY_BATCH_SIZE}" + ) # Throttle scan progress persistence: minimum progress delta (fraction 0-1) # between two persisted progress updates. PROGRESS_THROTTLE_DELTA = env.float("DJANGO_SCAN_PROGRESS_THROTTLE_DELTA", default=0.01) @@ -356,30 +367,36 @@ def _bulk_update_resource_failed_findings_counts( raise -def _copy_compliance_requirement_rows( - tenant_id: str, rows: list[dict[str, Any]] -) -> None: - """Stream compliance requirement rows into Postgres using COPY. +class ComplianceRowScopeError(ValueError): + """A compliance requirement row does not belong to the scan being ingested.""" - We leverage the admin connection (when available) to bypass the COPY + RLS - restriction, writing only the fields required by - ``ComplianceRequirementOverview``. - Args: - tenant_id: Target tenant UUID. - rows: List of row dictionaries prepared by - :func:`create_compliance_requirements`. +def _compliance_requirement_rows_to_csv( + rows: list[dict[str, Any]], tenant_id: str, scan_id: str +) -> io.StringIO: + """Serialize compliance requirement rows into a CSV buffer for COPY. + + COPY runs on the admin connection, which bypasses RLS, so every row is + checked against the expected tenant/scan before it is written: a mismatched + row would otherwise be inserted verbatim into another tenant's data. """ - csv_buffer = io.StringIO() writer = csv.writer(csv_buffer) datetime_now = datetime.now(tz=UTC) for row in rows: + row_tenant_id = str(row.get("tenant_id")) + row_scan_id = str(row.get("scan_id")) + if row_tenant_id != tenant_id or row_scan_id != scan_id: + raise ComplianceRowScopeError( + "Compliance requirement row does not belong to the scan being " + f"ingested (expected tenant {tenant_id} / scan {scan_id}, got " + f"tenant {row_tenant_id} / scan {row_scan_id})" + ) writer.writerow( [ str(row.get("id")), - str(row.get("tenant_id")), + row_tenant_id, (row.get("inserted_at") or datetime_now).isoformat(), row.get("compliance_id") or "", row.get("framework") or "", @@ -393,65 +410,100 @@ def _copy_compliance_requirement_rows( row.get("total_checks", 0), row.get("passed_findings", 0), row.get("total_findings", 0), - str(row.get("scan_id")), + row_scan_id, ] ) csv_buffer.seek(0) + return csv_buffer + + +def _copy_compliance_requirement_rows( + tenant_id: str, scan_id: str, rows: Iterable[dict[str, Any]], batch_size: int +) -> int: + """Replace a scan's compliance requirement rows using batched COPY. + + We leverage the admin connection (when available) to bypass the COPY + RLS + restriction. The scan's DELETE and every COPY batch run on one connection + inside a single transaction with a single commit, so the writer takes one + fsync per scan instead of one per batch, and a failed ingest rolls back + without committing a partial delete/insert (which a retry would otherwise + delete again, feeding dead rows to autovacuum). + + Args: + tenant_id: Target tenant UUID. + scan_id: Scan whose previous rows are replaced. + rows: Iterable of row dictionaries, consumed lazily batch by batch. + batch_size: Number of rows per COPY statement. + + Returns: + int: total number of rows staged and committed. + + Raises: + ComplianceRowScopeError: A row belongs to another tenant or scan. + """ + # Normalized once so the per-row scope check compares like with like even if + # the caller passes UUID instances instead of strings. + tenant_id = str(tenant_id) + scan_id = str(scan_id) + total_rows = 0 + batch_num = 0 copy_sql = ( "COPY compliance_requirements_overviews (" + ", ".join(COMPLIANCE_REQUIREMENT_COPY_COLUMNS) + ") FROM STDIN WITH (FORMAT CSV, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '\\N')" ) - try: - with psycopg_connection(MainRouter.admin_db) as connection: - connection.autocommit = False - try: - with connection.cursor() as cursor: - cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) - cursor.copy_expert(copy_sql, csv_buffer) - connection.commit() - except Exception: - connection.rollback() - raise - finally: - csv_buffer.close() + with psycopg_connection(MainRouter.admin_db) as connection: + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) + # Idempotent re-run: clearing this scan's rows inside the same + # transaction keeps delete + reinsert atomic. + cursor.execute( + "DELETE FROM compliance_requirements_overviews " + "WHERE tenant_id = %s AND scan_id = %s", + [tenant_id, scan_id], + ) + for batch, _is_last in batched(rows, batch_size): + if not batch: + continue + batch_num += 1 + csv_buffer = _compliance_requirement_rows_to_csv( + batch, tenant_id, scan_id + ) + try: + cursor.copy_expert(copy_sql, csv_buffer) + finally: + csv_buffer.close() + total_rows += len(batch) + logger.info( + f"Compliance COPY batch {batch_num}: staged {len(batch)} rows " + f"({total_rows} total)" + ) + connection.commit() + except Exception: + connection.rollback() + raise + + return total_rows -def _persist_compliance_requirement_rows( - tenant_id: str, rows: Iterable[dict[str, Any]], batch_size: int = 10000 +def _bulk_create_compliance_requirement_rows( + tenant_id: str, scan_id: str, rows: Iterable[dict[str, Any]], batch_size: int ) -> int: - """Persist compliance requirement rows using batched COPY with ORM fallback. + """Replace a scan's compliance requirement rows via the ORM. - ``rows`` is consumed lazily in batches, so peak memory stays at ~``batch_size`` - rows instead of the full set. A batch that fails COPY falls back to an ORM - ``bulk_create`` of just that batch. - - Args: - tenant_id: Target tenant UUID. - rows: Iterable of row dictionaries reflecting the compliance overview - state for a scan. - batch_size: Number of rows per COPY batch (default: 10000). - - Returns: - int: total number of rows persisted. + Fallback for when COPY is unavailable; the delete and every ``bulk_create`` + share one RLS transaction so the replacement stays atomic. """ total_rows = 0 - batch_num = 0 - - for batch, _is_last in batched(rows, batch_size): - if not batch: - continue - batch_num += 1 - try: - _copy_compliance_requirement_rows(tenant_id, batch) - except Exception as error: - logger.exception( - f"COPY bulk insert for compliance requirements batch {batch_num} " - "failed; falling back to ORM bulk_create for this batch", - exc_info=error, - ) + with rls_transaction(tenant_id): + ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete() + for batch, _is_last in batched(rows, batch_size): + if not batch: + continue fallback_objects = [ ComplianceRequirementOverview( id=row["id"], @@ -473,20 +525,58 @@ def _persist_compliance_requirement_rows( ) for row in batch ] - with rls_transaction(tenant_id): - ComplianceRequirementOverview.objects.bulk_create( - fallback_objects, batch_size=500 - ) - - total_rows += len(batch) - logger.info( - f"Compliance COPY batch {batch_num}: inserted {len(batch)} rows " - f"({total_rows} total)" - ) - + ComplianceRequirementOverview.objects.bulk_create( + fallback_objects, batch_size=500 + ) + total_rows += len(batch) return total_rows +def _persist_compliance_requirement_rows( + tenant_id: str, + scan_id: str, + rows_factory: Callable[[], Iterable[dict[str, Any]]], + batch_size: int | None = None, +) -> int: + """Persist a scan's compliance requirement rows, replacing any previous ones. + + ``rows_factory`` must return a fresh row iterator on every call: the COPY + path consumes it lazily in batches (peak memory ~``batch_size`` rows), and + if COPY fails the whole ingest falls back to a single ORM transaction that + re-iterates the rows. + + Args: + tenant_id: Target tenant UUID. + scan_id: Scan whose compliance overview rows are being replaced. + rows_factory: Callable returning an iterable of row dictionaries. + batch_size: Rows per COPY/bulk_create batch (default: + ``COMPLIANCE_COPY_BATCH_SIZE``). + + Returns: + int: total number of rows persisted. + """ + if batch_size is None: + batch_size = COMPLIANCE_COPY_BATCH_SIZE + + try: + return _copy_compliance_requirement_rows( + tenant_id, scan_id, rows_factory(), batch_size + ) + except ComplianceRowScopeError: + # Cross-tenant/scan rows are a bug in the caller, not a COPY failure: + # retrying through the ORM would persist the very rows we rejected. + raise + except Exception as error: + logger.exception( + "COPY bulk insert for compliance requirements failed; " + "falling back to ORM bulk_create", + exc_info=error, + ) + return _bulk_create_compliance_requirement_rows( + tenant_id, scan_id, rows_factory(), batch_size + ) + + def _create_compliance_summaries( tenant_id: str, scan_id: str, requirement_statuses: dict ) -> None: @@ -885,15 +975,19 @@ def _process_finding_micro_batch( # Denormalized resource arrays populated directly on insert # (was previously a separate bulk_update; saves a CASE WHEN # over thousands of rows per micro-batch). - resource_regions=[resource_instance.region] - if resource_instance.region - else [], - resource_services=[resource_instance.service] - if resource_instance.service - else [], - resource_types=[resource_instance.type] - if resource_instance.type - else [], + resource_regions=( + [resource_instance.region] + if resource_instance.region + else [] + ), + resource_services=( + [resource_instance.service] + if resource_instance.service + else [] + ), + resource_types=( + [resource_instance.type] if resource_instance.type else [] + ), ) findings_to_create.append(finding_instance) resource_denormalized_data.append( @@ -1708,8 +1802,10 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): ) # Yield rows lazily (consumed batch-by-batch by COPY) so peak memory - # stays bounded; tally requirement_statuses in the same pass. + # stays bounded; tally requirement_statuses in the same pass. The + # ORM fallback re-iterates from scratch, so the tally resets first. def _iter_compliance_requirement_rows(): + requirement_statuses.clear() for region in regions: region_stats = region_requirement_stats.get(region, {}) region_findings = findings_count_by_compliance.get(region, {}) @@ -1773,12 +1869,10 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): "total_findings": total_findings, } - # Idempotent re-run: clear this scan's rows before re-inserting. - with rls_transaction(tenant_id): - ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete() - + # The delete of the scan's previous rows happens inside the same + # transaction as the inserts (see _copy_compliance_requirement_rows). requirements_created = _persist_compliance_requirement_rows( - tenant_id, _iter_compliance_requirement_rows() + tenant_id_str, scan_id_str, _iter_compliance_requirement_rows ) # Create pre-aggregated summaries for fast compliance overview lookups diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 7a1ff54131..a91ff85c01 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -11,6 +11,7 @@ from api.compliance import ( from api.db_router import READ_REPLICA_ALIAS from api.db_utils import delete_related_daily_task, rls_transaction from api.decorators import handle_provider_deletion, set_tenant +from api.exceptions import ProviderDeletedException from api.models import ( Finding, Integration, @@ -666,7 +667,13 @@ class AttackPathsScanRLSTask(RLSTask): scan_id = kwargs.get("scan_id") if tenant_id and scan_id: - logger.error(f"Attack paths scan task {task_id} failed: {exc}") + if isinstance(exc, ProviderDeletedException): + logger.warning( + f"Attack paths scan task {task_id} stopped because its provider " + f"or tenant was deleted: {exc}" + ) + else: + logger.error(f"Attack paths scan task {task_id} failed: {exc}") attack_paths_db_utils.fail_attack_paths_scan(tenant_id, scan_id, str(exc)) diff --git a/api/src/backend/tasks/tests/test_attack_paths_aws.py b/api/src/backend/tasks/tests/test_attack_paths_aws.py new file mode 100644 index 0000000000..dc2c59d614 --- /dev/null +++ b/api/src/backend/tasks/tests/test_attack_paths_aws.py @@ -0,0 +1,102 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import neo4j.exceptions +import pytest +from tasks.jobs.attack_paths import aws + +DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound" + + +def _make_neo4j_error(code: str) -> neo4j.exceptions.Neo4jError: + return neo4j.exceptions.Neo4jError._hydrate_neo4j( + code=code, + message="graph query failed", + ) + + +def _resource_functions(failing_sync, following_sync): + return { + "failing_sync": failing_sync, + "following_sync": following_sync, + "permission_relationships": MagicMock(), + "resourcegroupstaggingapi": MagicMock(), + } + + +def test_sync_aws_account_reraises_database_not_found_immediately(): + error = _make_neo4j_error(DATABASE_NOT_FOUND_CODE) + failing_sync = MagicMock(side_effect=error) + following_sync = MagicMock() + + with ( + patch.object( + aws.cartography_aws, + "RESOURCE_FUNCTIONS", + _resource_functions(failing_sync, following_sync), + ), + patch.object(aws.db_utils, "update_attack_paths_scan_progress"), + patch.object(aws.utils, "stringify_exception") as stringify_exception, + patch.object(aws.logger, "warning") as warning, + pytest.raises(neo4j.exceptions.Neo4jError) as exc_info, + ): + aws.sync_aws_account( + SimpleNamespace(uid="123456789012"), + [ + "failing_sync", + "following_sync", + "permission_relationships", + "resourcegroupstaggingapi", + ], + {}, + MagicMock(), + ) + + assert exc_info.value is error + following_sync.assert_not_called() + stringify_exception.assert_not_called() + warning.assert_not_called() + + +@pytest.mark.parametrize( + "error", + [ + _make_neo4j_error("Neo.ClientError.Statement.SyntaxError"), + RuntimeError("resource sync failed"), + ], + ids=["different-neo4j-error", "non-neo4j-error"], +) +def test_sync_aws_account_warns_and_continues_for_other_exceptions(error): + failing_sync = MagicMock(side_effect=error) + following_sync = MagicMock() + + with ( + patch.object( + aws.cartography_aws, + "RESOURCE_FUNCTIONS", + _resource_functions(failing_sync, following_sync), + ), + patch.object(aws.db_utils, "update_attack_paths_scan_progress"), + patch.object( + aws.utils, + "stringify_exception", + return_value="formatted failure", + ), + patch.object(aws.logger, "warning") as warning, + ): + failed_syncs = aws.sync_aws_account( + SimpleNamespace(uid="123456789012"), + [ + "failing_sync", + "following_sync", + "permission_relationships", + "resourcegroupstaggingapi", + ], + {}, + MagicMock(), + ) + + assert failed_syncs == {"failing_sync": "formatted failure"} + following_sync.assert_called_once_with() + warning.assert_called_once() + assert "Continuing to the next AWS sync function" in warning.call_args.args[0] diff --git a/api/src/backend/tasks/tests/test_attack_paths_scan.py b/api/src/backend/tasks/tests/test_attack_paths_scan.py index 3be885a7fc..4409e5f19d 100644 --- a/api/src/backend/tasks/tests/test_attack_paths_scan.py +++ b/api/src/backend/tasks/tests/test_attack_paths_scan.py @@ -1,3 +1,4 @@ +import logging from contextlib import nullcontext from datetime import UTC, datetime, timedelta from types import SimpleNamespace @@ -5,7 +6,9 @@ from unittest.mock import MagicMock, call, patch from uuid import uuid4 import pytest +from api.attack_paths.database import GraphDatabaseQueryException from api.db_utils import rls_transaction +from api.exceptions import ProviderDeletedException from api.models import ( AttackPathsScan, Finding, @@ -250,6 +253,32 @@ class TestAttackPathsRun: mock_starting.assert_not_called() mock_create_db.assert_not_called() + @pytest.mark.parametrize( + ("ingestion_error", "temporary_database_missing"), + [ + (RuntimeError("ingestion boom"), False), + ( + GraphDatabaseQueryException( + message="Graph not found: db-scan-id", + code="Neo.ClientError.Database.DatabaseNotFound", + ), + True, + ), + ( + GraphDatabaseQueryException( + message="Graph not found: db-tenant-id", + code="Neo.ClientError.Database.DatabaseNotFound", + ), + False, + ), + ], + ids=[ + "regular-error", + "temporary-database-missing", + "sink-database-missing", + ], + ) + @patch("tasks.jobs.attack_paths.scan.logger") @patch( "tasks.jobs.attack_paths.scan.utils.stringify_exception", return_value="Cartography failed: ingestion boom", @@ -302,6 +331,9 @@ class TestAttackPathsRun: mock_drop_db, mock_event_loop, mock_stringify, + mock_logger, + ingestion_error, + temporary_database_missing, tenants_fixture, aws_provider, scans_fixture, @@ -321,7 +353,11 @@ class TestAttackPathsRun: session_ctx = MagicMock() session_ctx.__enter__.return_value = mock_session session_ctx.__exit__.return_value = False - ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom")) + ingestion_fn = MagicMock(side_effect=ingestion_error) + if temporary_database_missing: + mock_finish.side_effect = DatabaseError( + "Save with update_fields did not affect any rows" + ) with ( patch( @@ -337,13 +373,28 @@ class TestAttackPathsRun: return_value=ingestion_fn, ), ): - with pytest.raises(RuntimeError, match="ingestion boom"): + with pytest.raises(type(ingestion_error)): attack_paths_run(str(tenant.id), str(scan.id), "task-456") failure_args = mock_finish.call_args[0] assert failure_args[0] is attack_paths_scan assert failure_args[1] == StateChoices.FAILED assert failure_args[2] == {"global_error": "Cartography failed: ingestion boom"} + mock_drop_db.assert_called_once_with("db-scan-id") + if temporary_database_missing: + mock_logger.warning.assert_any_call("Cartography failed: ingestion boom") + mock_logger.exception.assert_not_called() + mock_logger.log.assert_called_once_with( + logging.WARNING, + f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` " + "(row may have been deleted): Save with update_fields did not affect " + "any rows", + exc_info=False, + ) + else: + mock_logger.exception.assert_called_once_with( + "Cartography failed: ingestion boom" + ) @patch( "tasks.jobs.attack_paths.scan.utils.stringify_exception", @@ -1265,6 +1316,33 @@ class TestAttackPathsScanRLSTaskOnFailure: mock_fail.assert_called_once_with("t-1", "s-1", "boom") + def test_on_failure_logs_provider_deletion_as_warning(self): + from tasks.tasks import AttackPathsScanRLSTask + + task = AttackPathsScanRLSTask() + error = ProviderDeletedException("provider deleted") + + with ( + patch("tasks.tasks.logger") as mock_logger, + patch( + "tasks.tasks.attack_paths_db_utils.fail_attack_paths_scan" + ) as mock_fail, + ): + task.on_failure( + exc=error, + task_id="task-abc", + args=(), + kwargs={"tenant_id": "t-1", "scan_id": "s-1"}, + _einfo=None, + ) + + mock_logger.warning.assert_called_once_with( + "Attack paths scan task task-abc stopped because its provider or tenant " + "was deleted: provider deleted" + ) + mock_logger.error.assert_not_called() + mock_fail.assert_called_once_with("t-1", "s-1", "provider deleted") + def test_on_failure_skips_when_missing_kwargs(self): from tasks.tasks import AttackPathsScanRLSTask @@ -1896,7 +1974,7 @@ class TestSyncNodes: mock_source_1.run.return_value = [row] mock_source_2 = MagicMock() mock_source_2.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1000) with patch( "tasks.jobs.attack_paths.sync.graph_database.get_session", @@ -1933,7 +2011,7 @@ class TestSyncNodes: src_1.run.return_value = [row] src_2 = MagicMock() src_2.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1000) sink.write_nodes.side_effect = lambda *_a, **_kw: call_order.append( "sink:write" ) @@ -1969,18 +2047,15 @@ class TestSyncNodes: src_2.run.return_value = [row_b] src_3 = MagicMock() src_3.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1) - with ( - patch( - "tasks.jobs.attack_paths.sync.graph_database.get_session", - side_effect=[ - _make_session_ctx(src_1), - _make_session_ctx(src_2), - _make_session_ctx(src_3), - ], - ), - patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 1), + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1), + _make_session_ctx(src_2), + _make_session_ctx(src_3), + ], ): result = sync_module.sync_nodes("src", "tgt", "t-1", "p-1", sink, []) @@ -2009,17 +2084,14 @@ class TestSyncNodes: src_1.run.return_value = [row] src_2 = MagicMock() src_2.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=2) - with ( - patch( - "tasks.jobs.attack_paths.sync.graph_database.get_session", - side_effect=[ - _make_session_ctx(src_1), - _make_session_ctx(src_2), - ], - ), - patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 2), + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1), + _make_session_ctx(src_2), + ], ): result = sync_module.sync_nodes( "src", "tgt", "t-1", "p-1", sink, normalized_lists @@ -2037,7 +2109,7 @@ class TestSyncNodes: def test_sync_nodes_empty_source_returns_zero(self): src = MagicMock() src.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1000) with patch( "tasks.jobs.attack_paths.sync.graph_database.get_session", @@ -2066,7 +2138,7 @@ class TestSyncRelationships: src_1.run.return_value = [row] src_2 = MagicMock() src_2.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1000) sink.write_relationships.side_effect = lambda *_a, **_kw: call_order.append( "sink:write" ) @@ -2104,18 +2176,15 @@ class TestSyncRelationships: src_2.run.return_value = [row_b] src_3 = MagicMock() src_3.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1) - with ( - patch( - "tasks.jobs.attack_paths.sync.graph_database.get_session", - side_effect=[ - _make_session_ctx(src_1), - _make_session_ctx(src_2), - _make_session_ctx(src_3), - ], - ), - patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 1), + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1), + _make_session_ctx(src_2), + _make_session_ctx(src_3), + ], ): total = sync_module.sync_relationships("src", "tgt", "p-1", sink) @@ -2140,17 +2209,14 @@ class TestSyncRelationships: src_1.run.return_value = rows src_2 = MagicMock() src_2.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=2) - with ( - patch( - "tasks.jobs.attack_paths.sync.graph_database.get_session", - side_effect=[ - _make_session_ctx(src_1), - _make_session_ctx(src_2), - ], - ), - patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 2), + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1), + _make_session_ctx(src_2), + ], ): total = sync_module.sync_relationships("src", "tgt", "p-1", sink) @@ -2163,7 +2229,7 @@ class TestSyncRelationships: def test_sync_relationships_empty_source_returns_zero(self): src = MagicMock() src.run.return_value = [] - sink = MagicMock() + sink = MagicMock(sync_batch_size=1000) with patch( "tasks.jobs.attack_paths.sync.graph_database.get_session", @@ -3056,6 +3122,61 @@ class TestCleanupStaleAttackPathsScans: ap_scan.refresh_from_db() assert ap_scan.state == StateChoices.FAILED + @pytest.mark.parametrize( + ("age_seconds", "should_clean"), + [ + (960 * 60 - 1, False), + (960 * 60, False), + (960 * 60 + 1, True), + ], + ) + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._revoke_task") + @patch("tasks.jobs.attack_paths.cleanup._ping_workers") + def test_stale_threshold_boundary_is_strict( + self, + mock_ping, + mock_revoke, + mock_drop_db, + mock_recover, + age_seconds, + should_clean, + tenants_fixture, + aws_provider, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + now = datetime.now(tz=UTC) + ap_scan, task_result = self._create_executing_scan( + tenants_fixture[0], + aws_provider, + started_at=now - timedelta(seconds=age_seconds), + worker="live-worker@host", + ) + mock_ping.return_value = ({"live-worker@host"}, set()) + + with patch("tasks.jobs.attack_paths.cleanup.datetime") as mock_datetime: + mock_datetime.now.return_value = now + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == int(should_clean) + ap_scan.refresh_from_db() + expected_state = StateChoices.FAILED if should_clean else StateChoices.EXECUTING + assert ap_scan.state == expected_state + if should_clean: + mock_revoke.assert_called_once_with(task_result, terminate=True) + mock_drop_db.assert_called_once() + mock_recover.assert_called_once() + else: + mock_revoke.assert_not_called() + mock_drop_db.assert_not_called() + mock_recover.assert_not_called() + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") @patch( diff --git a/api/src/backend/tasks/tests/test_scan.py b/api/src/backend/tasks/tests/test_scan.py index 2a66985276..cad5e3d343 100644 --- a/api/src/backend/tasks/tests/test_scan.py +++ b/api/src/backend/tasks/tests/test_scan.py @@ -26,6 +26,7 @@ from prowler.lib.check.models import Severity from prowler.lib.outputs.finding import Status from tasks.jobs.scan import ( _ATTACK_SURFACE_MAPPING_CACHE, + ComplianceRowScopeError, _aggregate_findings_by_region, _bulk_update_resource_failed_findings_counts, _copy_compliance_requirement_rows, @@ -2314,9 +2315,9 @@ class TestCreateComplianceRequirements: create_compliance_requirements(tenant_id, scan_id) mock_persist.assert_called_once() - persisted_rows = mock_persist.call_args[0][1] + rows_factory = mock_persist.call_args[0][2] requirement_row = next( - row for row in persisted_rows if row["requirement_id"] == "1.1" + row for row in rows_factory() if row["requirement_id"] == "1.1" ) assert requirement_row["requirement_status"] == "FAIL" @@ -2454,18 +2455,26 @@ class TestComplianceRequirementCopy: } with patch.object(MainRouter, "admin_db", "admin"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) mock_psycopg_connection.assert_called_once_with("admin") connection.cursor.assert_called_once() - cursor.execute.assert_called_once() + # One execute for set_config plus one for the scan's DELETE. + assert cursor.execute.call_count == 2 + delete_sql, delete_params = cursor.execute.call_args_list[1][0] + assert "DELETE FROM compliance_requirements_overviews" in delete_sql + assert delete_params == [str(row["tenant_id"]), str(row["scan_id"])] cursor.copy_expert.assert_called_once() + connection.commit.assert_called_once() csv_rows = list(csv.reader(StringIO(captured["data"]))) assert csv_rows[0][0] == str(row["id"]) assert csv_rows[0][5] == "" assert csv_rows[0][-1] == str(row["scan_id"]) + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.filter") @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") @patch("tasks.jobs.scan.rls_transaction") @patch( @@ -2473,7 +2482,7 @@ class TestComplianceRequirementCopy: side_effect=Exception("copy failed"), ) def test_persist_compliance_requirement_rows_fallback( - self, mock_copy, mock_rls_transaction, mock_bulk_create + self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_filter ): inserted_at = datetime.now(UTC) row = { @@ -2494,16 +2503,22 @@ class TestComplianceRequirementCopy: } tenant_id = row["tenant_id"] + scan_id = str(row["scan_id"]) ctx = MagicMock() ctx.__enter__.return_value = None ctx.__exit__.return_value = False mock_rls_transaction.return_value = ctx - _persist_compliance_requirement_rows(tenant_id, [row]) + _persist_compliance_requirement_rows(tenant_id, scan_id, lambda: [row]) - mock_copy.assert_called_once_with(tenant_id, [row]) + mock_copy.assert_called_once() + assert mock_copy.call_args[0][0] == tenant_id + assert mock_copy.call_args[0][1] == scan_id mock_rls_transaction.assert_called_once_with(tenant_id) + # The fallback replaces the scan's rows: delete + insert atomically. + mock_filter.assert_called_once_with(scan_id=scan_id) + mock_filter.return_value.delete.assert_called_once() mock_bulk_create.assert_called_once() args, kwargs = mock_bulk_create.call_args @@ -2515,13 +2530,18 @@ class TestComplianceRequirementCopy: @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") @patch("tasks.jobs.scan.rls_transaction") - @patch("tasks.jobs.scan._copy_compliance_requirement_rows") + @patch("tasks.jobs.scan._copy_compliance_requirement_rows", return_value=0) def test_persist_compliance_requirement_rows_no_rows( self, mock_copy, mock_rls_transaction, mock_bulk_create ): - _persist_compliance_requirement_rows(str(uuid.uuid4()), []) + # Even with no rows the COPY path runs: it must clear the scan's + # previous rows so a re-run with fewer findings drops stale data. + total = _persist_compliance_requirement_rows( + str(uuid.uuid4()), str(uuid.uuid4()), lambda: [] + ) - mock_copy.assert_not_called() + assert total == 0 + mock_copy.assert_called_once() mock_rls_transaction.assert_not_called() mock_bulk_create.assert_not_called() @@ -2610,11 +2630,12 @@ class TestComplianceRequirementCopy: ] with patch.object(MainRouter, "admin_db", "admin"): - _copy_compliance_requirement_rows(tenant_id, rows) + _copy_compliance_requirement_rows(tenant_id, str(scan_id), rows, 2000) mock_psycopg_connection.assert_called_once_with("admin") connection.cursor.assert_called_once() - cursor.execute.assert_called_once() + # set_config + DELETE of the scan's previous rows. + assert cursor.execute.call_count == 2 cursor.copy_expert.assert_called_once() csv_rows = list(csv.reader(StringIO(captured["data"]))) @@ -2644,6 +2665,60 @@ class TestComplianceRequirementCopy: assert csv_rows[2][5] == "2.0" assert csv_rows[2][9] == "MANUAL" + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_batches_share_one_transaction( + self, mock_psycopg_connection, settings + ): + """Every COPY batch runs on the same connection with a single commit.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + inserted_at = datetime.now(UTC) + rows = [ + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": "1.0", + "description": f"Requirement {index}", + "region": "us-east-1", + "requirement_id": f"req-{index}", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": scan_id, + } + for index in range(3) + ] + + with patch.object(MainRouter, "admin_db", "admin"): + total = _copy_compliance_requirement_rows(tenant_id, scan_id, rows, 1) + + assert total == 3 + # One connection, three COPY statements, one commit for the whole scan. + mock_psycopg_connection.assert_called_once_with("admin") + assert cursor.copy_expert.call_count == 3 + connection.commit.assert_called_once() + connection.rollback.assert_not_called() + @patch("tasks.jobs.scan.psycopg_connection") def test_copy_compliance_requirement_rows_null_values( self, mock_psycopg_connection, settings @@ -2691,7 +2766,9 @@ class TestComplianceRequirementCopy: } with patch.object(MainRouter, "admin_db", "admin"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) csv_rows = list(csv.reader(StringIO(captured["data"]))) assert len(csv_rows) == 1 @@ -2747,7 +2824,9 @@ class TestComplianceRequirementCopy: } with patch.object(MainRouter, "admin_db", "admin"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) # Verify CSV was generated (csv module handles escaping automatically) csv_rows = list(csv.reader(StringIO(captured["data"]))) @@ -2808,7 +2887,9 @@ class TestComplianceRequirementCopy: before_call = datetime.now(UTC) with patch.object(MainRouter, "admin_db", "admin"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) after_call = datetime.now(UTC) csv_rows = list(csv.reader(StringIO(captured["data"]))) @@ -2861,12 +2942,84 @@ class TestComplianceRequirementCopy: with patch.object(MainRouter, "admin_db", "admin"): with pytest.raises(Exception, match="COPY command failed"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) # Verify rollback was called connection.rollback.assert_called_once() connection.commit.assert_not_called() + @pytest.mark.parametrize("mismatched_field", ["tenant_id", "scan_id"]) + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_rejects_out_of_scope_rows( + self, mock_psycopg_connection, mismatched_field, settings + ): + """COPY bypasses RLS, so rows from another tenant/scan must be rejected.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + row = { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "compliance_id": "test", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": scan_id, + } + row[mismatched_field] = str(uuid.uuid4()) + + with patch.object(MainRouter, "admin_db", "admin"): + with pytest.raises(ComplianceRowScopeError): + _copy_compliance_requirement_rows(tenant_id, scan_id, [row], 2000) + + cursor.copy_expert.assert_not_called() + connection.rollback.assert_called_once() + connection.commit.assert_not_called() + + @patch("tasks.jobs.scan.ComplianceRequirementOverview") + @patch("tasks.jobs.scan.rls_transaction") + @patch( + "tasks.jobs.scan._copy_compliance_requirement_rows", + side_effect=ComplianceRowScopeError("out of scope"), + ) + def test_persist_compliance_requirement_rows_does_not_fall_back_on_scope_error( + self, mock_copy, mock_rls_transaction, mock_model + ): + """A scope violation is a caller bug: the ORM fallback must not persist it.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + with pytest.raises(ComplianceRowScopeError): + _persist_compliance_requirement_rows(tenant_id, scan_id, lambda: []) + + mock_copy.assert_called_once() + mock_rls_transaction.assert_not_called() + mock_model.objects.filter.assert_not_called() + mock_model.objects.bulk_create.assert_not_called() + @patch("tasks.jobs.scan.psycopg_connection") def test_copy_compliance_requirement_rows_transaction_rollback_on_set_config_error( self, mock_psycopg_connection, settings @@ -2909,7 +3062,9 @@ class TestComplianceRequirementCopy: with patch.object(MainRouter, "admin_db", "admin"): with pytest.raises(Exception, match="SET prowler.tenant_id failed"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) # Verify rollback was called connection.rollback.assert_called_once() @@ -2955,7 +3110,9 @@ class TestComplianceRequirementCopy: } with patch.object(MainRouter, "admin_db", "admin"): - _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + _copy_compliance_requirement_rows( + str(row["tenant_id"]), str(row["scan_id"]), [row], 2000 + ) # Verify commit was called and rollback was not connection.commit.assert_called_once() @@ -2966,9 +3123,10 @@ class TestComplianceRequirementCopy: @patch("tasks.jobs.scan._copy_compliance_requirement_rows") def test_persist_compliance_requirement_rows_success(self, mock_copy): """Test successful COPY path without fallback to ORM.""" - mock_copy.return_value = None # Success, no exception + mock_copy.return_value = 1 # Success, no exception tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) rows = [ { "id": uuid.uuid4(), @@ -2984,16 +3142,21 @@ class TestComplianceRequirementCopy: "passed_checks": 1, "failed_checks": 0, "total_checks": 1, - "scan_id": uuid.uuid4(), + "scan_id": scan_id, } ] - _persist_compliance_requirement_rows(tenant_id, rows) + total = _persist_compliance_requirement_rows(tenant_id, scan_id, lambda: rows) - # Verify COPY was called - mock_copy.assert_called_once_with(tenant_id, rows) + assert total == 1 + mock_copy.assert_called_once() + copy_args = mock_copy.call_args[0] + assert copy_args[0] == tenant_id + assert copy_args[1] == scan_id + assert list(copy_args[2]) == rows @patch("tasks.jobs.scan.logger") + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.filter") @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") @patch("tasks.jobs.scan.rls_transaction") @patch( @@ -3001,7 +3164,12 @@ class TestComplianceRequirementCopy: side_effect=Exception("COPY failed"), ) def test_persist_compliance_requirement_rows_fallback_logging( - self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_logger + self, + mock_copy, + mock_rls_transaction, + mock_bulk_create, + mock_filter, + mock_logger, ): """Test logger.exception is called when COPY fails and fallback occurs.""" tenant_id = str(uuid.uuid4()) @@ -3027,7 +3195,9 @@ class TestComplianceRequirementCopy: ctx.__exit__.return_value = False mock_rls_transaction.return_value = ctx - _persist_compliance_requirement_rows(tenant_id, [row]) + _persist_compliance_requirement_rows( + tenant_id, str(row["scan_id"]), lambda: [row] + ) # Verify logger.exception was called mock_logger.exception.assert_called_once() @@ -3036,6 +3206,7 @@ class TestComplianceRequirementCopy: assert "falling back to ORM" in args[0] assert kwargs.get("exc_info") is not None + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.filter") @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") @patch("tasks.jobs.scan.rls_transaction") @patch( @@ -3043,7 +3214,7 @@ class TestComplianceRequirementCopy: side_effect=Exception("copy failed"), ) def test_persist_compliance_requirement_rows_fallback_multiple_rows( - self, mock_copy, mock_rls_transaction, mock_bulk_create + self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_filter ): """Test ORM fallback with multiple rows.""" tenant_id = str(uuid.uuid4()) @@ -3090,10 +3261,14 @@ class TestComplianceRequirementCopy: ctx.__exit__.return_value = False mock_rls_transaction.return_value = ctx - _persist_compliance_requirement_rows(tenant_id, rows) + total = _persist_compliance_requirement_rows( + tenant_id, str(scan_id), lambda: rows + ) - mock_copy.assert_called_once_with(tenant_id, rows) + assert total == 2 + mock_copy.assert_called_once() mock_rls_transaction.assert_called_once_with(tenant_id) + mock_filter.assert_called_once_with(scan_id=str(scan_id)) mock_bulk_create.assert_called_once() args, kwargs = mock_bulk_create.call_args @@ -3117,6 +3292,7 @@ class TestComplianceRequirementCopy: assert objects[1].passed_checks == 2 assert objects[1].failed_checks == 3 + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.filter") @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") @patch("tasks.jobs.scan.rls_transaction") @patch( @@ -3124,7 +3300,7 @@ class TestComplianceRequirementCopy: side_effect=Exception("copy failed"), ) def test_persist_compliance_requirement_rows_fallback_all_fields( - self, mock_copy, mock_rls_transaction, mock_bulk_create + self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_filter ): """Test ORM fallback correctly maps all fields from row dict to model.""" tenant_id = str(uuid.uuid4()) @@ -3154,7 +3330,7 @@ class TestComplianceRequirementCopy: ctx.__exit__.return_value = False mock_rls_transaction.return_value = ctx - _persist_compliance_requirement_rows(tenant_id, [row]) + _persist_compliance_requirement_rows(tenant_id, str(scan_id), lambda: [row]) args, kwargs = mock_bulk_create.call_args objects = args[0] diff --git a/api/uv.lock b/api/uv.lock index e579bb7b92..de878d9dc9 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -4673,8 +4673,8 @@ wheels = [ [[package]] name = "prowler" -version = "5.32.0" -source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#5dac8a0a53272e4db68c476fb969dc03e88beb68" } +version = "5.35.0" +source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#f5ea116763aeffede9f399c8934fc280eaccd315" } dependencies = [ { name = "alibabacloud-actiontrail20200706" }, { name = "alibabacloud-credentials" }, @@ -4762,7 +4762,7 @@ dependencies = [ [[package]] name = "prowler-api" -version = "1.36.0" +version = "1.37.0" source = { virtual = "." } dependencies = [ { name = "cartography" }, diff --git a/claude_plugins/prowler/.claude-plugin/plugin.json b/claude_plugins/prowler/.claude-plugin/plugin.json index 7bf822e2e7..c77187c3b0 100644 --- a/claude_plugins/prowler/.claude-plugin/plugin.json +++ b/claude_plugins/prowler/.claude-plugin/plugin.json @@ -22,7 +22,7 @@ "api_key": { "type": "string", "title": "Prowler API key", - "description": "API key token used to authenticate with Prowler Cloud / Prowler App via the Prowler MCP server. Create one at https://cloud.prowler.com.", + "description": "API key token used to authenticate with Prowler (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server) via the Prowler MCP server. Create one at https://cloud.prowler.com.", "sensitive": true, "required": true } diff --git a/claude_plugins/prowler/skills/framework-compliance-triage/SKILL.md b/claude_plugins/prowler/skills/framework-compliance-triage/SKILL.md index 1af29f82b9..f8a9ded0c6 100644 --- a/claude_plugins/prowler/skills/framework-compliance-triage/SKILL.md +++ b/claude_plugins/prowler/skills/framework-compliance-triage/SKILL.md @@ -38,12 +38,12 @@ If the framework is not supported, tell the user, suggest they request it or con ### 1.1 Connect to Prowler Cloud -Verify the Prowler MCP connection by calling `prowler_app_search_providers` — a successful response returns the list of providers. If the call fails, walk the user through troubleshooting: internet connectivity, Prowler Cloud credentials, and permissions on the Prowler Cloud account. +Verify the Prowler MCP connection by calling `prowler_search_providers` — a successful response returns the list of providers. If the call fails, walk the user through troubleshooting: internet connectivity, Prowler Cloud credentials, and permissions on the Prowler Cloud account. For getting accurate information about configurations use `prowler_docs_search` to pull relevant instructions from the Prowler documentation. ### 1.2 Verify the provider is configured (or configure it) -Call `prowler_app_search_providers` to check whether the target provider (AWS account, Azure Subscription, GitHub Account...) exists in the user's Prowler Cloud account. Handle the result based on what's found: +Call `prowler_search_providers` to check whether the target provider (AWS account, Azure Subscription, GitHub Account...) exists in the user's Prowler Cloud account. Handle the result based on what's found: - **Provider not present.** Guide the user through adding and configuring it. Retrieve the relevant connection, credential, and permission instructions with `prowler_docs_search`. - **Provider present but misconfigured** (missing credentials, insufficient permissions, etc.). Walk the user through fixing the configuration, pulling the relevant guidance with `prowler_docs_search`. @@ -57,15 +57,15 @@ Call `prowler_app_search_providers` to check whether the target provider (AWS ac The flow needs at least one completed scan with a compliance report available. -Look for a completed scan first: call `prowler_app_list_scans` with the selected `provider_id` and `state: ["completed"]`, then call `prowler_app_get_compliance_overview` with each `scan_id` to find one whose compliance report is available. If one is found, continue to the next section. +Look for a completed scan first: call `prowler_list_scans` with the selected `provider_id` and `state: ["completed"]`, then call `prowler_get_compliance_overview` with each `scan_id` to find one whose compliance report is available. If one is found, continue to the next section. -If no completed scan has a report, call `prowler_app_list_scans` again with `state: ["available", "executing"]` to detect a scan in progress. +If no completed scan has a report, call `prowler_list_scans` again with `state: ["available", "executing"]` to detect a scan in progress. > **Checkpoint — Scan-in-progress decision** *(conditional: an in-progress scan was detected)* > > Tell the user a scan is already running and ask whether to wait for it to complete or start a fresh one. Wait for the answer. -If no scan is running (or the user chose to start a fresh one), trigger a new scan with `prowler_app_trigger_scan` and the `provider_id`. The link `https://cloud.prowler.com/scans?filter%5Bprovider_uid__in%5D={provider_id}` lets the user monitor progress. +If no scan is running (or the user chose to start a fresh one), trigger a new scan with `prowler_trigger_scan` and the `provider_id`. The link `https://cloud.prowler.com/scans?filter%5Bprovider_uid__in%5D={provider_id}` lets the user monitor progress. When a scan is in progress (either pre-existing and elected to wait, or just triggered), stop the flow and ask the user to return when it's completed — restart this section to re-check the results. @@ -85,7 +85,7 @@ Status taxonomy for failed requirements and their findings: ### Report template -A fresh report is rendered like this (substituting values from the `prowler_app_get_compliance_framework_state_details` Prowler MCP tool response): +A fresh report is rendered like this (substituting values from the `prowler_get_compliance_framework_state_details` Prowler MCP tool response): ````markdown # Compliance report: @@ -120,7 +120,7 @@ A fresh report is rendered like this (substituting values from the `prowler_app_ Resolve the report path for the current `compliance_id` and provider account. -If the file does not exist, call `prowler_app_get_compliance_framework_state_details` for the target scan, render the template above, and write the file with one initialization entry in the activity log. +If the file does not exist, call `prowler_get_compliance_framework_state_details` for the target scan, render the template above, and write the file with one initialization entry in the activity log. If the file exists, read it and compare its `Scan ID` to the target scan from section 1.3. When the scan matches, reuse the file and summarize remaining `[FAIL]` and `[IN PROGRESS]` items in chat. @@ -128,7 +128,7 @@ If the file exists, read it and compare its `Scan ID` to the target scan from se > > Tell the user the report on disk was generated from a different scan and ask whether to refresh it from the new scan. Wait for the answer. -On confirmation, regenerate the failed-requirements section from the new `prowler_app_get_compliance_framework_state_details` response, carry forward the **Global remediation approach** block and the full activity log, and append an activity-log entry noting the scan change. +On confirmation, regenerate the failed-requirements section from the new `prowler_get_compliance_framework_state_details` response, carry forward the **Global remediation approach** block and the full activity log, and append an activity-log entry noting the scan change. Once the file is current, surface the top failing requirements in chat: sort by finding count descending, show the top 5 with their codes and counts, and point to the file path for the full list. @@ -174,7 +174,7 @@ Once approved, the loop proceeds through the batch without further prompts unles Pick the first `[FAIL]` requirement at the top of the failed-requirements section. Move its status and every finding under it to `[IN PROGRESS]`, and add a `**Fix plan**:` sub-bullet describing what will be done. -Call `prowler_app_get_finding_details` for each `finding_id` to retrieve the failing resource and the Prowler Hub's remediation guidance for that check using the tool `prowler_hub_get_check_details` with the `check_id` from the finding details. Summarize the guidance in chat, and append it to the `**Fix plan**` note for each finding. +Call `prowler_get_finding_details` for each `finding_id` to retrieve the failing resource and the Prowler Hub's remediation guidance for that check using the tool `prowler_hub_get_check_details` with the `check_id` from the finding details. Summarize the guidance in chat, and append it to the `**Fix plan**` note for each finding. If a finding does not apply to the target resource (Organization-only check on a User account, paid-tier feature, missing resource type, etc.), set the requirement status to `[SKIPPED]` with the reason, log it in the activity log, and move on without attempting the fix — even if it was missed during §3.2. @@ -194,6 +194,6 @@ Move to the next `[FAIL]` requirement and repeat from section 3.3. > **Checkpoint — Rescan trigger** *(conditional: no `[FAIL]` requirements remain; all are `[FIXED-UNVERIFIED]` or `[SKIPPED]`)* > -> Summarize what was applied, list any `[SKIPPED]` items with reasons, and ask whether to trigger a fresh scan with `prowler_app_trigger_scan` to verify the fixes end-to-end. Wait for the answer. +> Summarize what was applied, list any `[SKIPPED]` items with reasons, and ask whether to trigger a fresh scan with `prowler_trigger_scan` to verify the fixes end-to-end. Wait for the answer. On confirmation, trigger the rescan. When it completes, restart section 2.1 with the carry-forward path — requirements no longer in the new FAIL list move to `[PASS]`, anything still failing reverts to `[FAIL]` with the previous fix attempt visible in the activity log. diff --git a/docs/developer-guide/lighthouse-architecture.mdx b/docs/developer-guide/lighthouse-architecture.mdx index 631e0e7ae4..e12eff9549 100644 --- a/docs/developer-guide/lighthouse-architecture.mdx +++ b/docs/developer-guide/lighthouse-architecture.mdx @@ -132,7 +132,7 @@ The MCP client manages connections to the Prowler MCP Server using a singleton p - **Connection Management**: Retry logic with configurable attempts and delays - **Tool Discovery**: Fetches available tools from MCP server on initialization -- **Authentication Injection**: Automatically adds JWT tokens to `prowler_app_*` tool calls +- **Authentication Injection**: Automatically adds JWT tokens to `prowler_*` tool calls - **Reconnection**: Supports forced reconnection after server restarts Key constants: @@ -141,10 +141,14 @@ Key constants: - `RECONNECT_INTERVAL_MS`: 5 minutes before retry after failure ```typescript -// Authentication injection for prowler_app tools +// Authentication injection for core prowler_ tools (Hub/Docs excluded) private handleBeforeToolCall = ({ name, args }) => { - // Only inject auth for prowler_app_* tools (user-specific data) - if (!name.startsWith("prowler_app_")) { + // Only inject auth for prowler_* tools (user-specific data). + // The legacy prowler_app_ prefix is also accepted for a resilient rollout. + if ( + !name.startsWith("prowler_") && + !name.startsWith("prowler_app_") + ) { return { args }; } @@ -307,7 +311,7 @@ MCP tools are organized into three namespaces based on authentication requiremen | Namespace | Auth Required | Description | |-----------|---------------|-------------| -| `prowler_app_*` | Yes (JWT) | Prowler Cloud and Prowler Local Server tools for findings, providers, scans, resources | +| `prowler_*` | Yes (JWT) | Prowler Cloud, Prowler Private Cloud, and Prowler Local Server tools for findings, providers, scans, resources | | `prowler_hub_*` | No | Security checks catalog, compliance frameworks | | `prowler_docs_*` | No | Documentation search and retrieval | @@ -315,7 +319,7 @@ MCP tools are organized into three namespaces based on authentication requiremen 1. User authenticates with Prowler Local Server, receiving a JWT token 2. Token is stored in session and propagated via `authContextStorage` -3. MCP client injects `Authorization: Bearer ` header for `prowler_app_*` calls +3. MCP client injects `Authorization: Bearer ` header for `prowler_*` calls 4. MCP Server validates token and applies RLS filtering ### Tool Execution Pattern @@ -323,7 +327,7 @@ MCP tools are organized into three namespaces based on authentication requiremen The agent uses meta-tools rather than direct tool registration: ``` -Agent needs data → describe_tool("prowler_app_search_findings") +Agent needs data → describe_tool("prowler_search_findings") → Returns parameter schema → execute_tool with parameters → MCP client adds auth header → MCP Server executes → Results returned to agent → Agent continues reasoning diff --git a/docs/developer-guide/mcp-server.mdx b/docs/developer-guide/mcp-server.mdx index 1b9c49a18b..6d409ccc02 100644 --- a/docs/developer-guide/mcp-server.mdx +++ b/docs/developer-guide/mcp-server.mdx @@ -18,11 +18,15 @@ The Prowler MCP Server brings the entire Prowler ecosystem to AI assistants thro The server follows a modular architecture with three independent sub-servers: -| Sub-Server | Auth Required | Description | -|------------|---------------|-------------| -| `prowler_app` | Yes | Full access to Prowler Cloud and Prowler Local Server features | -| Prowler Hub | No | Security checks catalog with **over 2,000 checks**, fixers, and **70+ compliance frameworks** | -| Prowler Documentation | No | Full-text search and retrieval of official documentation | +| Sub-Server | Tool Prefix | Auth Required | Description | +|------------|-------------|---------------|-------------| +| Prowler | `prowler_` | Yes | Full access to Prowler Cloud, Prowler Private Cloud, and Prowler Local Server features | +| Prowler Hub | `prowler_hub_` | No | Security checks catalog with **over 2,000 checks**, fixers, and **70+ compliance frameworks** | +| Prowler Documentation | `prowler_docs_` | No | Full-text search and retrieval of official documentation | + + +The core Prowler sub-server is served under the `prowler_` tool prefix, while its source lives in the `prowler_app/` module for historical reasons. Tool names use the prefix; import paths use the module. + For a complete list of tools and their descriptions, see the [Tools Reference](/getting-started/basic-usage/prowler-mcp-tools). @@ -413,7 +417,7 @@ uv run prowler-mcp uv run prowler-mcp --transport http --host 0.0.0.0 --port 8000 # Run with environment variables -PROWLER_APP_API_KEY="pk_xxx" uv run prowler-mcp +PROWLER_API_KEY="pk_xxx" uv run prowler-mcp ``` For complete installation and deployment options, see: diff --git a/docs/docs.json b/docs/docs.json index d9ac4c2bd8..b9bc195182 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -72,9 +72,9 @@ "group": "Prowler MCP", "pages": [ "getting-started/products/prowler-mcp", - "getting-started/installation/prowler-mcp", "getting-started/basic-usage/prowler-mcp", - "getting-started/basic-usage/prowler-mcp-tools" + "getting-started/basic-usage/prowler-mcp-tools", + "getting-started/installation/prowler-mcp" ] }, { diff --git a/docs/getting-started/basic-usage/prowler-mcp-tools.mdx b/docs/getting-started/basic-usage/prowler-mcp-tools.mdx index d225617f0e..acb50c9e81 100644 --- a/docs/getting-started/basic-usage/prowler-mcp-tools.mdx +++ b/docs/getting-started/basic-usage/prowler-mcp-tools.mdx @@ -10,7 +10,7 @@ Complete reference guide for all tools available in the Prowler MCP Server. Tool |----------|------------|------------------------| | Prowler Hub | 10 tools | No | | Prowler Documentation | 2 tools | No | -| Prowler Cloud & Prowler Local Server | 32 tools | Yes | +| Prowler Cloud, Private Cloud & Local Server | 32 tools | Yes | ## Tool Naming Convention @@ -18,11 +18,11 @@ All tools follow a consistent naming pattern with prefixes: - `prowler_hub_*` - Prowler Hub catalog and compliance tools - `prowler_docs_*` - Prowler documentation search and retrieval -- `prowler_app_*` - Prowler Cloud and App (Self-Managed) management tools +- `prowler_*` - Prowler Cloud, Prowler Private Cloud & Prowler Local Server management tools -## Prowler Cloud and Prowler Local Server Tools +## Prowler Tools -Manage Prowler Cloud or Prowler Local Server features. **Requires authentication.** +Manage your Prowler deployment — Prowler Cloud, Prowler Private Cloud, or Prowler Local Server. **Requires authentication.** These tools require a valid API key. See the [Configuration Guide](/getting-started/basic-usage/prowler-mcp) for authentication setup. @@ -32,44 +32,44 @@ These tools require a valid API key. See the [Configuration Guide](/getting-star Tools for searching, viewing, and analyzing security findings across all cloud providers. -- **`prowler_app_search_security_findings`** - Search and filter security findings with advanced filtering options (severity, status, provider, region, service, check ID, date range, muted status) -- **`prowler_app_get_finding_details`** - Get comprehensive details about a specific finding including remediation guidance, check metadata, and resource relationships -- **`prowler_app_get_findings_overview`** - Get aggregate statistics and trends about security findings as a markdown report +- **`prowler_search_security_findings`** - Search and filter security findings with advanced filtering options (severity, status, provider, region, service, check ID, date range, muted status) +- **`prowler_get_finding_details`** - Get comprehensive details about a specific finding including remediation guidance, check metadata, and resource relationships +- **`prowler_get_findings_overview`** - Get aggregate statistics and trends about security findings as a markdown report ### Finding Groups Management Tools for listing finding groups aggregated by check ID, viewing complete group counters, and drilling down into affected resources. -- **`prowler_app_list_finding_groups`** - List latest or historical finding groups with filters for provider, region, service, resource, category, check, severity, status, muted state, delta, date range, and sorting -- **`prowler_app_get_finding_group_details`** - Get complete details for a specific finding group including counters, description, timestamps, and impacted providers -- **`prowler_app_list_finding_group_resources`** - List actionable unmuted resources affected by a finding group by default, including nested resource and provider data plus the `finding_id` for remediation details. Set `include_muted` to include suppressed resources +- **`prowler_list_finding_groups`** - List latest or historical finding groups with filters for provider, region, service, resource, category, check, severity, status, muted state, delta, date range, and sorting +- **`prowler_get_finding_group_details`** - Get complete details for a specific finding group including counters, description, timestamps, and impacted providers +- **`prowler_list_finding_group_resources`** - List actionable unmuted resources affected by a finding group by default, including nested resource and provider data plus the `finding_id` for remediation details. Set `include_muted` to include suppressed resources ### Provider Management Tools for managing cloud provider connections in Prowler. -- **`prowler_app_search_providers`** - Search and view configured providers with their connection status -- **`prowler_app_connect_provider`** - Register and connect a provider with credentials for security scanning -- **`prowler_app_delete_provider`** - Permanently remove a provider from Prowler +- **`prowler_search_providers`** - Search and view configured providers with their connection status +- **`prowler_connect_provider`** - Register and connect a provider with credentials for security scanning +- **`prowler_delete_provider`** - Permanently remove a provider from Prowler ### Scan Management Tools for managing and monitoring security scans. -- **`prowler_app_list_scans`** - List and filter security scans across all providers -- **`prowler_app_get_scan`** - Get comprehensive details about a specific scan (progress, duration, resource counts) -- **`prowler_app_trigger_scan`** - Trigger a manual security scan for a provider -- **`prowler_app_schedule_daily_scan`** - Schedule automated daily scans for continuous monitoring -- **`prowler_app_update_scan`** - Update scan name for better organization +- **`prowler_list_scans`** - List and filter security scans across all providers +- **`prowler_get_scan`** - Get comprehensive details about a specific scan (progress, duration, resource counts) +- **`prowler_trigger_scan`** - Trigger a manual security scan for a provider +- **`prowler_schedule_daily_scan`** - Schedule automated daily scans for continuous monitoring +- **`prowler_update_scan`** - Update scan name for better organization ### Resources Management Tools for searching, viewing, and analyzing cloud resources discovered by Prowler. -- **`prowler_app_list_resources`** - List and filter cloud resources with advanced filtering options (provider, region, service, resource type, tags) -- **`prowler_app_get_resource`** - Get comprehensive details about a specific resource including configuration, metadata, and finding relationships -- **`prowler_app_get_resource_events`** - Get the timeline of cloud API actions performed on a resource (AWS CloudTrail). Shows who did what and when, with full request/response payloads -- **`prowler_app_get_resources_overview`** - Get aggregate statistics about cloud resources as a markdown report +- **`prowler_list_resources`** - List and filter cloud resources with advanced filtering options (provider, region, service, resource type, tags) +- **`prowler_get_resource`** - Get comprehensive details about a specific resource including configuration, metadata, and finding relationships +- **`prowler_get_resource_events`** - Get the timeline of cloud API actions performed on a resource (AWS CloudTrail). Shows who did what and when, with full request/response payloads +- **`prowler_get_resources_overview`** - Get aggregate statistics about cloud resources as a markdown report ### Muting Management @@ -77,33 +77,33 @@ Tools for managing finding muting, including pattern-based bulk muting (mutelist #### Mutelist (Pattern-Based Muting) -- **`prowler_app_get_mutelist`** - Retrieve the current mutelist configuration for the tenant -- **`prowler_app_set_mutelist`** - Create or update the mutelist configuration for pattern-based bulk muting -- **`prowler_app_delete_mutelist`** - Remove the mutelist configuration from the tenant +- **`prowler_get_mutelist`** - Retrieve the current mutelist configuration for the tenant +- **`prowler_set_mutelist`** - Create or update the mutelist configuration for pattern-based bulk muting +- **`prowler_delete_mutelist`** - Remove the mutelist configuration from the tenant #### Mute Rules (Finding-Specific Muting) -- **`prowler_app_list_mute_rules`** - Search and filter mute rules with pagination support -- **`prowler_app_get_mute_rule`** - Retrieve comprehensive details about a specific mute rule -- **`prowler_app_create_mute_rule`** - Create a new mute rule to mute specific findings with documentation and audit trail -- **`prowler_app_update_mute_rule`** - Update a mute rule's name, reason, or enabled status -- **`prowler_app_delete_mute_rule`** - Delete a mute rule from the system +- **`prowler_list_mute_rules`** - Search and filter mute rules with pagination support +- **`prowler_get_mute_rule`** - Retrieve comprehensive details about a specific mute rule +- **`prowler_create_mute_rule`** - Create a new mute rule to mute specific findings with documentation and audit trail +- **`prowler_update_mute_rule`** - Update a mute rule's name, reason, or enabled status +- **`prowler_delete_mute_rule`** - Delete a mute rule from the system ### Attack Paths Analysis Tools for analyzing privilege escalation chains and security misconfigurations using graph-based analysis. Attack Paths maps relationships between cloud resources, permissions, and security findings to detect how privileges can be escalated and how misconfigurations can be exploited. -- **`prowler_app_list_attack_paths_scans`** - List Attack Paths scans with filtering by provider, provider type, and scan state (available, scheduled, executing, completed, failed, cancelled) -- **`prowler_app_list_attack_paths_queries`** - Discover available Attack Paths queries for a completed scan, including query names, descriptions, and required parameters -- **`prowler_app_run_attack_paths_query`** - Execute an Attack Paths query against a completed scan and retrieve graph results with nodes (cloud resources, findings, virtual nodes) and relationships (access paths, role assumptions, security group memberships) -- **`prowler_app_get_attack_paths_cartography_schema`** - Retrieve the Cartography graph schema (node labels, relationships, properties) for writing accurate custom openCypher queries +- **`prowler_list_attack_paths_scans`** - List Attack Paths scans with filtering by provider, provider type, and scan state (available, scheduled, executing, completed, failed, cancelled) +- **`prowler_list_attack_paths_queries`** - Discover available Attack Paths queries for a completed scan, including query names, descriptions, and required parameters +- **`prowler_run_attack_paths_query`** - Execute an Attack Paths query against a completed scan and retrieve graph results with nodes (cloud resources, findings, virtual nodes) and relationships (access paths, role assumptions, security group memberships) +- **`prowler_get_attack_paths_cartography_schema`** - Retrieve the Cartography graph schema (node labels, relationships, properties) for writing accurate custom openCypher queries ### Compliance Management Tools for viewing compliance status and framework details across all cloud providers. -- **`prowler_app_get_compliance_overview`** - Get high-level compliance status across all frameworks for a specific scan or provider, including pass/fail statistics per framework -- **`prowler_app_get_compliance_framework_state_details`** - Get detailed requirement-level breakdown for a specific compliance framework, including failed requirements and associated finding IDs +- **`prowler_get_compliance_overview`** - Get high-level compliance status across all frameworks for a specific scan or provider, including pass/fail statistics per framework +- **`prowler_get_compliance_framework_state_details`** - Get detailed requirement-level breakdown for a specific compliance framework, including failed requirements and associated finding IDs ## Prowler Hub Tools @@ -145,7 +145,7 @@ Search and access official Prowler documentation. **No authentication required.* - Use natural language to interact with the tools through your AI assistant - Tools can be combined for complex workflows - Filter options are available on most list tools -- Authentication is only required for Prowler Cloud and Prowler Local Server tools +- Authentication is only required for Prowler tools (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server) ## Additional Resources diff --git a/docs/getting-started/basic-usage/prowler-mcp.mdx b/docs/getting-started/basic-usage/prowler-mcp.mdx index a06f54375e..120e5c038b 100644 --- a/docs/getting-started/basic-usage/prowler-mcp.mdx +++ b/docs/getting-started/basic-usage/prowler-mcp.mdx @@ -7,10 +7,10 @@ Configure your MCP client to connect to Prowler MCP Server. ## Step 1: Get Your API Key -**Authentication is optional**: Prowler Hub and Prowler Documentation features work without authentication. An API key is only required for Prowler Cloud and Prowler Local Server features. +**Authentication is optional**: Prowler Hub and Prowler Documentation features work without authentication. An API key is only required for Prowler tools (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server). -To use Prowler Cloud or Prowler Local Server features. To get the API key, please refer to the [API Keys](/user-guide/tutorials/prowler-app-api-keys) guide. +An API key authenticates the Prowler tools (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server). To get the API key, please refer to the [API Keys](/user-guide/tutorials/prowler-app-api-keys) guide. Keep the API key secure. Never share it publicly or commit it to version control. @@ -18,12 +18,14 @@ Keep the API key secure. Never share it publicly or commit it to version control ## Step 2: Configure Your MCP Host/Client -Choose the configuration based on your deployment: +Most users should use the **Cloud MCP Server** — it needs no installation and is maintained by Prowler. The [Local MCP Server](#local-mcp-server-configuration) configuration is provided afterwards for users who run the server themselves. -- **HTTP Mode**: Prowler Cloud MCP Server or self-hosted Prowler MCP Server. -- **STDIO Mode**: Local installation only (runs as subprocess of your MCP client). +- **Cloud MCP Server (HTTP)**: the managed server at `https://mcp.prowler.com/mcp` (or your own self-hosted HTTP server). +- **Local MCP Server (STDIO)**: local installation only (runs as a subprocess of your MCP client). -### HTTP Mode +## Cloud MCP Server Configuration (Recommended) + +Connect to the **Cloud MCP Server** at `https://mcp.prowler.com/mcp` over HTTP. This is the recommended path — no installation, always up to date. The same configuration works for a self-hosted HTTP server: just swap the URL. @@ -61,10 +63,10 @@ Choose the configuration based on your deployment: "args": [ "https://mcp.prowler.com/mcp", // or your self-hosted Prowler MCP Server URL "--header", - "Authorization: Bearer ${PROWLER_APP_API_KEY}" + "Authorization: Bearer ${PROWLER_API_KEY}" ], "env": { - "PROWLER_APP_API_KEY": "" + "PROWLER_API_KEY": "" } } } @@ -96,10 +98,10 @@ Choose the configuration based on your deployment: "args": [ "https://mcp.prowler.com/mcp", "--header", - "Authorization: Bearer ${PROWLER_APP_API_KEY}" + "Authorization: Bearer ${PROWLER_API_KEY}" ], "env": { - "PROWLER_APP_API_KEY": "" + "PROWLER_API_KEY": "" } } } @@ -110,8 +112,8 @@ Choose the configuration based on your deployment: Run the following command: ```bash - export PROWLER_APP_API_KEY="" - claude mcp add --transport http prowler https://mcp.prowler.com/mcp --header "Authorization: Bearer $PROWLER_APP_API_KEY" --scope user + export PROWLER_API_KEY="" + claude mcp add --transport http prowler https://mcp.prowler.com/mcp --header "Authorization: Bearer $PROWLER_API_KEY" --scope user ``` @@ -137,9 +139,9 @@ Choose the configuration based on your deployment: -### STDIO Mode +## Local MCP Server Configuration -STDIO mode is only available when running the MCP server locally. +STDIO mode is only available when running the **Local MCP Server** on your own machine. See the [Installation guide](/getting-started/installation/prowler-mcp) to set it up first. @@ -152,7 +154,7 @@ STDIO mode is only available when running the MCP server locally. "command": "uvx", "args": ["/absolute/path/to/prowler/mcp_server/"], "env": { - "PROWLER_APP_API_KEY": "", + "PROWLER_API_KEY": "", "API_BASE_URL": "https://api.prowler.com/api/v1" } } @@ -179,7 +181,7 @@ STDIO mode is only available when running the MCP server locally. "--rm", "-i", "--env", - "PROWLER_APP_API_KEY=", + "PROWLER_API_KEY=", "--env", "API_BASE_URL=https://api.prowler.com/api/v1", "prowlercloud/prowler-mcp" @@ -205,7 +207,7 @@ Restart your MCP client and start asking questions: ## Authentication Methods -Prowler MCP Server supports two authentication methods to connect to Prowler Cloud or Prowler Local Server: +Prowler MCP Server supports two authentication methods to connect to Prowler (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server): ### API Key (Recommended) diff --git a/docs/getting-started/installation/prowler-app.mdx b/docs/getting-started/installation/prowler-app.mdx index 0c2d032315..1994b8ad0f 100644 --- a/docs/getting-started/installation/prowler-app.mdx +++ b/docs/getting-started/installation/prowler-app.mdx @@ -128,8 +128,8 @@ To update the environment file: Edit the `.env` file and change version values: ```env -PROWLER_UI_VERSION="5.34.0" -PROWLER_API_VERSION="5.34.0" +PROWLER_UI_VERSION="5.35.0" +PROWLER_API_VERSION="5.35.0" ``` diff --git a/docs/getting-started/installation/prowler-mcp.mdx b/docs/getting-started/installation/prowler-mcp.mdx index 06f204da6a..5be469b5b2 100644 --- a/docs/getting-started/installation/prowler-mcp.mdx +++ b/docs/getting-started/installation/prowler-mcp.mdx @@ -5,12 +5,12 @@ title: "Installation" There are **two ways** to use Prowler MCP Server: - + **No installation required** - Just configuration Use `https://mcp.prowler.com/mcp` - + **Local installation** - Full control Install via Docker, PyPI, or source code @@ -18,8 +18,8 @@ There are **two ways** to use Prowler MCP Server: -For "Option 1: Managed by Prowler", go directly to the [Configuration Guide](/getting-started/basic-usage/prowler-mcp#hosted-server-configuration-recommended) to set up your Claude Desktop, Cursor, or other MCP client. -**This guide is focused on local installation, "Option 2: Run Locally"**. +For the Cloud MCP Server, go directly to the [Configuration Guide](/getting-started/basic-usage/prowler-mcp#cloud-mcp-server-configuration-recommended) to set up your Claude Desktop, Cursor, or other MCP client. +**This guide is focused on local installation, the Local MCP Server**. ## Installation Methods @@ -51,7 +51,7 @@ Choose one of the following installation methods: ```bash docker run --rm -i \ - -e PROWLER_APP_API_KEY="pk_your_api_key" \ + -e PROWLER_API_KEY="pk_your_api_key" \ -e API_BASE_URL="https://api.prowler.com/api/v1" \ prowlercloud/prowler-mcp ``` @@ -143,7 +143,7 @@ Choose one of the following installation methods: ## Updating Prowler MCP Server -When running Prowler MCP Server locally ("Option 2: Run Locally"), upgrade to the latest version using the same method chosen for installation. The hosted server (`https://mcp.prowler.com/mcp`) is always kept up to date by Prowler and requires no action. +When running the Local MCP Server, upgrade to the latest version using the same method chosen for installation. The Cloud MCP Server (`https://mcp.prowler.com/mcp`) is always kept up to date by Prowler and requires no action. @@ -219,19 +219,19 @@ Configure the server using environment variables: | Variable | Description | Required | Default | |----------|-------------|----------|---------| -| `PROWLER_APP_API_KEY` | Prowler API key | Only for STDIO mode | - | +| `PROWLER_API_KEY` | Prowler API key | Only for STDIO mode | - | | `API_BASE_URL` | Custom Prowler API endpoint | No | `https://api.prowler.com/api/v1` | | `PROWLER_MCP_TRANSPORT_MODE` | Default transport mode (overwritten by `--transport` argument) | No | `stdio` | ```bash macOS/Linux -export PROWLER_APP_API_KEY="pk_your_api_key_here" +export PROWLER_API_KEY="pk_your_api_key_here" export API_BASE_URL="https://api.prowler.com/api/v1" export PROWLER_MCP_TRANSPORT_MODE="http" ``` ```bash Windows PowerShell -$env:PROWLER_APP_API_KEY="pk_your_api_key_here" +$env:PROWLER_API_KEY="pk_your_api_key_here" $env:API_BASE_URL="https://api.prowler.com/api/v1" $env:PROWLER_MCP_TRANSPORT_MODE="http" ``` @@ -246,7 +246,7 @@ Never commit your API key to version control. Use environment variables or secur For convenience, create a `.env` file in the `mcp_server` directory: ```bash .env -PROWLER_APP_API_KEY=pk_your_api_key_here +PROWLER_API_KEY=pk_your_api_key_here API_BASE_URL=https://api.prowler.com/api/v1 PROWLER_MCP_TRANSPORT_MODE=stdio ``` diff --git a/docs/getting-started/products/prowler-cloud-lighthouse.mdx b/docs/getting-started/products/prowler-cloud-lighthouse.mdx index 04bfd58224..d32118d468 100644 --- a/docs/getting-started/products/prowler-cloud-lighthouse.mdx +++ b/docs/getting-started/products/prowler-cloud-lighthouse.mdx @@ -24,6 +24,9 @@ The Agentic Cloud Defender does more than answer questions, it helps teams **fin Switch between the standard interface and a chat-first agentic view. + + Open Lighthouse AI as a side panel from any page to get help in context. + Credentials are validated automatically when a provider is configured. @@ -37,6 +40,20 @@ Promoting the chat to a top-level view gives Lighthouse AI the room it needs for Lighthouse AI chat view in Prowler Cloud +### Side Panel + +You do not have to switch to the full chat view to reach Lighthouse AI. A side panel is available on every page of Prowler Cloud. While collapsed it stays out of the way; open it from any dashboard, findings list, or configuration screen to ask questions without leaving what you are working on. Open it using the Lighthouse AI button, circled in red in the image below. + +Collapsed Lighthouse AI side panel on a Prowler Cloud page, with the button to open it circled in red + +Once open, the panel slides in alongside your current page and shares the same agent, tools, and persistent chat sessions as the full Chat View, so a conversation started in the panel can be reopened and continued later from either place. + +Lighthouse AI side panel open alongside a Prowler Cloud page + +- **Available everywhere:** Summon the assistant from any page while you keep working in the normal view. +- **Context-aware help:** Ask about the findings, resources, or compliance data you are currently looking at. +- **Continuous sessions:** Conversations opened in the side panel are saved alongside the rest of your chat history. + ### Tool Usage Lighthouse AI on Prowler Cloud renders the agent's work as it happens, so responses are easier to follow and to trust. Tool calls and reasoning steps appear in the order they occur within the conversation. @@ -63,6 +80,53 @@ At the top of the configuration page, the optional **Business Context** field le Lighthouse AI on Prowler Cloud supports OpenAI, Amazon Bedrock, and OpenAI-compatible providers, with GPT-5.5 as the default. For per-provider setup and how to switch the default provider or model, see [Using Multiple LLM Providers](/user-guide/tutorials/prowler-cloud-lighthouse-multi-llm). +## Capabilities + +Lighthouse AI works through the [Prowler MCP Server](/getting-started/products/prowler-mcp), which gives the agent a growing catalog of tools to explore and act on your security data. These actions run inside Prowler and never modify your cloud resources. Everything the agent can do maps to one of the following capability areas. + +### Findings and Finding Groups + +- Search and filter security findings across every connected provider by severity, status, region, service, check, date range, and muted state. +- Retrieve full finding details, including remediation guidance, check metadata, and affected resources. +- Summarize findings with aggregate statistics and trends. +- Browse finding groups aggregated by check and drill down into the specific resources each group affects. + +### Resources + +- List and filter cloud resources by provider, region, service, resource type, and tags. +- Inspect a resource's configuration, metadata, and related findings. +- Review the timeline of cloud API actions performed on a resource (AWS CloudTrail), including who did what and when. +- Get an aggregate overview of the resources Prowler has discovered. + +### Compliance + +- Review high-level compliance status across all frameworks, with pass/fail statistics per framework. +- Get a requirement-level breakdown for a specific framework, including failed requirements and their associated findings. + +### Attack Paths + +- List Attack Paths scans and discover the queries available for each completed scan. +- Run graph-based queries to reveal privilege-escalation chains and exploitable misconfigurations. +- Retrieve the Cartography graph schema to build accurate custom queries. + +### Scans and Providers + +- List, inspect, and rename security scans across providers. +- Trigger manual scans and schedule automated daily scans for continuous monitoring. +- Search connected providers and check their connection status, connect new providers, or remove existing ones. + +### Muting + +- Manage the mutelist for pattern-based bulk muting. +- Create, update, list, and delete finding-specific mute rules, each with a documented reason and audit trail. + +### Security Check Catalog and Documentation + +- Browse and search the Prowler Hub catalog of security checks and compliance frameworks, including check code and automated fixers. +- Search and retrieve official Prowler documentation to answer how-to and product questions. + +For the complete list of underlying tools, see the [Prowler MCP Tools Reference](/getting-started/basic-usage/prowler-mcp-tools). + ## FAQ **Which LLM providers are supported?** @@ -71,7 +135,11 @@ OpenAI (GPT models, including the default GPT-5.5), Amazon Bedrock (Claude, Llam **Can Lighthouse AI change my cloud environment?** -No. Lighthouse AI has read-only access to security data and no tools to modify resources, even when the connected cloud credentials would allow changes. +No. Lighthouse AI cannot modify the resources in your connected cloud providers (AWS, Azure, GCP, and others). It has read-only access to that environment and no tools to change it, even when the connected cloud credentials would allow it. + +**Can Lighthouse AI change my Prowler Cloud environment?** + +Yes. Lighthouse AI can take action within Prowler Cloud itself, such as connecting or removing providers, triggering and scheduling scans, and managing mute rules and the mutelist. See [Capabilities](#capabilities) for the full list of what it can do. These actions only affect your Prowler Cloud workspace, never the resources in your cloud providers. ## Looking for the Open Source Version? diff --git a/docs/getting-started/products/prowler-mcp.mdx b/docs/getting-started/products/prowler-mcp.mdx index b35ac3d611..93159b2e7d 100644 --- a/docs/getting-started/products/prowler-mcp.mdx +++ b/docs/getting-started/products/prowler-mcp.mdx @@ -8,8 +8,29 @@ title: "Overview" **Preview Feature**: This MCP server is currently under active development. Features and functionality may change. We welcome your feedback—please report any issues on [GitHub](https://github.com/prowler-cloud/prowler/issues) or join our [Slack community](https://goto.prowler.com/slack) to discuss and share your thoughts. +## Quickest Way to Connect: Cloud MCP Server + +The fastest way to get started is the **Cloud MCP Server** at `https://mcp.prowler.com/mcp` — no installation, always up to date, and maintained by Prowler. Just point your MCP client at the URL and authenticate with a [Prowler API key](/user-guide/tutorials/prowler-app-api-keys) as a Bearer token: + +```json +{ + "mcpServers": { + "prowler": { + "url": "https://mcp.prowler.com/mcp", + "headers": { + "Authorization": "Bearer " + } + } + } +} +``` + + + Step-by-step setup for Claude Desktop, Claude Code, Cursor, and other clients. + + -Prowler MCP Server can run as a local instance, or as the hosted **Prowler MCP** at `https://mcp.prowler.com/mcp`. The hosted server also provides tools for Prowler Cloud-specific features such as [Alerts](/user-guide/tutorials/prowler-alerts), [Scan Scheduling](/user-guide/tutorials/prowler-scan-scheduling), and [Findings Triage](/user-guide/tutorials/prowler-app-findings-triage). See [Deployment Options](#deployment-options). +Prefer to run it yourself? The **Local MCP Server** runs on your own machine or infrastructure. The Cloud MCP Server additionally provides tools for Prowler Cloud-specific features such as [Alerts](/user-guide/tutorials/prowler-alerts), [Scan Scheduling](/user-guide/tutorials/prowler-scan-scheduling), and [Findings Triage](/user-guide/tutorials/prowler-app-findings-triage). See [Cloud vs Local MCP Server](#cloud-vs-local-mcp-server). ## What is the Model Context Protocol? @@ -20,9 +41,9 @@ The [Model Context Protocol (MCP)](https://modelcontextprotocol.io) is an open s The Prowler MCP Server provides three main integration points: -### 1. Prowler Cloud and Prowler Local Server +### 1. Prowler Cloud, Private Cloud & Local Server -Full access to Prowler Cloud and Prowler Local Server for: +Full access to your Prowler deployment — Prowler Cloud, Prowler Private Cloud, or Prowler Local Server — for: - **Findings Analysis**: Query, filter, and analyze security findings across all your cloud environments - **Provider Management**: Create, configure, and manage your configured Prowler providers (AWS, Azure, GCP, etc.) - **Scan Orchestration**: Trigger on-demand scans and schedule recurring security assessments @@ -48,12 +69,53 @@ Search and retrieve official Prowler documentation: ## MCP Server Architecture -The following diagram illustrates the Prowler MCP Server architecture and its integration points: +The following diagram illustrates the Prowler MCP Server architecture and its integration points. MCP clients connect to either the **Cloud MCP Server** (recommended) or a **Local MCP Server**; both expose the same tools and reach the same Prowler backends: -![Prowler MCP Server Schema](/images/prowler_mcp_schema.png) +```mermaid +flowchart LR + subgraph HOSTS["MCP Clients"] + chat["Chat Interfaces
(Claude Desktop, LobeChat)"] + ide["IDEs and Code Editors
(Claude Code, Cursor)"] + apps["Other AI Applications
(5ire, custom agents)"] + end + + subgraph SERVERS["Prowler MCP Server"] + direction TB + cloud["☁️ Cloud MCP Server (Recommended)
mcp.prowler.com/mcp · HTTP
Managed by Prowler · always up to date
Adds Cloud-only tools (Alerts,
Scan Scheduling, Findings Triage)"] + local["💻 Local MCP Server
Self-run · STDIO or HTTP
Python 3.12+ or Docker
You manage updates"] + end + + subgraph TOOLS["Prowler MCP Tools"] + prowler_tools["prowler_* tools
(API key or JWT auth)
Findings · Providers · Scans
Resources · Muting · Compliance
Attack Paths"] + hub_tools["prowler_hub_* tools
(no auth)
Checks Catalog · Check Code
Fixers · Compliance Frameworks"] + docs_tools["prowler_docs_* tools
(no auth)
Search · Document Retrieval"] + end + + api["Prowler API (REST)
Cloud · Private Cloud · Local Server"] + hub["hub.prowler.com
(REST)"] + docs["docs.prowler.com
(Mintlify)"] + + chat -->|HTTP| cloud + ide -->|HTTP| cloud + apps -->|HTTP| cloud + chat -->|STDIO or HTTP| local + ide -->|STDIO or HTTP| local + apps -->|STDIO or HTTP| local + + cloud --> prowler_tools + cloud --> hub_tools + cloud --> docs_tools + local --> prowler_tools + local --> hub_tools + local --> docs_tools + + prowler_tools -->|REST| api + hub_tools -->|REST| hub + docs_tools -->|REST| docs +``` The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components: -- Prowler Cloud and Prowler Local Server for security operations +- Prowler Cloud, Prowler Private Cloud, or Prowler Local Server for security operations - Prowler Hub for security knowledge - Prowler Documentation for guidance and reference. @@ -92,8 +154,8 @@ REQUIREMENTS: DATA TO FETCH: Use these MCP tools in this order: -1. Prowler app list providers - To get all available configured provider in the account -2. Prowler app get latest findings - To get findings information, if there are so many you can use the filter_fields to get less information, or pagination to get in different batches +1. Prowler list providers - To get all available configured provider in the account +2. Prowler get latest findings - To get findings information, if there are so many you can use the filter_fields to get less information, or pagination to get in different batches 3. For most critical findings you can get more context and remediation with Prowler Hub to get remediations for example DESIGN REQUIREMENTS: @@ -130,66 +192,48 @@ Generate the complete HTML file and display it > -## Deployment Options +## Cloud vs Local MCP Server -Prowler MCP Server can be used in three ways: +There are two ways to run the Prowler MCP Server. For almost everyone, the **Cloud MCP Server** is the right choice — it needs no installation and is maintained by Prowler. The **Local MCP Server** exists for users who need to run it on their own machine or infrastructure. -### 1. Prowler Cloud MCP Server +| | ☁️ **Cloud MCP Server** (Recommended) | 💻 **Local MCP Server** | +|---|---|---| +| **Endpoint** | `https://mcp.prowler.com/mcp` | Runs on your machine or infrastructure | +| **Setup** | Just configure your MCP client | Install via Docker, or source | +| **Transport** | HTTP | STDIO (subprocess) or self-hosted HTTP | +| **Maintenance** | Managed by Prowler, always up to date | You manage updates | +| **Requirements** | None (just an MCP client) | Python 3.12+ or Docker | +| **Cloud-only tools** | ✅ Alerts, Scan Scheduling, Findings Triage | ❌ Not available | +| **Authentication** | API key or JWT token | API key/JWT (HTTP) or env vars (STDIO) | -**Use Prowler's managed MCP server at `https://mcp.prowler.com/mcp`** +### ☁️ Cloud MCP Server (Recommended) -- No installation required. -- Managed and maintained by Prowler team. -- Authentication to Prowler Cloud or Prowler Local Server via API key or JWT token. -- Includes tools for Prowler Cloud-specific features such as Alerts, Scan Scheduling, and Findings Triage. +Prowler's managed MCP server at `https://mcp.prowler.com/mcp`. No installation, always up to date, and it includes tools for Prowler Cloud-specific features such as Alerts, Scan Scheduling, and Findings Triage. This is the path we recommend for nearly all users — go straight to the [Configuration guide](/getting-started/basic-usage/prowler-mcp#cloud-mcp-server-configuration-recommended). -### 2. Local STDIO Mode +### 💻 Local MCP Server -**Run the server locally on your machine** +Run the server yourself when you need full control over the deployment. It connects to Prowler Cloud, Prowler Private Cloud, or Prowler Local Server and can run in two modes: -- Runs as a subprocess of the MCP client. -- Possibility to connect to Prowler Local Server. -- Authentication to Prowler Cloud or Prowler Local Server via environment variables. -- Requires Python 3.12+ or Docker. +- **STDIO mode** — the server runs as a subprocess of your MCP client. Authentication via environment variables. +- **Self-hosted HTTP mode** — deploy your own remote HTTP server. Authentication via API key or JWT token. -### 3. Self-Hosted HTTP Mode - -**Deploy your own remote MCP server** - -- Full control over deployment. -- Possibility to connect to Prowler Local Server. -- Authentication to Prowler Local Server via API key or JWT token. -- Requires Python 3.12+ or Docker. - -## Requirements - -Requirements vary based on deployment option: - -**For Prowler Cloud MCP Server:** -- Prowler Cloud account and API key (only for Prowler Cloud and Prowler Local Server features) - -**For self-hosted STDIO/HTTP Mode:** -- Python 3.12+ or Docker -- Network access to: - - `https://hub.prowler.com` (for Prowler Hub) - - `https://docs.prowler.com` (for Prowler Documentation) - - Prowler Cloud API or Prowler Local Server API (for Prowler Cloud and Prowler Local Server features) +Both require Python 3.12+ or Docker, plus network access to `https://hub.prowler.com` (Prowler Hub), `https://docs.prowler.com` (Prowler Documentation), and the Prowler API or Prowler Local Server API (Prowler features). See the [Installation guide](/getting-started/installation/prowler-mcp) to get started. -**No Authentication Required**: Prowler Hub and Prowler Documentation features work without authentication in both deployment options. A Prowler API key is only required to access Prowler Cloud or Prowler Local Server features. +**No Authentication Required**: Prowler Hub and Prowler Documentation features work without authentication on both the Cloud and Local MCP Server. A Prowler API key is only required to access Prowler features (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server). ## Next Steps - - Install the Prowler MCP Server using uv or Docker - - Configure your MCP client to connect to the server + Connect your MCP client to the Cloud MCP Server + + + Explore all available tools and capabilities - - Explore all available tools and capabilities + + Run the Local MCP Server yourself using Docker, source, or uvx diff --git a/docs/images/lighthouse-architecture.mmd b/docs/images/lighthouse-architecture.mmd index 47407544e9..6798801fb8 100644 --- a/docs/images/lighthouse-architecture.mmd +++ b/docs/images/lighthouse-architecture.mmd @@ -15,7 +15,7 @@ flowchart TB llm["LLM Provider
(OpenAI / Bedrock / OpenAI-compatible)"] subgraph MCP["Prowler MCP Server"] - app_tools["prowler_app_* tools
(auth required)"] + app_tools["prowler_* tools
(auth required)"] hub_tools["prowler_hub_* tools
(no auth)"] docs_tools["prowler_docs_* tools
(no auth)"] end @@ -29,7 +29,7 @@ flowchart TB agent <-->|LLM API| llm agent --> metatools metatools --> mcpclient - mcpclient -->|MCP HTTP · Bearer token
for prowler_app_* only| app_tools + mcpclient -->|MCP HTTP · Bearer token
for prowler_* only| app_tools mcpclient -->|MCP HTTP| hub_tools mcpclient -->|MCP HTTP| docs_tools app_tools -->|REST| api diff --git a/docs/images/organizations/authentication-details.png b/docs/images/organizations/authentication-details.png index 2b4ae782cf..aec5060afd 100644 Binary files a/docs/images/organizations/authentication-details.png and b/docs/images/organizations/authentication-details.png differ diff --git a/docs/images/organizations/onboarding-flow.svg b/docs/images/organizations/onboarding-flow.svg index f6e11fc0a3..b5ba7858a0 100644 --- a/docs/images/organizations/onboarding-flow.svg +++ b/docs/images/organizations/onboarding-flow.svg @@ -3,41 +3,37 @@ + + + Onboarding Flow - + 1 - Create Management - Account Role - - Quick Create or Manual - Allows Prowler to - discover your org - structure + Start the Wizard + + In Prowler Cloud + Enter your Org ID + and OU/root target - - - - - 2 - Deploy StackSet - - In AWS Console - Creates ProwlerScan - role in every - member account + Deploy the Roles + + Single CF Stack + Management role + + StackSet to members + in one CF stack @@ -46,11 +42,11 @@ 3 - Run the Wizard - - In Prowler Cloud - Discovers accounts, - tests connections + Discover & Connect + + In Prowler Cloud + Discovers accounts, + tests connections @@ -59,13 +55,13 @@ 4 - Launch Scans - - Automatic - Scans run on all - connected accounts - on your schedule + Launch Scans + + Automatic + Scans run on all + connected accounts + on your schedule - Steps 1 and 2 are done once in AWS | Steps 3 and 4 are done in Prowler Cloud + Step 2 runs once in AWS | Steps 1, 3 and 4 are in Prowler Cloud diff --git a/docs/images/organizations/two-roles-architecture.svg b/docs/images/organizations/two-roles-architecture.svg index c67588b049..f8c40d5b21 100644 --- a/docs/images/organizations/two-roles-architecture.svg +++ b/docs/images/organizations/two-roles-architecture.svg @@ -47,7 +47,7 @@ - Deploy: Quick Create link or Manual + Deploy: single stack or standalone @@ -86,7 +86,7 @@ - Deploy: via CloudFormation StackSet + Deploy: StackSet (single stack) Prowler discovers diff --git a/docs/images/prowler-app/lighthouse/prowler-cloud/side-panel-closed.png b/docs/images/prowler-app/lighthouse/prowler-cloud/side-panel-closed.png new file mode 100644 index 0000000000..c36da34a95 Binary files /dev/null and b/docs/images/prowler-app/lighthouse/prowler-cloud/side-panel-closed.png differ diff --git a/docs/images/prowler-app/lighthouse/prowler-cloud/side-panel-open.png b/docs/images/prowler-app/lighthouse/prowler-cloud/side-panel-open.png new file mode 100644 index 0000000000..fd10f5a00b Binary files /dev/null and b/docs/images/prowler-app/lighthouse/prowler-cloud/side-panel-open.png differ diff --git a/docs/images/prowler_mcp_schema.mmd b/docs/images/prowler_mcp_schema.mmd index 96973546f6..2b8508cd18 100644 --- a/docs/images/prowler_mcp_schema.mmd +++ b/docs/images/prowler_mcp_schema.mmd @@ -1,29 +1,40 @@ flowchart LR - subgraph HOSTS["MCP Hosts"] + subgraph HOSTS["MCP Clients"] chat["Chat Interfaces
(Claude Desktop, LobeChat)"] ide["IDEs and Code Editors
(Claude Code, Cursor)"] apps["Other AI Applications
(5ire, custom agents)"] end - subgraph MCP["Prowler MCP Server"] - app_tools["prowler_app_* tools
(JWT or API key auth)
Findings · Providers · Scans
Resources · Muting · Compliance
Attack Paths"] + subgraph SERVERS["Prowler MCP Server"] + direction TB + cloud["☁️ Cloud MCP Server (Recommended)
mcp.prowler.com/mcp · HTTP
Managed by Prowler · always up to date
Adds Cloud-only tools (Alerts,
Scan Scheduling, Findings Triage)"] + local["💻 Local MCP Server
Self-run · STDIO or HTTP
Python 3.12+ or Docker
You manage updates"] + end + + subgraph TOOLS["Prowler MCP Tools"] + prowler_tools["prowler_* tools
(API key or JWT auth)
Findings · Providers · Scans
Resources · Muting · Compliance
Attack Paths"] hub_tools["prowler_hub_* tools
(no auth)
Checks Catalog · Check Code
Fixers · Compliance Frameworks"] docs_tools["prowler_docs_* tools
(no auth)
Search · Document Retrieval"] end - api["Prowler API
(REST)"] + api["Prowler API (REST)
Cloud · Private Cloud · Local Server"] hub["hub.prowler.com
(REST)"] docs["docs.prowler.com
(Mintlify)"] - chat -->|STDIO or HTTP| app_tools - chat -->|STDIO or HTTP| hub_tools - chat -->|STDIO or HTTP| docs_tools - ide -->|STDIO or HTTP| app_tools - ide -->|STDIO or HTTP| hub_tools - ide -->|STDIO or HTTP| docs_tools - apps -->|STDIO or HTTP| app_tools - apps -->|STDIO or HTTP| hub_tools - apps -->|STDIO or HTTP| docs_tools - app_tools -->|REST| api + chat -->|HTTP| cloud + ide -->|HTTP| cloud + apps -->|HTTP| cloud + chat -->|STDIO or HTTP| local + ide -->|STDIO or HTTP| local + apps -->|STDIO or HTTP| local + + cloud --> prowler_tools + cloud --> hub_tools + cloud --> docs_tools + local --> prowler_tools + local --> hub_tools + local --> docs_tools + + prowler_tools -->|REST| api hub_tools -->|REST| hub docs_tools -->|REST| docs diff --git a/docs/images/prowler_mcp_schema.png b/docs/images/prowler_mcp_schema.png deleted file mode 100644 index 8a8884fa5e..0000000000 Binary files a/docs/images/prowler_mcp_schema.png and /dev/null differ diff --git a/docs/user-guide/cli/tutorials/parallel-execution.mdx b/docs/user-guide/cli/tutorials/parallel-execution.mdx index 93b8ef381b..d659116b50 100644 --- a/docs/user-guide/cli/tutorials/parallel-execution.mdx +++ b/docs/user-guide/cli/tutorials/parallel-execution.mdx @@ -184,7 +184,7 @@ $combinedCsv | Export-Csv -Path "CombinedCSV.csv" -NoTypeInformation ## TODO: Additional Improvements -Some services need to instantiate another service to perform a check. For instance, `cloudwatch` will instantiate Prowler's `iam` service to perform the `cloudwatch_cross_account_sharing_disabled` check. When the `iam` service is instantiated, it will perform the `__init__` function, and pull all the information required for that service. This provides an opportunity for an improvement in the above script to group related services together so that the `iam` services (or any other cross-service references) isn't repeatedily instantiated by grouping dependant services together. A complete mapping between these services still needs to be further investigated, but these are the cross-references that have been noted: +Some services need to instantiate another service to perform a check. For instance, `cloudwatch` will instantiate Prowler's `iam` service to perform the `cloudwatch_cross_account_sharing_disabled` check. When the `iam` service is instantiated, it will perform the `__init__` function, and pull all the information required for that service. This provides an opportunity for an improvement in the above script to group related services together so that the `iam` services (or any other cross-service references) aren't repeatedly instantiated by grouping dependent services together. A complete mapping between these services still needs to be further investigated, but these are the cross-references that have been noted: * inspector2 needs lambda and ec2 * cloudwatch needs iam diff --git a/docs/user-guide/cli/tutorials/reporting.mdx b/docs/user-guide/cli/tutorials/reporting.mdx index 19a14c9ae2..47e46dc2aa 100644 --- a/docs/user-guide/cli/tutorials/reporting.mdx +++ b/docs/user-guide/cli/tutorials/reporting.mdx @@ -114,7 +114,7 @@ The CSV format follows a standardized structure across all providers. The follow #### CSV Headers Mapping -The following table shows the mapping between the CSV headers and the the providers fields: +The following table shows the mapping between the CSV headers and the providers fields: | Open Source Consolidated| AWS| GCP| AZURE| KUBERNETES |----------|----------|----------|----------|---------- diff --git a/docs/user-guide/providers/aws/organizations.mdx b/docs/user-guide/providers/aws/organizations.mdx index c0e9ae0e6b..b48df1c442 100644 --- a/docs/user-guide/providers/aws/organizations.mdx +++ b/docs/user-guide/providers/aws/organizations.mdx @@ -2,6 +2,8 @@ title: 'AWS Organizations in Prowler' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + **Using Prowler Cloud?** You can onboard your entire AWS Organization through the UI with automatic account discovery, OU-aware tree selection, and bulk connection testing — no scripts or YAML files required. @@ -71,11 +73,43 @@ The additional fields in CSV header output are as follows: ## Deploying Prowler IAM Roles Across AWS Organizations + + When onboarding multiple AWS accounts into Prowler Cloud, it is important to deploy the Prowler Scan IAM Role in each account. The most efficient way to do this across an AWS Organization is by leveraging AWS CloudFormation StackSets, which rolls out infrastructure—like IAM roles—to all accounts centrally from the Management or Delegated Admin account. -When using Infrastructure as Code (IaC), Terraform is recommended to manage this deployment systematically. +### Native CloudFormation StackSet Deployment (Recommended) -### Recommended Approach +The [Prowler Scan IAM Role CloudFormation template](https://github.com/prowler-cloud/prowler/blob/master/permissions/templates/cloudformation/prowler-scan-role.yml) can deploy the role across your entire AWS Organization on its own—no third-party modules required. When launched in the **Management Account** (or a **Delegated Administrator** account) with `DeployStackSet=true` and `EnableOrganizations=true`, it creates a service-managed CloudFormation StackSet that rolls the ProwlerScan role out to every account under the target Organizational Unit (or the organization root), and keeps new accounts covered automatically through auto-deployment. + +To deploy from the CloudFormation console: open **CloudFormation → Create stack → With new resources**, choose **Upload a template file** and select `prowler-scan-role.yml` (or paste its S3 URL), then set the parameters below on the **Specify stack details** step. Leave the **Configure stack options** step at its defaults. + +Deploy a single CloudFormation Stack in the Management Account with the following parameters: + +| Parameter | Description | Default | +| --- | --- | --- | +| `ExternalId` | External ID provided by Prowler Cloud to secure role assumption. | — | +| `DeployLocalRole` | Create the ProwlerScan role in this (Management) account. | `true` | +| `DeployStackSet` | Create a service-managed StackSet that deploys the role to member accounts. | `false` | +| `AWSOrganizationalUnitId` | Target OU (`ou-xxxx-yyyyyyyy`) or organization root (`r-xxxx`) for the StackSet. Required when `DeployStackSet=true`. | `""` | +| `DeployFromDelegatedAdmin` | Set to `true` when deploying from a Delegated Administrator account instead of the Management Account (uses `CallAs: DELEGATED_ADMIN`). | `false` | +| `EnableOrganizations` | Add AWS Organizations permissions to the Management Account role: read-only account discovery plus the StackSet-management permissions the deployment needs. Set to `true` when deploying in the Management Account. | `false` | +| `FailureTolerancePercentage` | Percentage of accounts in which the StackSet operation can fail before CloudFormation stops the operation. | `10` | +| `RetainStacksOnAccountRemoval` | Keep the role in an account after it leaves the Organization or OU. | `false` | + + +On the review step, select **"I acknowledge that AWS CloudFormation might create IAM resources with custom names"** — the template provisions the named `ProwlerScan` IAM role, so the stack requires the `CAPABILITY_NAMED_IAM` capability and fails without this acknowledgment. (The quick-create link handles this for you.) + + + +The service-managed StackSet does **not** deploy to the Management Account itself. Keeping `DeployLocalRole=true` ensures the role also exists there, so a single stack covers both the Management and member accounts. + +Trusted access for CloudFormation StackSets must be enabled in the Organization (see the note at the top of this page) before `DeployStackSet` will work. + +Deploying for the CLI or a self-hosted Prowler (not Prowler Cloud)? Also set `AccountId` to the account you assume the role from and `IAMPrincipal` to your identity — the defaults target Prowler Cloud. See [Aligning the trust policy with your identity](/user-guide/providers/aws/authentication#trust-policy-align-iamprincipal-with-your-identity). + + + +### Alternative: Deploy with Terraform - **Use StackSets** from the **Management Account** (or a Delegated Admin/Security Account). - **Use Terraform** to orchestrate the deployment. diff --git a/docs/user-guide/providers/aws/role-assumption.mdx b/docs/user-guide/providers/aws/role-assumption.mdx index b714beafbd..54dd0e214a 100644 --- a/docs/user-guide/providers/aws/role-assumption.mdx +++ b/docs/user-guide/providers/aws/role-assumption.mdx @@ -77,6 +77,15 @@ The template requires the following parameters: - **AccountId:** *(Optional)* AWS Account ID that will assume the role (default: Prowler Cloud account) - **IAMPrincipal:** *(Optional)* The IAM principal allowed to assume the role (default: `role/prowler*`) + +From the CLI you assume the role with **your own** identity, not from Prowler Cloud. The `AccountId` and `IAMPrincipal` defaults target Prowler Cloud, so set **`AccountId`** to the account you run Prowler from and **`IAMPrincipal`** to your identity (for example `role/` or `user/`). Otherwise `sts:AssumeRole` fails with `AccessDenied`. See [Aligning the trust policy with your identity](/user-guide/providers/aws/authentication#trust-policy-align-iamprincipal-with-your-identity). + + + +To deploy the role across an entire AWS Organization from a single stack (Management Account role plus a service-managed StackSet for the member accounts), the template also accepts `DeployLocalRole`, `DeployStackSet`, `AWSOrganizationalUnitId`, `DeployFromDelegatedAdmin`, `EnableOrganizations`, `FailureTolerancePercentage`, and `RetainStacksOnAccountRemoval`. See [AWS Organizations in Prowler](/user-guide/providers/aws/organizations#native-cloudformation-stackset-deployment-recommended) for the full parameter reference. + + + When running Prowler CLI, include the External ID using the `-I/--external-id` flag: ```sh diff --git a/docs/user-guide/providers/oci/authentication.mdx b/docs/user-guide/providers/oci/authentication.mdx index 9627e88cd4..cc0e75cded 100644 --- a/docs/user-guide/providers/oci/authentication.mdx +++ b/docs/user-guide/providers/oci/authentication.mdx @@ -434,7 +434,7 @@ prowler oci --oci-config-file /path/to/config **Cause**: Insufficient IAM permissions -**Solution**: Add required policies (see [Required Permissions](./getting-started-oci.md#required-permissions)) +**Solution**: Add required policies (see [Required Permissions](/user-guide/providers/oci/getting-started-oci#required-permissions)) ### Configuration Validation diff --git a/docs/user-guide/providers/oci/getting-started-oci.mdx b/docs/user-guide/providers/oci/getting-started-oci.mdx index 2f610cd09c..6a8c8de3e9 100644 --- a/docs/user-guide/providers/oci/getting-started-oci.mdx +++ b/docs/user-guide/providers/oci/getting-started-oci.mdx @@ -58,7 +58,7 @@ Before you begin, ensure you have: ### Authentication -Prowler supports multiple authentication methods for OCI. For detailed authentication setup, see the [OCI Authentication Guide](./authentication). +Prowler supports multiple authentication methods for OCI. For detailed authentication setup, see the [OCI Authentication Guide](/user-guide/providers/oci/authentication). **Note:** OCI Session Authentication and Config File Authentication both use the same `~/.oci/config` file. The difference is how the config file is generated - automatically via browser (session auth) or manually with API keys. @@ -107,7 +107,7 @@ The easiest and most secure method is using OCI session authentication, which au #### Alternative: Manual API Key Setup -If you prefer to manually generate API keys instead of using browser-based session authentication, see the detailed instructions in the [Authentication Guide](./authentication#config-file-authentication-manual-api-key-setup). +If you prefer to manually generate API keys instead of using browser-based session authentication, see the detailed instructions in the [Authentication Guide](/user-guide/providers/oci/authentication#config-file-authentication-manual-api-key-setup). **Note:** Both methods use the same `~/.oci/config` file - the difference is that manual setup uses static API keys while session authentication uses temporary session tokens. diff --git a/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx b/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx index 2e8b5eb62e..188ac3540d 100644 --- a/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx +++ b/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx @@ -272,6 +272,8 @@ python aws_org_generator.py \ 4. Deploy to all organizational units 5. Use a unique external ID (e.g., `prowler-org-2024-abc123`) + Alternatively, deploy the same template as a **single stack** with `DeployStackSet=true` and `AWSOrganizationalUnitId` set to your root/OU ID — it creates the StackSet for you. See [Native CloudFormation StackSet Deployment](../providers/aws/organizations#native-cloudformation-stackset-deployment-recommended). + {/* TODO: Add screenshot of CloudFormation StackSets deployment */} diff --git a/docs/user-guide/tutorials/prowler-app-attack-paths.mdx b/docs/user-guide/tutorials/prowler-app-attack-paths.mdx index 32df3c50ba..22e56e5634 100644 --- a/docs/user-guide/tutorials/prowler-app-attack-paths.mdx +++ b/docs/user-guide/tutorials/prowler-app-attack-paths.mdx @@ -283,7 +283,7 @@ In addition to the upstream schema, Prowler enriches the graph with: AI assistants connected through Prowler MCP Server can fetch the exact Cartography schema for the active scan via the - `prowler_app_get_attack_paths_cartography_schema` tool. This guarantees that + `prowler_get_attack_paths_cartography_schema` tool. This guarantees that generated queries match the schema version pinned by the running Prowler release. @@ -427,10 +427,10 @@ Attack Paths capabilities are also available through the [Prowler MCP Server](/g The following MCP tools are available for Attack Paths: -- **`prowler_app_list_attack_paths_scans`** - List and filter Attack Paths scans. -- **`prowler_app_list_attack_paths_queries`** - Discover available queries for a completed scan. -- **`prowler_app_run_attack_paths_query`** - Execute a query and retrieve graph results with nodes and relationships. -- **`prowler_app_get_attack_paths_cartography_schema`** - Retrieve the Cartography graph schema for custom openCypher queries. +- **`prowler_list_attack_paths_scans`** - List and filter Attack Paths scans. +- **`prowler_list_attack_paths_queries`** - Discover available queries for a completed scan. +- **`prowler_run_attack_paths_query`** - Execute a query and retrieve graph results with nodes and relationships. +- **`prowler_get_attack_paths_cartography_schema`** - Retrieve the Cartography graph schema for custom openCypher queries. ### Example Questions diff --git a/docs/user-guide/tutorials/prowler-app-scan-configuration.mdx b/docs/user-guide/tutorials/prowler-app-scan-configuration.mdx index 8d50f0072c..60e0db2f4c 100644 --- a/docs/user-guide/tutorials/prowler-app-scan-configuration.mdx +++ b/docs/user-guide/tutorials/prowler-app-scan-configuration.mdx @@ -8,7 +8,7 @@ import { SubscriptionBanner } from "/snippets/subscription-banner.mdx" -Scan Configuration lets you override, per provider, specific values in the default configuration Prowler's checks use during a scan. Each configuration modifies how specific checks behave, e.g.: thresholds, allowed values, retention windows, and you attach it to the providers that you want to use it on their next scan. +Scan Configuration lets you override, per provider, specific values in the default configuration Prowler's checks use during a scan. Each configuration can modify how specific checks behave, such as thresholds, allowed values, and retention windows, or exclude checks and services from the scan scope. Attach it to the providers that should use it on their next scan. @@ -54,6 +54,24 @@ gcp: storage_min_retention_days: 30 ``` +### Limiting the Scan Scope + + + +Use `excluded_checks` to skip individual checks and `excluded_services` to skip every check in a service for the matching provider type: + +```yaml +aws: + excluded_checks: + - s3_bucket_public_access + excluded_services: + - ec2 +``` + + +When a Scan Configuration excludes checks or services, Prowler calculates overviews, aggregations, and other result-based information from the reduced scan scope. The displayed information reflects only the checks and services that ran, not a complete assessment of the provider. Consider the applied Scan Configuration when interpreting totals and security posture. + + ## Creating a Scan Configuration diff --git a/docs/user-guide/tutorials/prowler-cloud-aws-organizations.mdx b/docs/user-guide/tutorials/prowler-cloud-aws-organizations.mdx index 8bb6e320ae..8b2124e004 100644 --- a/docs/user-guide/tutorials/prowler-cloud-aws-organizations.mdx +++ b/docs/user-guide/tutorials/prowler-cloud-aws-organizations.mdx @@ -8,12 +8,14 @@ import { SubscriptionBanner } from "/snippets/subscription-banner.mdx" -Prowler Cloud enables you to onboard all AWS accounts in your Organization through a single guided wizard. Instead of connecting accounts one by one, you can discover every account in your AWS Organization, select the ones you want to monitor, test connectivity, and launch scans — all from the Prowler Cloud UI. +Prowler Cloud onboards every AWS account in your Organization through a single guided wizard. Instead of connecting accounts one by one, you can discover every account in your AWS Organization, select the ones you want to monitor, test connectivity, and launch scans — all from the Prowler Cloud UI. For CLI-based multi-account scanning, see [AWS Organizations in Prowler CLI](/user-guide/providers/aws/organizations). +To follow this guide you need an active [Prowler Cloud](https://cloud.prowler.com) account and access to your AWS Organization [management account](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_introduction.html) (or a registered delegated administrator account). + ## Overview ### Individual Accounts vs Organizations @@ -25,225 +27,17 @@ For CLI-based multi-account scanning, see [AWS Organizations in Prowler CLI](/us ### How It Works -Before using the AWS Organizations wizard, you need to deploy **two Identity and Access Management (IAM) roles** in your AWS environment. The onboarding follows this sequence: + + +Onboarding deploys the **ProwlerScan Identity and Access Management (IAM) role** in your management account and in every member account. A **single CloudFormation stack** — launched from the wizard's **Create Stack in Management Account** button ([Step 2](#step-2-authenticate-with-your-management-account)) — creates the management account role **and** a service-managed StackSet that rolls the role out to your member accounts in one operation. Prefer to deploy the roles yourself? See [Deploy the Roles Manually](#deploy-the-roles-manually). - Onboarding flow: 1. Create Management Account Role (Quick Create or Manual), 2. Deploy StackSet, 3. Run the Wizard, 4. Launch Scans + Onboarding flow: 1. Start the Wizard, 2. Deploy the Roles (single CloudFormation stack), 3. Discover and Connect, 4. Launch Scans -## Key Concepts +## Step 1: Start the Organization Wizard -### What Is an External ID? - -An **External ID** is a security token that Prowler generates unique to your tenant. When Prowler assumes the IAM role in your AWS account, it presents this External ID to prove its identity. - -This prevents the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html) — a scenario where an unauthorized party could trick AWS into granting access to your account. By requiring the External ID, only your specific Prowler tenant can assume the role. - -You don't need to create the External ID yourself — Prowler generates it automatically and displays it in the wizard for you to copy. - -### Two Roles Architecture - -Prowler requires **two separate IAM roles** deployed in different places, each with a distinct purpose: - -| Role | Where it lives | What it does | How to deploy it | -|------|---------------|--------------|------------------| -| **ProwlerScan** (management account) | Your management (root) account only | Discovers the Organization structure **and** scans the management account. Has additional Organizations discovery permissions. | Via **Quick Create** link or **manually** in the IAM Console ([Step 1](#step-1-create-the-management-account-role)). Cannot be deployed via StackSet. | -| **ProwlerScan** (member accounts) | Every member account | Scans the account for security findings. | Via **CloudFormation StackSet** ([Step 2](#step-2-deploy-the-cloudformation-stackset)). Automated across all accounts. | - - - Two Roles Architecture: ProwlerScan in management account (Quick Create or Manual, discovery + scanning) and ProwlerScan in member accounts (via StackSet, scanning only) - - - -**Same name, different permissions.** Both roles are named `ProwlerScan` — Prowler expects a consistent role name across all accounts. The management account role has the same scanning permissions as member accounts, plus additional Organizations discovery permissions (see [Step 1](#step-1-create-the-management-account-role) for the full list). - - -### What Is a CloudFormation StackSet? - -A [CloudFormation StackSet](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/what-is-cfnstacksets.html) lets you deploy the same CloudFormation template across multiple AWS accounts in a single operation. Prowler uses a StackSet to deploy the **ProwlerScan** IAM role into every member account of your organization, so you don't have to create the role manually in each account. - -## Prerequisites - -### Prowler Cloud Account - -You need an active [Prowler Cloud](https://cloud.prowler.com) account. Each AWS account you connect will count as a provider in your subscription. See [Billing Impact](#billing-impact) for details. - -### AWS Organization Enabled - -Your AWS environment must have [AWS Organizations](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_introduction.html) enabled. You will need access to the **management account** (or a delegated administrator account) to provide the Organization ID and IAM Role ARN. - -## Step 1: Create the Management Account Role - -The first role you need to create is the **management account role**. This role allows Prowler to discover your Organization structure — listing accounts, OUs, and hierarchy. - - -**StackSets do not deploy to the management account.** Organizational CloudFormation StackSets with service-managed permissions only target member accounts — this is an AWS limitation, not a Prowler one. You must create the management account role separately, either via the Quick Create link ([Option A](#option-a-quick-create-link-fastest)) or manually ([Option B](#option-b-create-the-role-manually)). - - - -**The role must be named `ProwlerScan`** — the same name as the role deployed to member accounts via StackSet. Prowler expects a consistent role name across all accounts in the Organization. If you use a different name, connection tests and scans will fail for the management account. - - -### Option A: Quick Create Link (Fastest) - -The Prowler wizard provides a one-click link that opens the AWS Console with the CloudFormation template pre-configured. This creates a **CloudFormation Stack** (not a StackSet) that deploys the ProwlerScan role with Organizations permissions enabled in your management account. - - -**[Open Quick Create Stack in AWS Console →](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateURL=https%3A%2F%2Fprowler-cloud-public.s3.eu-west-1.amazonaws.com%2Fpermissions%2Ftemplates%2Faws%2Fcloudformation%2Fprowler-scan-role.yml&stackName=Prowler¶m_EnableOrganizations=true)** - -Opens the CloudFormation Console with the Prowler scan role template and `EnableOrganizations=true` pre-filled. You will need to enter the **ExternalId** parameter manually — copy it from the Prowler wizard ([Step 4](#step-4-authenticate-with-your-management-account)). - - -1. Click **[Open Quick Create Stack in AWS Console →](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateURL=https%3A%2F%2Fprowler-cloud-public.s3.eu-west-1.amazonaws.com%2Fpermissions%2Ftemplates%2Faws%2Fcloudformation%2Fprowler-scan-role.yml&stackName=Prowler¶m_EnableOrganizations=true)** or use the **Create Stack in Management Account** button in the Prowler wizard (which also pre-fills the ExternalId). -2. Enter the **ExternalId** parameter if not pre-filled. -3. Check **"I acknowledge that AWS CloudFormation might create IAM resources with custom names"** and click **Create stack**. -4. Wait for the stack to reach **CREATE_COMPLETE** status. - -Take note of the **Role ARN** from the stack's **Outputs** tab — you will need it in the wizard. - -### Option B: Create the Role Manually - -1. Sign in to the [AWS IAM Console](https://console.aws.amazon.com/iam/) in your **management account**. - -2. Go to **Roles > Create role** and select **Custom trust policy**. - -3. Paste the following trust policy. This allows Prowler Cloud to assume the role using your tenant's External ID (you will get this from the Prowler wizard in [Step 3](#step-3-start-the-organization-wizard)): - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::232136659152:root" - }, - "Action": "sts:AssumeRole", - "Condition": { - "StringEquals": { - "sts:ExternalId": "" - }, - "StringLike": { - "aws:PrincipalArn": "arn:aws:iam::232136659152:role/prowler*" - } - } - } - ] -} -``` - -Replace `` with the External ID shown in the Prowler wizard. - -4. Attach the following AWS managed policies: - - **SecurityAudit** - - **ViewOnlyAccess** - - This allows Prowler to also scan the management account for security findings, just like any other account. - -5. Create an additional inline policy with the following permissions. These are specific to the management account and allow Prowler to discover your Organization structure: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ProwlerOrganizationDiscovery", - "Effect": "Allow", - "Action": [ - "organizations:DescribeAccount", - "organizations:DescribeOrganization", - "organizations:ListAccounts", - "organizations:ListAccountsForParent", - "organizations:ListOrganizationalUnitsForParent", - "organizations:ListRoots", - "organizations:ListTagsForResource" - ], - "Resource": "*" - }, - { - "Sid": "ProwlerStackSetManagement", - "Effect": "Allow", - "Action": [ - "organizations:RegisterDelegatedAdministrator", - "iam:CreateServiceLinkedRole" - ], - "Resource": "*" - } - ] -} -``` - - -You can optionally restrict the `Resource` field to your specific Organization ARN (e.g., `arn:aws:organizations::123456789012:organization/o-abc123def4`) instead of `"*"` to minimize the blast radius. - - -6. Name the role **`ProwlerScan`** and click **Create role**. Take note of the **Role ARN** — you will need it in the Prowler wizard. - -The ARN follows this format: `arn:aws:iam:::role/ProwlerScan` - - -The role **must** be named `ProwlerScan`. Do not use a different name. - - - -If you just created the role, it may take up to **60 seconds** for AWS to propagate it. If you get an error in the Prowler wizard, wait a moment and try again. - - -## Step 2: Deploy the CloudFormation StackSet - -After creating the management account role, the next step is to deploy the **ProwlerScan** role to your member accounts using a CloudFormation StackSet. This is the recommended method for consistent, scalable deployment across your entire organization. - -The StackSet uses **service-managed permissions**, which means AWS Organizations handles the cross-account deployment automatically — you don't need to create execution roles manually in each account. The StackSet deploys the ProwlerScan IAM role in every target member account, enabling Prowler to assume that role for cross-account scanning. - - -**Trusted access required:** CloudFormation StackSets must have trusted access enabled in your management account. Verify this in the AWS Console under **AWS Organizations > Settings > Trusted access for AWS CloudFormation StackSets**. - - - -**The Quick Create link creates a Stack, not a StackSet.** The link in the Prowler wizard creates a CloudFormation **Stack** that deploys the ProwlerScan role in your management account only ([Step 1](#step-1-create-the-management-account-role)). To deploy the role across **member accounts**, you must create a StackSet manually as described below. AWS does not support Quick Create links for StackSets. - - - -**[Open StackSets Console →](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacksets/create)** - -Opens the CloudFormation StackSets creation page directly. You will need to paste the template URL and ExternalId manually. - - -1. Click the link above or navigate to **CloudFormation > StackSets > Create StackSet** in your management account. -2. Choose **Service-managed permissions**. -3. Select **Amazon S3 URL** as the template source and paste the following URL: - ``` - https://prowler-cloud-public.s3.eu-west-1.amazonaws.com/permissions/templates/aws/cloudformation/prowler-scan-role.yml - ``` -4. Set the **ExternalId** parameter to the External ID shown in the Prowler wizard. -5. Choose your deployment targets (entire organization or specific OUs). -6. Select the AWS regions where you want the role deployed. -7. Click **Create StackSet**. - -### Verify StackSet Deployment - -After deploying, verify that all stack instances completed successfully: - -1. In the CloudFormation Console, go to **StackSets** and select your Prowler StackSet. -2. Click the **Stack instances** tab. -3. Confirm that all instances show **Status: CURRENT** and **Stack status: CREATE_COMPLETE**. - -Deployment typically takes **2–5 minutes** for medium-sized organizations. Large organizations (500+ accounts) may take longer. - - -**Prefer Terraform?** You can deploy the ProwlerScan role using Terraform instead. See the [StackSets deployment guide](/user-guide/providers/aws/organizations#deploying-prowler-iam-roles-across-aws-organizations) for the Terraform module. - - -### Key Considerations - -- **Service-managed permissions**: Always select **Service-managed permissions** when creating the StackSet. This lets AWS Organizations manage the deployment automatically across current and future member accounts. -- **Least privilege**: The ProwlerScan role deployed by the StackSet uses `SecurityAudit` and `ViewOnlyAccess` — AWS managed policies that grant read-only access — plus a small set of additional read-only permissions for services not covered by those policies. See the [CloudFormation template](https://prowler-cloud-public.s3.eu-west-1.amazonaws.com/permissions/templates/aws/cloudformation/prowler-scan-role.yml) for the full list. Prowler does not make any changes to your accounts. -- **New accounts**: When you add new accounts to your AWS Organization, the StackSet automatically deploys the ProwlerScan role to them if you targeted the organization root or the relevant OU. Combined with Prowler's 6-hour automatic sync, new accounts are onboarded end-to-end without manual intervention. -- **Management account**: Organizational StackSets **do not deploy to the management account itself**. If you want to scan the management account, you need to create the ProwlerScan role there separately using a regular CloudFormation Stack. - -## Step 3: Start the Organization Wizard - -Now that both roles are deployed — the management account role (Step 1) and the ProwlerScan role in member accounts (Step 2) — you can start the Prowler wizard. +The Prowler wizard walks you through the entire flow: deploying both roles from a single CloudFormation stack, discovering your accounts, testing connectivity, and launching scans. ### Open the Wizard @@ -280,29 +74,50 @@ Now that both roles are deployed — the management account role (Step 1) and th Click **Next** to proceed to the authentication phase. -## Step 4: Authenticate with Your Management Account +## Step 2: Authenticate with Your Management Account -The wizard's **Authentication Details** page guides you through three actions: deploying the roles in AWS, entering the management account Role ARN, and confirming the deployment. +The **Authentication Details** page guides you through three actions: deploying the roles in AWS, entering the deployment account Role ARN, and confirming the deployment. The deployment account is either the management account or, when delegated administrator mode is selected, the delegated administrator account. ### External ID -The wizard displays a **Prowler External ID** at the top — auto-generated and unique to your tenant. Click the copy icon to copy it. You will need this External ID for both the management account Stack and the member accounts StackSet. +The wizard displays a **Prowler External ID** at the top — auto-generated and unique to your tenant. Click the copy icon to copy it. The External ID is pre-filled into the deployment link, and the single stack applies it to both the management account role and the member-account StackSet. Learn more in [What Is an External ID?](#what-is-an-external-id). ### Deploy the Roles -The wizard provides two deployment actions: + -1. **Create Stack in Management Account** — opens a Quick Create link that deploys the ProwlerScan role with `EnableOrganizations=true` in your management account ([Step 1](#step-1-create-the-management-account-role)). The External ID is pre-filled. +The wizard deploys the deployment account role and the member-account StackSet in a **single** CloudFormation Stack: -2. **Open StackSets Console** — links to the CloudFormation StackSets console where you create a StackSet for member accounts ([Step 2](#step-2-deploy-the-cloudformation-stackset)). Copy the template URL shown in the wizard and paste the External ID manually. + +**Prefer to use your own role?** You do not have to use the Quick Create template. Create the ProwlerScan role yourself — through the IAM Console, Terraform, or your own CloudFormation [(Following this guide)](#deploy-the-roles-manually) — and paste its ARN into the Role ARN field below. The role must use the external ID from the earlier step and include the trust policy and permissions described in [Deploy the Roles Manually](#deploy-the-roles-manually). + + +1. **Organizational Unit or Root ID** — enter the AWS OU (`ou-xxxx-yyyyyyyy`) or organization root (`r-xxxx`) you want to onboard. Prowler rolls the ProwlerScan role out to every member account under this target. Find it in the [AWS Organizations Console](https://console.aws.amazon.com/organizations/); use the **root ID** (`r-`) to cover the entire organization or an **OU ID** (`ou-`) to target a specific unit. + +2. *(Optional)* Check **"I'm deploying from a delegated administrator account"** if you launch the stack from a delegated administrator account instead of the management account. + +3. **Create Stack in Management Account** — or **Create Stack in Delegated Administrator Account** when delegated administrator mode is selected — opens a Quick Create link that deploys, in a single stack: the ProwlerScan role in the account where you launch the stack (`DeployLocalRole`, with `EnableOrganizations=true`) **and** a service-managed StackSet (`DeployStackSet`) that rolls the role out to your member accounts. The External ID, OU/Root ID, and deployment options are pre-filled. - Authentication Details form showing External ID, two deployment buttons (Create Stack in Management Account and Open StackSets Console), Management Account Role ARN field, and deployment confirmation checkbox + Authentication Details form showing External ID, Organizational Unit or Root ID field, delegated administrator checkbox, deployment account stack button, deployment account Role ARN field, and deployment confirmation checkbox -### Enter the Management Account Role ARN + +**Finding your Organizational Unit or Root ID.** In the [AWS Organizations Console](https://console.aws.amazon.com/organizations/) the root (`r-…`) and OU (`ou-…`) IDs appear in the account tree, or run these from your management account: -Paste the **Role ARN** of the management account role you created in [Step 1](#step-1-create-the-management-account-role) into the **Management Account Role ARN** field. +```bash +# Root ID — deploys the role to the entire organization +aws organizations list-roots --query 'Roots[0].Id' --output text + +# OU IDs under the root — to target a specific unit instead +aws organizations list-organizational-units-for-parent --parent-id r-xxxx \ + --query 'OrganizationalUnits[].{Name:Name,Id:Id}' --output table +``` + + +### Enter the Deployment Account Role ARN + +Paste the **Role ARN** created by the stack above into the **Management Account Role ARN** field or, when delegated administrator mode is selected, the **Delegated Administrator Account Role ARN** field. The ARN follows this format: ``` @@ -312,12 +127,16 @@ arn:aws:iam:::role/ProwlerScan For example: `arn:aws:iam::123456789012:role/ProwlerScan` - Management Account Role ARN field in the Authentication Details form + Deployment account Role ARN field in the Authentication Details form + +It may take up to **60 seconds** for AWS to generate the IAM Role ARN after the stack completes. If the wizard reports an error, wait a moment and try again. + + ### Confirm and Discover -1. Check the box: **"The Stack and StackSet have been successfully deployed in AWS"**. +1. Check the box: **"The Stack has been successfully deployed in AWS"**. 2. Click **Authenticate**. Here's what happens behind the scenes: @@ -325,7 +144,7 @@ Here's what happens behind the scenes: - An asynchronous discovery is triggered to query your AWS Organization structure. - You will see a **"Gathering AWS Accounts..."** spinner — this typically takes **30 seconds to 2 minutes** depending on your organization size. -## Step 5: Select Accounts to Scan +## Step 3: Select Accounts to Scan ### Understanding the Tree View @@ -336,6 +155,7 @@ Once discovery completes, the wizard displays a **hierarchical tree view** of yo - The tree supports up to **5 levels of nesting** (Root > OUs > Sub-OUs > Accounts). +- If you deployed the stack for just one OU, that OU will be preselected in the tree. - **Selecting an OU** automatically selects all accounts within it. - **Individual overrides**: deselect specific accounts even if the parent OU is selected. - The header shows **"X of Y accounts selected"** to track your selection. @@ -352,14 +172,12 @@ Only **ACTIVE** accounts can be selected for scanning: | **CLOSED** | No | Account has been closed. | -**Your existing data is safe.** If an AWS account is already connected to Prowler as an individual provider, it will appear in the tree with a checkmark indicator. +**Your existing data is safe.** If an AWS account is already connected to Prowler as an individual provider, it appears in the tree with a checkmark indicator. When you proceed: - The existing provider is **linked** to the organization — it is **not** duplicated. - All your **historical scan data and findings are preserved** — nothing is overwritten. - There is **no additional billing** — the existing provider is reused. - -This is completely safe. You are simply associating the account with the organization for easier management. ### Custom Aliases @@ -368,14 +186,9 @@ You can edit the display name for each account before connecting. This alias is ### Blocked Accounts -Some accounts may appear as **blocked** (grayed out, not selectable). This happens when: -- The account is **already linked to a different organization** in Prowler (`linked_to_other_organization`). +Some accounts may appear as **blocked** (grayed out, not selectable) when the account is **already linked to a different organization** in Prowler (`linked_to_other_organization`). Hover over the blocked account to see the specific reason. -Hover over the blocked account to see the specific reason. - -## Step 6: Test Connections - -### How Connection Testing Works +## Step 4: Test Connections Click **Test Connections** to verify that Prowler can assume the **ProwlerScan** role in each selected member account. @@ -383,154 +196,225 @@ Click **Test Connections** to verify that Prowler can assume the **ProwlerScan** Connection testing in progress with spinners on each account -- Each account shows a real-time status indicator: - - **Spinner** — test in progress - - **Green checkmark (✓)** — connection successful - - **Red icon (✗)** — connection failed (hover to see the error) - -### All Tests Pass +Each account shows a real-time status indicator: +- **Spinner** — test in progress +- **Green checkmark (✓)** — connection successful +- **Red icon (✗)** — connection failed (hover to see the error) If every account connects successfully, you automatically advance to the next step. -### Some Tests Fail +### When Some Tests Fail -An error banner appears: **"There was a problem connecting to some accounts."** - -You have two options: +An error banner appears: **"There was a problem connecting to some accounts."** You have two options: **a) Fix and retry:** 1. Go to the AWS Console and verify the StackSet deployed to the failing accounts. 2. Check that the External ID in the StackSet matches the one shown in Prowler. 3. Return to Prowler and click **Test Connections** — only the **failed accounts are re-tested** (smart retry). Accounts that already passed are not tested again. - - Test Connections button - - **b) Skip and continue:** -Click **Skip Connection Validation** to proceed with only the accounts that connected successfully. The failed accounts will not be scanned. +Click **Skip Connection Validation** to proceed with only the accounts that connected successfully. The failed accounts will not be scanned. This option is only available when at least one account connected successfully. Connection test results showing failed accounts with error banner and Skip Connection Validation button - -**Skip Connection Validation** is only available when at least one account connected successfully. - +If **no accounts** connected successfully, you cannot proceed. Fix the underlying connection issues — see [Troubleshooting](#troubleshooting) — and retry before launching scans. -### All Tests Fail - -If **no accounts** connected successfully, you cannot proceed: - -> *"No accounts connected successfully. Fix the connection errors and retry before launching scans."* - -You must fix the underlying connection issues before continuing. See [Updating Credentials](#updating-credentials) below. - -### Updating Credentials - -If connection tests fail, here's how to fix common issues: - -1. Open the [CloudFormation Console](https://console.aws.amazon.com/cloudformation/) and check that your StackSet instances show **CREATE_COMPLETE** for the failing accounts. If not, update the StackSet to include the missing OUs. -2. Compare the **ExternalId** parameter in your StackSet with the External ID displayed in the Prowler wizard. They must match exactly. -3. After fixing the issue in AWS, return to Prowler and click **Test Connections**. Only the previously failed accounts will be re-tested. - -## Step 7: Launch Scans - -### Choose Scan Schedule +## Step 5: Launch Scans The Organizations wizard uses the same schedule controls described in [Scan Scheduling](/user-guide/tutorials/prowler-scan-scheduling#schedule-options). -### Launch - -Click **Save**, **Save and launch scan**, or **Launch scan**, depending on the selected schedule option. A toast notification confirms whether the schedule was saved, scans were launched, or both. The toast includes a link to the **Scans** page. Prowler redirects to the **Providers** page. - -Scans are only launched for accounts that are accessible (passed connection testing) and were selected. +Click **Save**, **Save and launch scan**, or **Launch scan**, depending on the selected schedule option. A toast notification confirms whether the schedule was saved, scans were launched, or both, and includes a link to the **Scans** page. Prowler then redirects to the **Providers** page. Scans launch only for accounts that passed connection testing and were selected. Launch Scan step showing Accounts Connected confirmation, scan schedule selector, and Launch scan button -### What Happens Next - +After launching: - Scans appear in the **Scans** page as they start and complete. - Results populate the **Overview** and **Findings** pages. -- Prowler runs an **automatic sync every 6 hours** to detect new accounts added to your Organization or accounts that have been removed. New accounts are onboarded automatically based on the parent OU configuration. +- Prowler runs an **automatic sync every 6 hours** to detect accounts added to or removed from your Organization. New accounts under the targeted OU or root are onboarded automatically. ## Billing Impact Each AWS account you connect through the Organizations wizard counts as one **provider** in your Prowler Cloud subscription. - **Already-connected accounts**: if an account was already linked as a provider, adding it to the organization does **not** incur additional billing. The existing provider is reused. -- **Large organizations**: connecting a 500-account organization will result in up to 500 providers on your subscription. Review your plan limits before proceeding. +- **Large organizations**: connecting a 500-account organization results in up to 500 providers on your subscription. Review your plan limits before proceeding. - **Deleted providers**: if you later remove an account, the deleted provider no longer counts toward your subscription. For pricing details, see [Prowler Cloud Pricing](https://prowler.com/pricing). ## Troubleshooting -### Invalid AWS Organization ID +### Only Some Accounts Connect -*"Must be a valid AWS Organization ID"* +Discovery succeeds and the tree view appears, but only one account — or a handful — passes the connection test. This almost always means the ProwlerScan role reached the deployment account but not every member account. -- Verify the Organization ID format: `o-` followed by 10–32 lowercase alphanumeric characters (e.g., `o-abc123def4`) -- Copy it directly from the [AWS Organizations Console](https://console.aws.amazon.com/organizations/) to avoid typos +- **Confirm the StackSet deployed.** Open the [CloudFormation Console](https://console.aws.amazon.com/cloudformation/) in the deployment account, select your Prowler StackSet, open the **Stack instances** tab, and confirm every instance shows **Status: CURRENT** and **Stack status: CREATE_COMPLETE**. Instances still in progress or in a failed state explain the missing accounts. +- **Check the targeted OU or root.** The single stack only rolls the role out to accounts under the **Organizational Unit or Root ID** you entered in [Step 2](#step-2-authenticate-with-your-management-account). Accounts in other OUs are not covered — redeploy targeting the organization root (`r-`) or add the missing OUs. +- **Verify the deployment account.** The role is created only in the account where you launched the stack. If you deployed from a **delegated administrator account**, confirm that account is a **registered delegated administrator** for CloudFormation StackSets (registered through AWS Organizations), not just a regular member account. A regular member account cannot create a service-managed StackSet, so only its own role is created — leaving every other account without the role. +- **Suspended accounts** cannot be scanned. Deselect them and proceed. -### Invalid IAM Role ARN +### No Accounts Connect -*"Must be a valid IAM Role ARN"* +No account passes the connection test. -- Verify the ARN format: `arn:aws:iam::<12-digit-account-id>:role/` -- Copy the ARN directly from the [IAM Console](https://console.aws.amazon.com/iam/) in your management account +- **External ID mismatch.** Compare the **ExternalId** parameter in your StackSet with the External ID shown in the Prowler wizard. They must match exactly. +- **StackSet not deployed.** Confirm the StackSet exists and its instances reached **CREATE_COMPLETE**. If you deployed the roles manually, verify [trusted access for CloudFormation StackSets](#member-account-role-stackset) is enabled. +- **IP-based policies.** If your accounts restrict access by IP, allow the [Prowler Cloud egress IPs](/security/networking). -### Authentication Failed +### Authentication Fails or Times Out -*"Authentication failed. Please verify the StackSet deployment and Role ARN"* +*"Authentication failed. Please verify the StackSet deployment and Role ARN"* or *"Authentication timed out"* -- Verify the management account role exists and was created in [Step 1](#step-1-create-the-management-account-role) -- Confirm the trust policy includes the correct External ID from the wizard -- Check the role has all Organizations discovery permissions listed in [Step 1](#step-1-create-the-management-account-role) -- Double-check the Role ARN format and account ID for typos +- Verify the deployment account role exists and is named exactly `ProwlerScan`. +- Confirm the trust policy includes the correct External ID from the wizard. +- Check the role has the Organizations discovery permissions listed in [Deploy the Roles Manually](#management-account-role). +- Double-check the Role ARN format and account ID for typos. +- Retry — the role can take up to **60 seconds** to propagate, and a second attempt often succeeds. For very large organizations (500+ accounts), allow extra time for discovery. -### Authentication Timed Out +### Invalid Organization ID or Role ARN -*"Authentication timed out"* +*"Must be a valid AWS Organization ID"* or *"Must be a valid IAM Role ARN"* -- Retry the authentication step — the second attempt often succeeds -- Check for AWS API rate limiting on the Organizations service -- For very large organizations (500+ accounts), allow extra time for discovery - -### Connection Test Fails for All Accounts - -No accounts pass the connection test. - -- Verify the CloudFormation StackSet was deployed — complete [Step 2](#step-2-deploy-the-cloudformation-stackset) and wait for stack instances to reach **CREATE_COMPLETE** -- Check that the **ExternalId** parameter in the StackSet matches the External ID shown in the Prowler wizard -- If your accounts use IP-based IAM policies, allow [Prowler Cloud egress IPs](/security/networking) - -### Connection Test Fails for Some Accounts - -Some accounts show a red icon while others pass. - -- Expand the StackSet deployment to include the OUs containing the failing accounts -- Suspended accounts cannot be scanned — deselect them and proceed -- Ensure the [STS regional endpoint](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) is enabled in the account's region -- After fixing, click **Test Connections** — only the failed accounts will be re-tested - -### No Accounts Connected Successfully - -*"No accounts connected successfully. Fix the connection errors and retry before launching scans."* - -- Hover over the red icon on each account to see the specific error -- Fix the underlying issues using the guidance above -- Click **Test Connections** to retry +- Organization ID format: `o-` followed by 10–32 lowercase alphanumeric characters (e.g., `o-abc123def4`). +- Role ARN format: `arn:aws:iam::<12-digit-account-id>:role/ProwlerScan`. +- Copy both directly from the AWS Console to avoid typos. ### Failed to Apply Discovery *"Failed to apply discovery"* -- Check the `blocked_reasons` field for any blocked accounts -- Retry the operation -- If the error persists, contact [Prowler Support](mailto:support@prowler.com) +- Check the `blocked_reasons` field for any blocked accounts and retry the operation. +- If the error persists, contact [Prowler Support](mailto:support@prowler.com). + +## Deploy the Roles Manually + +The wizard's **Create Stack** button is the fastest path, but you can create both roles yourself — for example with Terraform or your own CloudFormation — and paste the management account Role ARN into [Step 2](#step-2-authenticate-with-your-management-account). Both roles must be named `ProwlerScan`, since Prowler expects a consistent role name across all accounts. + + +**Prefer Terraform?** You can deploy the ProwlerScan role across the organization with Terraform instead of CloudFormation. See the [StackSets deployment guide](/user-guide/providers/aws/organizations#deploying-prowler-iam-roles-across-aws-organizations) for the module. + + +### Management Account Role + +The management account role lets Prowler discover your Organization structure — listing accounts, OUs, and hierarchy — and scan the management account itself. StackSets with service-managed permissions do not deploy to the management account, so this role is always created separately from the member-account StackSet. + +1. Sign in to the [AWS IAM Console](https://console.aws.amazon.com/iam/) in your **management account** (or delegated administrator account). +2. Go to **Roles > Create role** and select **Custom trust policy**. +3. Paste the following trust policy, replacing `` with the External ID shown in the Prowler wizard: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::232136659152:root" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:ExternalId": "" + }, + "StringLike": { + "aws:PrincipalArn": "arn:aws:iam::232136659152:role/prowler*" + } + } + } + ] +} +``` + +4. Attach the AWS managed policies **SecurityAudit** and **ViewOnlyAccess** so Prowler can scan the management account for security findings. +5. Add an inline policy with the Organizations discovery permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ProwlerOrganizationDiscovery", + "Effect": "Allow", + "Action": [ + "organizations:DescribeAccount", + "organizations:DescribeOrganization", + "organizations:ListAccounts", + "organizations:ListAccountsForParent", + "organizations:ListOrganizationalUnitsForParent", + "organizations:ListRoots", + "organizations:ListTagsForResource" + ], + "Resource": "*" + }, + { + "Sid": "ProwlerStackSetManagement", + "Effect": "Allow", + "Action": [ + "organizations:RegisterDelegatedAdministrator", + "iam:CreateServiceLinkedRole" + ], + "Resource": "*" + } + ] +} +``` + + +You can restrict the `Resource` field to your specific Organization ARN (e.g., `arn:aws:organizations::123456789012:organization/o-abc123def4`) instead of `"*"` to minimize the blast radius. + + +6. Name the role **`ProwlerScan`** and click **Create role**. The ARN follows the format `arn:aws:iam:::role/ProwlerScan` — paste it into the wizard. + +### Member Account Role (StackSet) + +Deploy the ProwlerScan role to every member account with a [CloudFormation StackSet](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/what-is-cfnstacksets.html), so you don't create the role manually in each account. + + +**Trusted access required.** CloudFormation StackSets must have trusted access enabled in your management account. Verify this under **AWS Organizations > Settings > Trusted access for AWS CloudFormation StackSets**. + + +1. In your management account, navigate to **CloudFormation > StackSets > Create StackSet** ([open directly](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacksets/create)). +2. Choose **Service-managed permissions** so AWS Organizations deploys the role automatically across current and future member accounts. +3. Select **Amazon S3 URL** as the template source and paste: + ``` + https://prowler-cloud-public.s3.eu-west-1.amazonaws.com/permissions/templates/aws/cloudformation/prowler-scan-role.yml + ``` +4. Set the **ExternalId** parameter to the External ID shown in the Prowler wizard. +5. Choose your deployment targets (entire organization or specific OUs) and regions, then click **Create StackSet**. +6. Open the **Stack instances** tab and confirm every instance shows **Status: CURRENT** and **Stack status: CREATE_COMPLETE**. Deployment typically takes **2–5 minutes**; large organizations (500+ accounts) may take longer. + +The StackSet role uses read-only access only (`SecurityAudit`, `ViewOnlyAccess`, plus a small set of additional read-only permissions). Prowler makes no changes to your accounts. See the [CloudFormation template](https://prowler-cloud-public.s3.eu-west-1.amazonaws.com/permissions/templates/aws/cloudformation/prowler-scan-role.yml) for the full list. When you add new accounts under the targeted OU or root, the StackSet deploys the role automatically, and Prowler's 6-hour sync onboards them end-to-end. + +## Key Concepts + +### What Is an External ID? + +An **External ID** is a security token that Prowler generates unique to your tenant. When Prowler assumes the IAM role in your AWS account, it presents this External ID to prove its identity. + +This prevents the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html) — a scenario where an unauthorized party could trick AWS into granting access to your account. By requiring the External ID, only your specific Prowler tenant can assume the role. Prowler generates it automatically and displays it in the wizard for you to copy. + +### Two Roles Architecture + +Prowler uses **two IAM roles**, both named `ProwlerScan` but deployed in different places: + +| Role | Where it lives | What it does | +|------|---------------|--------------| +| **ProwlerScan** (management account) | Your management (or delegated administrator) account | Discovers the Organization structure **and** scans that account. Includes additional Organizations discovery permissions. | +| **ProwlerScan** (member accounts) | Every member account | Scans the account for security findings. | + +Both roles share the name `ProwlerScan` because Prowler expects a consistent role name across all accounts. The single CloudFormation stack in [Step 2](#step-2-authenticate-with-your-management-account) deploys both at once. + + + Two Roles Architecture: ProwlerScan in management account (discovery + scanning) and ProwlerScan in member accounts (via StackSet, scanning only) + + +### What Is a CloudFormation StackSet? + +A [CloudFormation StackSet](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/what-is-cfnstacksets.html) deploys the same CloudFormation template across multiple AWS accounts in a single operation. Prowler uses a service-managed StackSet to deploy the **ProwlerScan** IAM role into every member account of your organization, so you don't create the role manually in each account. StackSets do not deploy to the management account, which is why that role is created separately. ## What's Next diff --git a/mcp_server/.env.template b/mcp_server/.env.template index 11b8caa724..10aabd84cf 100644 --- a/mcp_server/.env.template +++ b/mcp_server/.env.template @@ -1,3 +1,3 @@ -PROWLER_APP_API_KEY="pk_your_api_key_here" +PROWLER_API_KEY="pk_your_api_key_here" API_BASE_URL="https://api.prowler.com/api/v1" PROWLER_MCP_TRANSPORT_MODE="stdio" diff --git a/mcp_server/AGENTS.md b/mcp_server/AGENTS.md index a82cc42e33..1d3d808e6a 100644 --- a/mcp_server/AGENTS.md +++ b/mcp_server/AGENTS.md @@ -25,7 +25,7 @@ The Prowler MCP Server provides AI agents access to the Prowler ecosystem throug ## CRITICAL RULES ### Tool Implementation -- ALWAYS: Extend `BaseTool` ABC for Prowler App tools (auto-registration) +- ALWAYS: Extend `BaseTool` ABC for Prowler tools (auto-registration) - ALWAYS: Use `@mcp.tool()` decorator for Hub/Docs tools - NEVER: Manually register BaseTool subclasses - NEVER: Import tools directly in server.py @@ -56,7 +56,7 @@ await prowler_mcp_server.import_server(docs_mcp_server, prefix="prowler_docs") ### Tool Naming - `prowler_hub_*` - Catalog and compliance (no auth) - `prowler_docs_*` - Documentation search (no auth) -- `prowler_app_*` - Cloud/App management (auth required) +- `prowler_*` - Prowler Cloud, Private Cloud & Local Server management (auth required) --- diff --git a/mcp_server/CHANGELOG.md b/mcp_server/CHANGELOG.md index ec0a2e354f..5898f816c9 100644 --- a/mcp_server/CHANGELOG.md +++ b/mcp_server/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to the **Prowler MCP Server** are documented in this file. +## [0.8.0] (Prowler v5.35.0) + +### 🔄 Changed + +- Core Prowler tool namespace from the `prowler_app_*` prefix to `prowler_*` [(#12017)](https://github.com/prowler-cloud/prowler/pull/12017) + +--- + ## [0.7.2] (Prowler v5.28.1) ### 🐞 Fixed diff --git a/mcp_server/README.md b/mcp_server/README.md index e990f0f363..c0e4fdb0fe 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -6,9 +6,9 @@ ## Key Capabilities -### Prowler Cloud and Prowler App (Self-Managed) +### Prowler Cloud, Prowler Private Cloud & Prowler Local Server -Full access to Prowler Cloud platform and self-managed Prowler App for: +Full access to your Prowler data (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server) for: - **Findings Analysis**: Query, filter, and analyze security findings across all your cloud environments - **Finding Groups Analysis**: Triage findings grouped by check ID and drill down into affected resources - **Provider Management**: Create, configure, and manage your configured Prowler providers (AWS, Azure, GCP, etc.) @@ -49,7 +49,7 @@ For comprehensive guides and tutorials, see the official documentation: Prowler MCP Server can be used in three ways: -### 1. Prowler Cloud MCP Server (Recommended) +### 1. Hosted Prowler MCP (Recommended) **Use Prowler's managed MCP server at `https://mcp.prowler.com/mcp`** @@ -126,7 +126,7 @@ For complete tool descriptions and parameters, see the [Tools Reference](https:/ ### Tool Naming Convention All tools follow a consistent naming pattern with prefixes: -- `prowler_app_*` - Prowler Cloud and App (Self-Managed) management tools +- `prowler_*` - Prowler Cloud, Prowler Private Cloud & Prowler Local Server management tools - `prowler_hub_*` - Prowler Hub catalog and compliance tools - `prowler_docs_*` - Prowler documentation search and retrieval @@ -146,7 +146,7 @@ prowler_mcp_server/ **Key Features:** - **Modular Design**: Three independent sub-servers with prefixed namespacing -- **Auto-Discovery**: Prowler App tools are automatically discovered and registered +- **Auto-Discovery**: Prowler tools are automatically discovered and registered - **LLM Optimization**: Response models minimize token usage by excluding empty values - **Dual Transport**: Supports both STDIO (local) and HTTP (remote) modes @@ -174,17 +174,17 @@ The Prowler MCP Server enables powerful workflows through AI assistants: ## Requirements -**For Prowler Cloud MCP Server:** -- Prowler Cloud account and API key (only for Prowler Cloud/App features) +**For the hosted Prowler MCP:** +- Prowler Cloud account and API key (only for Prowler features) **For self-hosted STDIO/HTTP Mode:** - Python 3.12+ or Docker - Network access to: - `https://hub.prowler.com` (for Prowler Hub) - `https://docs.prowler.com` (for Prowler Documentation) - - Prowler Cloud API or self-hosted Prowler App API (for Prowler Cloud/App features) + - Prowler Cloud API or Prowler Local Server API (for Prowler features) -> **No Authentication Required**: Prowler Hub and Prowler Documentation features work without authentication. A Prowler API key is only required to access Prowler Cloud or Prowler App (Self-Managed) features. +> **No Authentication Required**: Prowler Hub and Prowler Documentation features work without authentication. A Prowler API key is only required to access Prowler features (Prowler Cloud, Prowler Private Cloud, or Prowler Local Server). ## Configuring MCP Hosts @@ -200,7 +200,7 @@ For developers looking to extend the MCP server with new tools or features: ## Related Products - **[Prowler Hub](https://hub.prowler.com)**: Browse security checks and compliance frameworks -- **[Prowler Cloud](https://cloud.prowler.com)**: Managed Prowler platform +- **[Prowler Cloud](https://cloud.prowler.com)**: Fully managed Prowler in the cloud - **[Lighthouse AI](https://docs.prowler.com/getting-started/products/prowler-lighthouse-ai)**: AI security analyst ## License diff --git a/mcp_server/prowler_mcp_server/__init__.py b/mcp_server/prowler_mcp_server/__init__.py index fe7af2dcea..1956a6ec41 100644 --- a/mcp_server/prowler_mcp_server/__init__.py +++ b/mcp_server/prowler_mcp_server/__init__.py @@ -5,7 +5,7 @@ This package provides MCP tools for accessing: - Prowler Hub: All security artifacts (detections, remediations and frameworks) supported by Prowler """ -__version__ = "0.5.0" +__version__ = "0.8.0" __author__ = "Prowler Team" __email__ = "engineering@prowler.com" diff --git a/mcp_server/prowler_mcp_server/prowler_app/models/__init__.py b/mcp_server/prowler_mcp_server/prowler_app/models/__init__.py index 899ab4e866..1b15e35ac9 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/models/__init__.py +++ b/mcp_server/prowler_mcp_server/prowler_app/models/__init__.py @@ -1,4 +1,4 @@ -"""Pydantic models for Prowler App MCP Server.""" +"""Pydantic models for Prowler MCP Server.""" from prowler_mcp_server.prowler_app.models.base import MinimalSerializerMixin from prowler_mcp_server.prowler_app.models.findings import ( diff --git a/mcp_server/prowler_mcp_server/prowler_app/models/finding_groups.py b/mcp_server/prowler_mcp_server/prowler_app/models/finding_groups.py index ae8431ba63..c2429012c3 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/models/finding_groups.py +++ b/mcp_server/prowler_mcp_server/prowler_app/models/finding_groups.py @@ -228,7 +228,7 @@ class FindingGroupResource(MinimalSerializerMixin): resource: FindingGroupResourceInfo = Field(description="Affected resource") provider: FindingGroupProviderInfo = Field(description="Affected provider") finding_id: str = Field( - description="Finding UUID to use with prowler_app_get_finding_details" + description="Finding UUID to use with prowler_get_finding_details" ) status: FindingStatus = Field(description="Finding status for this resource") severity: FindingSeverity = Field(description="Finding severity") diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/__init__.py b/mcp_server/prowler_mcp_server/prowler_app/tools/__init__.py index 4d740b6efe..8b5be76076 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/__init__.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/__init__.py @@ -1,4 +1,4 @@ -"""Domain-specific tools for Prowler App MCP Server. +"""Domain-specific tools for Prowler MCP Server. Each module in this package contains a BaseTool subclass that registers and implements tools for a specific domain (findings, providers, scans, etc.). diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/attack_paths.py b/mcp_server/prowler_mcp_server/prowler_app/tools/attack_paths.py index b08bbfe01f..5bd66760fa 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/attack_paths.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/attack_paths.py @@ -1,4 +1,4 @@ -"""Attack Paths tools for Prowler App MCP Server. +"""Attack Paths tools for Prowler MCP Server. This module provides tools for analyzing Attack Paths data from Neo4j graph database. Attack Paths help identify security risks by tracing potential attack vectors @@ -22,16 +22,16 @@ class AttackPathsTools(BaseTool): """Tools for Attack Paths analysis. Provides tools for: - - prowler_app_list_attack_paths_scans: Find completed scans ready for analysis - - prowler_app_list_attack_paths_queries: Discover available queries for a scan - - prowler_app_run_attack_paths_query: Execute query and analyze attack paths + - prowler_list_attack_paths_scans: Find completed scans ready for analysis + - prowler_list_attack_paths_queries: Discover available queries for a scan + - prowler_run_attack_paths_query: Execute query and analyze attack paths """ async def list_attack_paths_scans( self, provider_id: list[str] = Field( default=[], - description="Filter by Prowler's internal UUID(s) (v4) for specific provider(s). Use `prowler_app_search_providers` tool to find provider IDs", + description="Filter by Prowler's internal UUID(s) (v4) for specific provider(s). Use `prowler_search_providers` tool to find provider IDs", ), provider_type: list[str] = Field( default=[], @@ -73,8 +73,8 @@ class AttackPathsTools(BaseTool): Workflow: 1. Use this tool to find completed attack paths scans - 2. Use prowler_app_list_attack_paths_queries to see available queries for a scan - 3. Use prowler_app_run_attack_paths_query to execute analysis + 2. Use prowler_list_attack_paths_queries to see available queries for a scan + 3. Use prowler_run_attack_paths_query to execute analysis """ try: # Validate pagination @@ -113,7 +113,7 @@ class AttackPathsTools(BaseTool): async def list_attack_paths_queries( self, scan_id: str = Field( - description="UUID of a COMPLETED attack paths scan. Use `prowler_app_list_attack_paths_scans` with state=['completed'] to find scan IDs" + description="UUID of a COMPLETED attack paths scan. Use `prowler_list_attack_paths_scans` with state=['completed'] to find scan IDs" ), ) -> list[dict[str, Any]]: """Discover available Attack Paths queries for a completed scan. @@ -133,9 +133,9 @@ class AttackPathsTools(BaseTool): - aws-ec2-instances-internet-exposed: Find internet-exposed EC2 instances Workflow: - 1. Use prowler_app_list_attack_paths_scans to find a completed scan + 1. Use prowler_list_attack_paths_scans to find a completed scan 2. Use this tool to discover available queries - 3. Use prowler_app_run_attack_paths_query with query_id and any required parameters + 3. Use prowler_run_attack_paths_query with query_id and any required parameters """ try: api_response = await self.api_client.get( @@ -158,7 +158,7 @@ class AttackPathsTools(BaseTool): description="UUID of a COMPLETED attack paths scan. The scan must be in 'completed' state" ), query_id: str = Field( - description="Query ID to execute (e.g., 'aws-internet-exposed-ec2-sensitive-s3-access'). Use `prowler_app_list_attack_paths_queries` to discover available queries" + description="Query ID to execute (e.g., 'aws-internet-exposed-ec2-sensitive-s3-access'). Use `prowler_list_attack_paths_queries` to discover available queries" ), parameters: dict[str, str] = Field( default_factory=dict, @@ -194,7 +194,7 @@ class AttackPathsTools(BaseTool): Workflow: 1. Ensure scan is completed - 2. List available queries (use prowler_app_list_attack_paths_queries) + 2. List available queries (use prowler_list_attack_paths_queries) 3. Execute this tool with appropriate parameters 4. Analyze the returned graph for security insights """ @@ -231,7 +231,7 @@ class AttackPathsTools(BaseTool): async def get_attack_paths_cartography_schema( self, scan_id: str = Field( - description="UUID of a COMPLETED attack paths scan. Use `prowler_app_list_attack_paths_scans` with state=['completed'] to find scan IDs" + description="UUID of a COMPLETED attack paths scan. Use `prowler_list_attack_paths_scans` with state=['completed'] to find scan IDs" ), ) -> dict[str, Any]: """Retrieve the Cartography graph schema for a completed attack paths scan. @@ -253,10 +253,10 @@ class AttackPathsTools(BaseTool): - schema_content: Full Cartography schema markdown with node/relationship definitions Workflow: - 1. Use prowler_app_list_attack_paths_scans to find a completed scan + 1. Use prowler_list_attack_paths_scans to find a completed scan 2. Use this tool to get the schema for the scan's provider 3. Use the schema to craft custom openCypher queries - 4. Execute queries with prowler_app_run_attack_paths_query + 4. Execute queries with prowler_run_attack_paths_query """ try: api_response = await self.api_client.get( diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/compliance.py b/mcp_server/prowler_mcp_server/prowler_app/tools/compliance.py index 360dd5510d..33cdd22a69 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/compliance.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/compliance.py @@ -1,4 +1,4 @@ -"""Compliance framework tools for Prowler App MCP Server. +"""Compliance framework tools for Prowler MCP Server. This module provides tools for viewing compliance status and requirement details across all cloud providers. @@ -50,7 +50,7 @@ class ComplianceTools(BaseTool): if not scans_data: raise ValueError( f"No completed scans found for provider {provider_id}. " - "Run a scan first using prowler_app_trigger_scan." + "Run a scan first using prowler_trigger_scan." ) scan_id = scans_data[0]["id"] @@ -60,11 +60,11 @@ class ComplianceTools(BaseTool): self, scan_id: str | None = Field( default=None, - description="UUID of a specific scan to get compliance data for. Required if provider_id is not specified. Use `prowler_app_list_scans` to find scan IDs.", + description="UUID of a specific scan to get compliance data for. Required if provider_id is not specified. Use `prowler_list_scans` to find scan IDs.", ), provider_id: str | None = Field( default=None, - description="Prowler's internal UUID (v4) for a specific provider. If provided without scan_id, the tool will automatically find the latest completed scan for this provider. Use `prowler_app_search_providers` tool to find provider IDs.", + description="Prowler's internal UUID (v4) for a specific provider. If provided without scan_id, the tool will automatically find the latest completed scan for this provider. Use `prowler_search_providers` tool to find provider IDs.", ), ) -> dict[str, Any]: """Get high-level compliance overview across all frameworks for a specific scan. @@ -90,11 +90,11 @@ class ComplianceTools(BaseTool): Workflow: 1. Use this tool to get an overview of all compliance frameworks - 2. Use prowler_app_get_compliance_framework_state_details with a specific compliance_id to see which requirements failed + 2. Use prowler_get_compliance_framework_state_details with a specific compliance_id to see which requirements failed """ if not scan_id and not provider_id: return { - "error": "Either scan_id or provider_id must be provided. Use prowler_app_search_providers to find provider IDs or prowler_app_list_scans to find scan IDs." + "error": "Either scan_id or provider_id must be provided. Use prowler_search_providers to find provider IDs or prowler_list_scans to find scan IDs." } elif scan_id and provider_id: return { @@ -254,7 +254,7 @@ class ComplianceTools(BaseTool): async def get_compliance_framework_state_details( self, compliance_id: str = Field( - description="Compliance framework ID to get details for (e.g., 'cis_1.5_aws', 'pci_dss_v4.0_aws'). You can get compliance IDs from prowler_app_get_compliance_overview or consulting Prowler Hub/Prowler Documentation that you can also find in form of tools in this MCP Server", + description="Compliance framework ID to get details for (e.g., 'cis_1.5_aws', 'pci_dss_v4.0_aws'). You can get compliance IDs from prowler_get_compliance_overview or consulting Prowler Hub/Prowler Documentation that you can also find in form of tools in this MCP Server", ), scan_id: str | None = Field( default=None, @@ -262,14 +262,14 @@ class ComplianceTools(BaseTool): ), provider_id: str | None = Field( default=None, - description="Prowler's internal UUID (v4) for a specific provider. If provided without scan_id, the tool will automatically find the latest completed scan for this provider. Use `prowler_app_search_providers` tool to find provider IDs.", + description="Prowler's internal UUID (v4) for a specific provider. If provided without scan_id, the tool will automatically find the latest completed scan for this provider. Use `prowler_search_providers` tool to find provider IDs.", ), ) -> dict[str, Any]: """Get detailed requirement-level breakdown for a specific compliance framework. IMPORTANT: This tool returns DETAILED requirement information for a single compliance framework, focusing on FAILED requirements and their associated FAILED finding IDs. - Use this after prowler_app_get_compliance_overview to drill down into specific frameworks. + Use this after prowler_get_compliance_overview to drill down into specific frameworks. The markdown report includes: @@ -280,7 +280,7 @@ class ComplianceTools(BaseTool): 2. Failed Requirements Breakdown: - Each failed requirement's ID and description - Associated failed finding IDs for each failed requirement - - Use prowler_app_get_finding_details with these finding IDs for more details and remediation guidance + - Use prowler_get_finding_details with these finding IDs for more details and remediation guidance Default behavior: - Requires either scan_id OR provider_id @@ -289,14 +289,14 @@ class ComplianceTools(BaseTool): - Only shows failed requirements with their associated failed finding IDs Workflow: - 1. Use prowler_app_get_compliance_overview to identify frameworks with failures + 1. Use prowler_get_compliance_overview to identify frameworks with failures 2. Use this tool with the compliance_id to see failed requirements and their finding IDs - 3. Use prowler_app_get_finding_details with the finding IDs to get remediation guidance + 3. Use prowler_get_finding_details with the finding IDs to get remediation guidance """ # Validate that either scan_id or provider_id is provided if not scan_id and not provider_id: return { - "error": "Either scan_id or provider_id must be provided. Use prowler_app_search_providers to find provider IDs or prowler_app_list_scans to find scan IDs." + "error": "Either scan_id or provider_id must be provided. Use prowler_search_providers to find provider IDs or prowler_list_scans to find scan IDs." } # Resolve provider_id to latest scan_id if needed @@ -395,7 +395,7 @@ class ComplianceTools(BaseTool): report_lines.append("**Failed Finding IDs**: None found") report_lines.append("") report_lines.append( - "*Use `prowler_app_get_finding_details` with these finding IDs to get remediation guidance.*" + "*Use `prowler_get_finding_details` with these finding IDs to get remediation guidance.*" ) report_lines.append("") diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/finding_groups.py b/mcp_server/prowler_mcp_server/prowler_app/tools/finding_groups.py index 905a352740..05adf8db2b 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/finding_groups.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/finding_groups.py @@ -1,4 +1,4 @@ -"""Finding Groups tools for Prowler App MCP Server. +"""Finding Groups tools for Prowler MCP Server. This module provides read-only tools for finding group triage and drill-downs. """ @@ -233,8 +233,8 @@ class FindingGroupsTools(BaseTool): `date_to`, this uses `/finding-groups` with a maximum 2-day date window. Use this tool to find noisy or high-impact checks, then call - prowler_app_get_finding_group_details for complete counters or - prowler_app_list_finding_group_resources to drill into affected resources. + prowler_get_finding_group_details for complete counters or + prowler_list_finding_group_resources to drill into affected resources. """ try: self.api_client.validate_page_size(page_size) @@ -423,7 +423,7 @@ class FindingGroupsTools(BaseTool): Default behavior returns FAIL, unmuted resources so the result is actionable. Set `include_muted=True` to include accepted/suppressed resources too. Each row includes nested resource and provider data plus - `finding_id`. Use `prowler_app_get_finding_details(finding_id)` to + `finding_id`. Use `prowler_get_finding_details(finding_id)` to retrieve complete remediation guidance for a specific resource finding. """ try: diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/findings.py b/mcp_server/prowler_mcp_server/prowler_app/tools/findings.py index ec492c6a43..b556101cab 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/findings.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/findings.py @@ -1,4 +1,4 @@ -"""Security Findings tools for Prowler App MCP Server. +"""Security Findings tools for Prowler MCP Server. This module provides tools for searching, viewing, and analyzing security findings across all cloud providers. @@ -92,7 +92,7 @@ class FindingsTools(BaseTool): """Search and filter security findings across all cloud providers with rich filtering capabilities. IMPORTANT: This tool returns LIGHTWEIGHT findings. Use this for fast searching and filtering across many findings. - For complete details use prowler_app_get_finding_details on specific findings. + For complete details use prowler_get_finding_details on specific findings. Default behavior: - Returns latest findings from most recent scans (no date parameters needed) @@ -111,7 +111,7 @@ class FindingsTools(BaseTool): Workflow: 1. Use this tool to search and filter findings by severity, status, provider, service, region, etc. - 2. Use prowler_app_get_finding_details with the finding 'id' to get complete information about the finding + 2. Use prowler_get_finding_details with the finding 'id' to get complete information about the finding """ # Validate page_size parameter self.api_client.validate_page_size(page_size) @@ -187,9 +187,9 @@ class FindingsTools(BaseTool): """Retrieve comprehensive details about a specific security finding by its ID. IMPORTANT: This tool returns COMPLETE finding details. - Use this after finding a specific finding via prowler_app_search_security_findings + Use this after finding a specific finding via prowler_search_security_findings - This tool provides ALL information that prowler_app_search_security_findings returns PLUS: + This tool provides ALL information that prowler_search_security_findings returns PLUS: 1. Check Metadata (information about the check script that generated the finding): - title: Human-readable phrase used to summarize the check @@ -217,7 +217,7 @@ class FindingsTools(BaseTool): - resource_ids: List of UUIDs for cloud resources associated with this finding Workflow: - 1. Use prowler_app_search_security_findings to browse and filter findings + 1. Use prowler_search_security_findings to browse and filter findings 2. Use this tool with the finding 'id' to get remediation guidance and complete context """ params = { diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/muting.py b/mcp_server/prowler_mcp_server/prowler_app/tools/muting.py index 639f1ec3b7..37e1504165 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/muting.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/muting.py @@ -1,4 +1,4 @@ -"""Muting tools for Prowler App MCP Server. +"""Muting tools for Prowler MCP Server. This module provides tools for managing finding muting in Prowler, including: - Mutelist management (pattern-based bulk muting) @@ -43,7 +43,7 @@ class MutingTools(BaseTool): Workflow: 1. Use this tool to check if a mutelist is configured 2. Examine current muting patterns before making updates - 3. Use prowler_app_set_mutelist to create or update the configuration + 3. Use prowler_set_mutelist to create or update the configuration """ self.logger.info("Retrieving mutelist configuration...") @@ -61,7 +61,7 @@ class MutingTools(BaseTool): if len(data) == 0: return { "error": "No mutelist found", - "message": "No mutelist configuration exists for this tenant. Use prowler_app_set_mutelist to create one.", + "message": "No mutelist configuration exists for this tenant. Use prowler_set_mutelist to create one.", } # Return the first (and only) mutelist @@ -116,10 +116,10 @@ Structure: - Exceptions: Accounts, Regions, Resources to exclude from muting Workflow: - 1. Use prowler_app_get_mutelist to check existing configuration + 1. Use prowler_get_mutelist to check existing configuration 2. Build configuration object following Prowler mutelist format 3. Use this tool to create or update the mutelist - 4. Verify with prowler_app_get_mutelist + 4. Verify with prowler_get_mutelist """ self.logger.info("Setting mutelist configuration...") @@ -171,12 +171,12 @@ Structure: """Remove the mutelist configuration from the tenant. WARNING: This is a destructive operation that cannot be undone. - - The mutelist will need to be re-created with prowler_app_set_mutelist + - The mutelist will need to be re-created with prowler_set_mutelist - New findings from future scans will NOT be muted by the deleted mutelist - Previously muted findings remain muted (deletion doesn't un-mute them) Workflow: - 1. Use prowler_app_get_mutelist to confirm what will be deleted + 1. Use prowler_get_mutelist to confirm what will be deleted 2. Use this tool to permanently remove the mutelist 3. New scans will no longer apply mutelist-based muting """ @@ -229,7 +229,7 @@ Structure: """Search and filter mute rules with pagination support. IMPORTANT: This tool returns LIGHTWEIGHT mute rules without the full list of finding UIDs. - Use prowler_app_get_mute_rule to get complete details including all finding UIDs and creator information. + Use prowler_get_mute_rule to get complete details including all finding UIDs and creator information. Default behavior: - Returns all mute rules (both enabled and disabled) @@ -237,15 +237,15 @@ Structure: - Includes basic rule information without full finding UID lists Each mute rule includes: - - Core identification: id (UUID for prowler_app_get_mute_rule), name + - Core identification: id (UUID for prowler_get_mute_rule), name - Contextual information: reason, enabled status - State tracking: finding_count (number of findings currently muted) - Temporal data: inserted_at, updated_at timestamps Workflow: 1. Use this tool to search and filter mute rules by name, enabled status, or keywords - 2. Use prowler_app_get_mute_rule with the mute rule 'id' to get complete details including all finding UIDs - 3. Use prowler_app_update_mute_rule or prowler_app_delete_mute_rule to modify rules + 2. Use prowler_get_mute_rule with the mute rule 'id' to get complete details including all finding UIDs + 3. Use prowler_update_mute_rule or prowler_delete_mute_rule to modify rules """ self.logger.info("Listing mute rules...") self.api_client.validate_page_size(page_size) @@ -289,17 +289,17 @@ Structure: """Retrieve comprehensive details about a specific mute rule by its ID. IMPORTANT: This tool returns COMPLETE mute rule details including the full list of finding UIDs. - Use this after finding a rule via prowler_app_list_mute_rules. + Use this after finding a rule via prowler_list_mute_rules. - This tool provides ALL information that prowler_app_list_mute_rules returns PLUS: + This tool provides ALL information that prowler_list_mute_rules returns PLUS: - finding_uids: Complete list of finding UIDs that are muted by this rule - user_creator_id: UUID of the user who created the rule (audit trail) Workflow: - 1. Use prowler_app_list_mute_rules to find rules by name or filter criteria + 1. Use prowler_list_mute_rules to find rules by name or filter criteria 2. Use this tool with the rule 'id' to get complete details 3. Examine finding_uids list to understand which findings are muted - 4. Use prowler_app_update_mute_rule or prowler_app_delete_mute_rule to modify if needed + 4. Use prowler_update_mute_rule or prowler_delete_mute_rule to modify if needed """ self.logger.info(f"Retrieving mute rule {rule_id}...") @@ -323,7 +323,7 @@ Structure: description="Reason for muting these findings. Document why this security issue is acceptable or intentional (e.g., 'Development environment with controlled access', 'Legacy application requires IMDSv1')." ), finding_ids: list[str] = Field( - description="List of finding IDs (UUIDs) to mute. Get these from the prowler_app_search_security_findings tool. Must provide at least 1 finding ID." + description="List of finding IDs (UUIDs) to mute. Get these from the prowler_search_security_findings tool. Must provide at least 1 finding ID." ), ) -> dict[str, Any]: """Create a new mute rule to mute specific findings with documentation and audit trail. @@ -337,15 +337,15 @@ Structure: - Records creator for audit trail The mute rule includes: - - Core identification: id (UUID for prowler_app_get_mute_rule), name, reason + - Core identification: id (UUID for prowler_get_mute_rule), name, reason - Configuration: enabled status, finding_uids list - Audit trail: user_creator_id (UUID of the Prowler user from the tenant that created the rule), timestamps when the rule was created and last modified Workflow: - 1. Use prowler_app_search_security_findings to identify findings to mute + 1. Use prowler_search_security_findings to identify findings to mute 2. Use this tool with finding IDs, descriptive name, and documented reason - 3. Verify with prowler_app_get_mute_rule to confirm rule creation - 4. Check findings are muted with prowler_app_search_security_findings (filter by muted=true) + 3. Verify with prowler_get_mute_rule to confirm rule creation + 4. Check findings are muted with prowler_search_security_findings (filter by muted=true) """ self.logger.info(f"Creating mute rule '{name}'...") @@ -399,9 +399,9 @@ Structure: - enabled: Toggle rule active status (doesn't affect already-muted findings) Workflow: - 1. Use prowler_app_get_mute_rule to see current rule state + 1. Use prowler_get_mute_rule to see current rule state 2. Use this tool to update name, reason, or enabled status - 3. Verify changes with prowler_app_get_mute_rule + 3. Verify changes with prowler_get_mute_rule """ self.logger.info(f"Updating mute rule {rule_id}...") @@ -451,9 +451,9 @@ Structure: - Cannot be undone - rule must be recreated to restore Workflow: - 1. Use prowler_app_get_mute_rule to review what will be deleted + 1. Use prowler_get_mute_rule to review what will be deleted 2. Use this tool to permanently remove the rule - 3. Verify deletion with prowler_app_list_mute_rules (rule should no longer appear) + 3. Verify deletion with prowler_list_mute_rules (rule should no longer appear) """ self.logger.info(f"Deleting mute rule {rule_id}...") diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/providers.py b/mcp_server/prowler_mcp_server/prowler_app/tools/providers.py index b22d57d7b9..3ba417d677 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/providers.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/providers.py @@ -1,4 +1,4 @@ -"""Provider Management tools for Prowler App MCP Server. +"""Provider Management tools for Prowler MCP Server. This module provides tools for managing provider connections, including searching, connecting, and deleting providers. @@ -19,9 +19,9 @@ class ProvidersTools(BaseTool): """Tools for provider management operations Provides tools for: - - prowler_app_search_providers: Search and view configured providers with their connection status - - prowler_app_connect_provider: Connect or register a provider for security scanning in Prowler - - prowler_app_delete_provider: Permanently remove a provider from Prowler + - prowler_search_providers: Search and view configured providers with their connection status + - prowler_connect_provider: Connect or register a provider for security scanning in Prowler + - prowler_delete_provider: Permanently remove a provider from Prowler """ async def search_providers( @@ -145,7 +145,7 @@ class ProvidersTools(BaseTool): ) -> dict[str, Any]: """Register a provider to be scanned with Prowler. - This tool will register a provider in Prowler App, even if the UID is wrong. + This tool will register a provider in Prowler, even if the UID is wrong. If the provider is already registered, it will be updated with the new provided alias or credentials if provided. If credentials are provided, they will be added to the indicated provider, if the provider does not exist, it will be created and the credentials will be added to it. If the connection test is successful, the provider will be connected. @@ -292,13 +292,13 @@ class ProvidersTools(BaseTool): async def delete_provider( self, provider_id: str = Field( - description="Prowler's internal UUID (v4) for the provider to permanently remove, generated when the provider was registered in the system. Use `prowler_app_search_providers` tool to find the provider_id if you only know the alias or the provider's own identifier (provider_uid)" + description="Prowler's internal UUID (v4) for the provider to permanently remove, generated when the provider was registered in the system. Use `prowler_search_providers` tool to find the provider_id if you only know the alias or the provider's own identifier (provider_uid)" ), ) -> dict[str, Any]: """Permanently remove a registered provider from Prowler. WARNING: This is a destructive operation that cannot be undone. The provider will need to be - re-added with prowler_app_connect_provider if you want to scan it again. + re-added with prowler_connect_provider if you want to scan it again. The tool always returns the deletion status and message. """ diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/resources.py b/mcp_server/prowler_mcp_server/prowler_app/tools/resources.py index 011e013b91..88fcca25ae 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/resources.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/resources.py @@ -1,4 +1,4 @@ -"""Cloud Resources tools for Prowler App MCP Server. +"""Cloud Resources tools for Prowler MCP Server. This module provides tools for searching, viewing, and analyzing cloud resources across all providers. @@ -86,7 +86,7 @@ class ResourcesTools(BaseTool): IMPORTANT: This tool returns LIGHTWEIGHT resource information. Use this for fast searching and filtering across many resources. For complete configuration details, metadata, and finding - relationships, use prowler_app_get_resource on specific resources of interest. + relationships, use prowler_get_resource on specific resources of interest. This is the primary tool for browsing resources with rich filtering capabilities. Returns current state by default (latest scan per provider). Specify dates to query @@ -102,16 +102,16 @@ class ResourcesTools(BaseTool): - With dates: queries historical resource state (2-day maximum range between date_from and date_to) Each resource includes: - - Core identification: id (UUID for prowler_app_get_resource), uid, name + - Core identification: id (UUID for prowler_get_resource), uid, name - Location context: region, service, type - Security context: failed_findings_count (number of active security issues) - Tags: tags associated with the resource Useful Workflow: 1. Use this tool to search and filter resources by provider, region, service, tags, etc. - 2. Use prowler_app_get_resource with the resource 'id' to get complete configuration and metadata - 3. Use prowler_app_search_security_findings to find security issues for specific resources - 4. Use prowler_app_get_finding_details to get details about the security issues for specific resources + 2. Use prowler_get_resource with the resource 'id' to get complete configuration and metadata + 3. Use prowler_search_security_findings to find security issues for specific resources + 4. Use prowler_get_finding_details to get details about the security issues for specific resources """ # Validate page_size parameter self.api_client.validate_page_size(page_size) @@ -177,15 +177,15 @@ class ResourcesTools(BaseTool): async def get_resource( self, resource_id: str = Field( - description="Prowler's internal UUID (v4) for the resource to retrieve, generated when the resource was discovered in the system. Use `prowler_app_list_resources` tool to find the right ID" + description="Prowler's internal UUID (v4) for the resource to retrieve, generated when the resource was discovered in the system. Use `prowler_list_resources` tool to find the right ID" ), ) -> dict[str, Any]: """Retrieve comprehensive details about a specific resource by its ID. IMPORTANT: This tool provides COMPLETE resource details with all available information. - Use this after finding a specific resource via prowler_app_list_resources. + Use this after finding a specific resource via prowler_list_resources. - This tool provides ALL information that prowler_app_list_resources returns PLUS: + This tool provides ALL information that prowler_list_resources returns PLUS: 1. Configuration Details: - metadata: Provider-specific configuration (tags, policies, encryption settings, network rules) @@ -197,12 +197,12 @@ class ResourcesTools(BaseTool): 3. Security Relationships: - finding_ids: Prowler's internal UUIDs (v4) of all security findings associated with this resource - - Use prowler_app_get_finding_details on these IDs to get remediation guidance + - Use prowler_get_finding_details on these IDs to get remediation guidance Useful Workflow: - 1. Use prowler_app_list_resources to browse and filter across many resources + 1. Use prowler_list_resources to browse and filter across many resources 2. Use this tool to drill down into specific resources of interest - 3. Use prowler_app_get_finding_details to get details about the security issues for specific resources + 3. Use prowler_get_finding_details to get details about the security issues for specific resources """ params = {} @@ -348,7 +348,7 @@ class ResourcesTools(BaseTool): async def get_resource_events( self, resource_id: str = Field( - description="Prowler's internal UUID (v4) for the resource. Use `prowler_app_list_resources` to find the right ID, or get it from a finding's resource relationship via `prowler_app_get_finding_details`." + description="Prowler's internal UUID (v4) for the resource. Use `prowler_list_resources` to find the right ID, or get it from a finding's resource relationship via `prowler_get_finding_details`." ), lookback_days: int = Field( default=90, @@ -386,8 +386,8 @@ class ResourcesTools(BaseTool): - Identifying unauthorized or unexpected modifications Workflows: - 1. Resource browsing: prowler_app_list_resources → find resource → this tool for event history - 2. Incident investigation: prowler_app_get_finding_details → get resource ID from finding → this tool to identify who caused the issue, what they changed, and when + 1. Resource browsing: prowler_list_resources → find resource → this tool for event history + 2. Incident investigation: prowler_get_finding_details → get resource ID from finding → this tool to identify who caused the issue, what they changed, and when """ params = { "lookback_days": lookback_days, diff --git a/mcp_server/prowler_mcp_server/prowler_app/tools/scans.py b/mcp_server/prowler_mcp_server/prowler_app/tools/scans.py index 1df636ffc0..21d1431b71 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/tools/scans.py +++ b/mcp_server/prowler_mcp_server/prowler_app/tools/scans.py @@ -1,4 +1,4 @@ -"""Security Scans tools for Prowler App MCP Server. +"""Security Scans tools for Prowler MCP Server. This module provides tools for managing and monitoring Prowler security scans. """ @@ -20,18 +20,18 @@ class ScansTools(BaseTool): """Tools for security scan operations. Provides tools for: - - prowler_app_list_scans: Search and filter scans with rich filtering capabilities - - prowler_app_get_scan: Get comprehensive details about a specific scan - - prowler_app_trigger_scan: Trigger manual security scans for providers - - prowler_app_schedule_daily_scan: Schedule automated daily scans for continuous monitoring - - prowler_app_update_scan: Update scan names for better organization + - prowler_list_scans: Search and filter scans with rich filtering capabilities + - prowler_get_scan: Get comprehensive details about a specific scan + - prowler_trigger_scan: Trigger manual security scans for providers + - prowler_schedule_daily_scan: Schedule automated daily scans for continuous monitoring + - prowler_update_scan: Update scan names for better organization """ async def list_scans( self, provider_id: list[str] = Field( default=[], - description="Filter by Prowler's internal UUID(s) (v4) for specific provider(s), generated when the provider was registered. Use `prowler_app_search_providers` tool to find provider IDs", + description="Filter by Prowler's internal UUID(s) (v4) for specific provider(s), generated when the provider was registered. Use `prowler_search_providers` tool to find provider IDs", ), provider_type: list[str] = Field( default=[], @@ -56,7 +56,7 @@ class ScansTools(BaseTool): ), trigger: Literal["manual", "scheduled"] | None = Field( default=None, - description="Filter by how the scan was initiated. Options: 'manual' (user-initiated via prowler_app_trigger_scan), 'scheduled' (automated via prowler_app_schedule_daily_scan)", + description="Filter by how the scan was initiated. Options: 'manual' (user-initiated via prowler_trigger_scan), 'scheduled' (automated via prowler_schedule_daily_scan)", ), name: str | None = Field( default=None, @@ -75,7 +75,7 @@ class ScansTools(BaseTool): IMPORTANT: This tool returns LIGHTWEIGHT scan information. Use this for fast searching and filtering across many scans. For complete scan details including progress, duration, and resource counts, - use prowler_app_get_scan on specific scans of interest. + use prowler_get_scan on specific scans of interest. Default behavior: - Returns all scans @@ -83,15 +83,15 @@ class ScansTools(BaseTool): - Includes all scan states (available, scheduled, executing, completed, failed, cancelled) Each scan includes: - - Core identification: id (UUID for prowler_app_get_scan), name + - Core identification: id (UUID for prowler_get_scan), name - Execution context: state, trigger (manual/scheduled) - Temporal data: started_at, completed_at - Provider relationship: provider_id Workflow: 1. Use this tool to search and filter scans by provider, state, or date range - 2. Use prowler_app_get_scan with the scan 'id' to get progress, duration, and resource counts - 3. Use prowler_app_search_security_findings filtered by scan dates to analyze scan results + 2. Use prowler_get_scan with the scan 'id' to get progress, duration, and resource counts + 3. Use prowler_search_security_findings filtered by scan dates to analyze scan results """ # Validate pagination self.api_client.validate_page_size(page_size) @@ -128,15 +128,15 @@ class ScansTools(BaseTool): async def get_scan( self, scan_id: str = Field( - description="Prowler's internal UUID (v4) for the scan to retrieve, generated when the scan was created (e.g., '123e4567-e89b-12d3-a456-426614174000'). Use `prowler_app_list_scans` tool to find scan IDs" + description="Prowler's internal UUID (v4) for the scan to retrieve, generated when the scan was created (e.g., '123e4567-e89b-12d3-a456-426614174000'). Use `prowler_list_scans` tool to find scan IDs" ), ) -> dict[str, Any]: """Retrieve comprehensive details about a specific scan by its ID. IMPORTANT: This tool returns COMPLETE scan details. - Use this after finding a specific scan via prowler_app_list_scans. + Use this after finding a specific scan via prowler_list_scans. - This tool provides ALL information that prowler_app_list_scans returns PLUS: + This tool provides ALL information that prowler_list_scans returns PLUS: 1. Execution Details: - progress: Scan completion progress as percentage (0-100%) @@ -155,9 +155,9 @@ class ScansTools(BaseTool): - Understanding scan scheduling patterns Workflow: - 1. Use prowler_app_list_scans to browse and filter scans + 1. Use prowler_list_scans to browse and filter scans 2. Use this tool with the scan 'id' to monitor progress or view detailed results - 3. For completed scans, use prowler_app_search_security_findings filtered by date to analyze findings + 3. For completed scans, use prowler_search_security_findings filtered by date to analyze findings """ # Fetch scan with all fields params = { @@ -172,7 +172,7 @@ class ScansTools(BaseTool): async def trigger_scan( self, provider_id: str = Field( - description="Prowler's internal UUID (v4) for the provider to scan, generated when the provider was registered in the system (e.g., '4d0e2614-6385-4fa7-bf0b-c2e2f75c6877'). Use `prowler_app_search_providers` tool to find the provider ID" + description="Prowler's internal UUID (v4) for the provider to scan, generated when the provider was registered in the system (e.g., '4d0e2614-6385-4fa7-bf0b-c2e2f75c6877'). Use `prowler_search_providers` tool to find the provider ID" ), name: str | None = Field( default=None, @@ -182,14 +182,14 @@ class ScansTools(BaseTool): """Trigger a manual security scan for a provider. IMPORTANT: This tool returns immediately once the scan is created. - The scan will continue running in the background. Use `prowler_app_get_scan` + The scan will continue running in the background. Use `prowler_get_scan` with the returned scan ID to monitor progress and check when it completes. Example Useful Workflow: - 1. Use `prowler_app_search_providers` to find the provider_id you want to scan + 1. Use `prowler_search_providers` to find the provider_id you want to scan 2. Use this tool to trigger the scan - 3. Use `prowler_app_get_scan` with the returned scan 'id' to monitor progress - 4. Once completed, use `prowler_app_search_security_findings` to analyze results + 3. Use `prowler_get_scan` with the returned scan 'id' to monitor progress + 4. Once completed, use `prowler_search_security_findings` to analyze results """ try: # Build request data @@ -231,7 +231,7 @@ class ScansTools(BaseTool): return ScanCreationResult( scan=scan_info, status="success", - message=f"Scan {scan_id} created successfully. The scan may take some time to complete. Use prowler_app_get_scan tool with this ID to monitor progress.", + message=f"Scan {scan_id} created successfully. The scan may take some time to complete. Use prowler_get_scan tool with this ID to monitor progress.", ).model_dump() except Exception as e: @@ -245,7 +245,7 @@ class ScansTools(BaseTool): async def schedule_daily_scan( self, provider_id: str = Field( - description="Prowler's internal UUID (v4) for the provider to scan, generated when the provider was registered in the system (e.g., '4d0e2614-6385-4fa7-bf0b-c2e2f75c6877'). Use `prowler_app_search_providers` tool to find the provider ID" + description="Prowler's internal UUID (v4) for the provider to scan, generated when the provider was registered in the system (e.g., '4d0e2614-6385-4fa7-bf0b-c2e2f75c6877'). Use `prowler_search_providers` tool to find the provider ID" ), ) -> dict[str, Any]: """Schedule automated daily scans for a provider for continuous security monitoring. @@ -256,17 +256,17 @@ class ScansTools(BaseTool): you're not actively using the system. IMPORTANT: This tool returns immediately once the daily schedule is created. - The schedule will be set up in the background. Use `prowler_app_list_scans` + The schedule will be set up in the background. Use `prowler_list_scans` filtered by provider_id and trigger='scheduled' to view scheduled scans. IMPORTANT: This creates a PERSISTENT schedule. The provider will be scanned automatically every 24 hours until the provider is deleted. Example Useful Workflow: - 1. Use `prowler_app_search_providers` to find the provider_id you want to monitor + 1. Use `prowler_search_providers` to find the provider_id you want to monitor 2. Use this tool to create the daily schedule - 3. Use `prowler_app_list_scans` filtered by provider_id to view scheduled and completed scans - 4. Monitor findings over time with `prowler_app_search_security_findings` + 3. Use `prowler_list_scans` filtered by provider_id to view scheduled and completed scans + 4. Monitor findings over time with `prowler_search_security_findings` """ self.logger.info(f"Creating daily schedule for provider {provider_id}") task_response = await self.api_client.post( @@ -285,7 +285,7 @@ class ScansTools(BaseTool): ) if task_state == "available": - return_message = "Daily schedule created successfully. The schedule is being set up in the background. Use prowler_app_list_scans with provider_id filter to view scheduled scans." + return_message = "Daily schedule created successfully. The schedule is being set up in the background. Use prowler_list_scans with provider_id filter to view scheduled scans." else: return_message = "Daily schedule creation failed. Please try again later." @@ -297,7 +297,7 @@ class ScansTools(BaseTool): async def update_scan( self, scan_id: str = Field( - description="Prowler's internal UUID (v4) for the scan to update, generated when the scan was created (e.g., '123e4567-e89b-12d3-a456-426614174000'). Use `prowler_app_list_scans` tool to find the scan ID if you only know the provider or scan name. Returns an error if the scan ID is invalid or not found." + description="Prowler's internal UUID (v4) for the scan to update, generated when the scan was created (e.g., '123e4567-e89b-12d3-a456-426614174000'). Use `prowler_list_scans` tool to find the scan ID if you only know the provider or scan name. Returns an error if the scan ID is invalid or not found." ), name: str = Field( description="New human-friendly name for the scan (3-100 characters). Use descriptive names to improve organization and tracking, e.g., 'Production Security Audit - Q4 2025', 'Post-Deployment Compliance Check'. IMPORTANT: Only the scan name can be updated - other attributes (state, progress, duration) are read-only and managed by the system." @@ -309,7 +309,7 @@ class ScansTools(BaseTool): (state, progress, duration, etc.) are read-only and managed by the system. Example Useful Workflow: - 1. Use `prowler_app_list_scans` to find the scan you want to rename + 1. Use `prowler_list_scans` to find the scan you want to rename 2. Use this tool with the scan 'id' and new name """ api_response = await self.api_client.patch( diff --git a/mcp_server/prowler_mcp_server/prowler_app/utils/api_client.py b/mcp_server/prowler_mcp_server/prowler_app/utils/api_client.py index a6aacc3ce1..187364bee1 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/utils/api_client.py +++ b/mcp_server/prowler_mcp_server/prowler_app/utils/api_client.py @@ -1,4 +1,4 @@ -"""Shared API client utilities for Prowler App tools.""" +"""Shared API client utilities for Prowler tools.""" import asyncio from datetime import datetime, timedelta diff --git a/mcp_server/prowler_mcp_server/prowler_app/utils/auth.py b/mcp_server/prowler_mcp_server/prowler_app/utils/auth.py index 72c06000df..eff5d3a117 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/utils/auth.py +++ b/mcp_server/prowler_mcp_server/prowler_app/utils/auth.py @@ -10,7 +10,7 @@ from prowler_mcp_server.lib.logger import logger class ProwlerAppAuth: - """Handles authentication for Prowler App API using API keys or JWT tokens.""" + """Handles authentication for Prowler API using API keys or JWT tokens.""" def __init__( self, @@ -18,19 +18,23 @@ class ProwlerAppAuth: base_url: str = os.getenv("API_BASE_URL", "https://api.prowler.com/api/v1"), ): self.base_url = base_url.rstrip("/") - logger.info(f"Using Prowler App API base URL: {self.base_url}") + logger.info(f"Using Prowler API base URL: {self.base_url}") self.mode = mode self.access_token: str | None = None self.api_key: str | None = None if mode == "stdio": # STDIO mode - self.api_key = os.getenv("PROWLER_APP_API_KEY") + # PROWLER_API_KEY is the current variable; PROWLER_APP_API_KEY is kept + # as a backward-compatible fallback so existing setups keep working. + self.api_key = os.getenv("PROWLER_API_KEY") or os.getenv( + "PROWLER_APP_API_KEY" + ) if not self.api_key: - raise ValueError("PROWLER_APP_API_KEY environment variable is required") + raise ValueError("PROWLER_API_KEY environment variable is required") if not self.api_key.startswith("pk_"): - raise ValueError("Prowler App API key format is incorrect") + raise ValueError("Prowler API key format is incorrect") def _parse_jwt(self, token: str) -> dict | None: """Parse JWT token and return payload diff --git a/mcp_server/prowler_mcp_server/prowler_app/utils/tool_loader.py b/mcp_server/prowler_mcp_server/prowler_app/utils/tool_loader.py index b85c13af35..3a00474b5f 100644 --- a/mcp_server/prowler_mcp_server/prowler_app/utils/tool_loader.py +++ b/mcp_server/prowler_mcp_server/prowler_app/utils/tool_loader.py @@ -13,18 +13,27 @@ from prowler_mcp_server.lib.logger import logger from prowler_mcp_server.prowler_app.tools.base import BaseTool -def load_all_tools(mcp: FastMCP) -> None: - """Auto-discover and load all BaseTool subclasses from the tools package. +def load_all_tools( + mcp: FastMCP, + tools_package: str = "prowler_mcp_server.prowler_app.tools", +) -> None: + """Auto-discover and load all BaseTool subclasses from a tools package. This function: - 1. Dynamically imports all Python modules in the tools package - 2. Discovers all concrete BaseTool subclasses + 1. Dynamically imports all Python modules in the given tools package + 2. Discovers all concrete BaseTool subclasses defined in that package 3. Instantiates each tool class 4. Registers all tools with the provided FastMCP instance + ``BaseTool.__subclasses__()`` returns every subclass in the process, so the + discovered classes are filtered by ``__module__`` prefix. This keeps sibling + sub-servers (e.g. ``prowler_app`` and ``prowler_cloud``) from cross-registering + each other's tools, regardless of import order. + Args: mcp: The FastMCP instance to register tools with - TOOLS_PACKAGE: The package path containing tool modules (default: prowler_mcp_server.prowler_app.tools) + tools_package: The package path containing tool modules + (default: prowler_mcp_server.prowler_app.tools) Example: from fastmcp import FastMCP @@ -33,7 +42,7 @@ def load_all_tools(mcp: FastMCP) -> None: app = FastMCP("prowler-app") load_all_tools(app) """ - TOOLS_PACKAGE = "prowler_mcp_server.prowler_app.tools" + TOOLS_PACKAGE = tools_package logger.info(f"Auto-discovering tools from package: {TOOLS_PACKAGE}") # Import the tools package @@ -59,11 +68,14 @@ def load_all_tools(mcp: FastMCP) -> None: except Exception as e: logger.error(f"Failed to import module {module_name}: {e}") - # Discover all concrete BaseTool subclasses + # Discover all concrete BaseTool subclasses defined in this package only. + # __subclasses__() is process-wide, so filter by module to avoid sibling + # sub-servers cross-registering each other's tools. concrete_tools = [ tool_class for tool_class in BaseTool.__subclasses__() if not getattr(tool_class, "__abstractmethods__", None) + and tool_class.__module__.startswith(TOOLS_PACKAGE) ] logger.info(f"Discovered {len(concrete_tools)} tool classes") diff --git a/mcp_server/prowler_mcp_server/server.py b/mcp_server/prowler_mcp_server/server.py index 7c85641dee..a46ca12672 100644 --- a/mcp_server/prowler_mcp_server/server.py +++ b/mcp_server/prowler_mcp_server/server.py @@ -19,15 +19,15 @@ def setup_main_server(): except Exception as e: logger.error(f"Failed to mount Prowler Hub server: {e}") - # Mount Prowler App tools with prowler_app_ namespace + # Mount core Prowler tools with prowler_ namespace try: - logger.info("Mounting Prowler App server...") + logger.info("Mounting Prowler tools server...") from prowler_mcp_server.prowler_app.server import app_mcp_server - prowler_mcp_server.mount(app_mcp_server, namespace="prowler_app") - logger.info("Successfully mounted Prowler App server") + prowler_mcp_server.mount(app_mcp_server, namespace="prowler") + logger.info("Successfully mounted Prowler tools server") except Exception as e: - logger.error(f"Failed to mount Prowler App server: {e}") + logger.error(f"Failed to mount Prowler tools server: {e}") # Mount Prowler Documentation tools with prowler_docs_ namespace try: diff --git a/mcp_server/pyproject.toml b/mcp_server/pyproject.toml index 63aefdf931..f27e08b743 100644 --- a/mcp_server/pyproject.toml +++ b/mcp_server/pyproject.toml @@ -19,11 +19,13 @@ description = "MCP server for Prowler ecosystem" name = "prowler-mcp" readme = "README.md" requires-python = ">=3.12" -version = "0.5.0" +version = "0.8.0" [project.scripts] prowler-mcp = "prowler_mcp_server.main:main" +[tool.pytest] + [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/mcp_server/uv.lock b/mcp_server/uv.lock index 3258767442..a9afbeafd7 100644 --- a/mcp_server/uv.lock +++ b/mcp_server/uv.lock @@ -676,7 +676,7 @@ wheels = [ [[package]] name = "prowler-mcp" -version = "0.5.0" +version = "0.8.0" source = { editable = "." } dependencies = [ { name = "fastmcp" }, diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index e2f20c952f..408c76d3e1 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to the **Prowler SDK** are documented in this file. +## [5.35.0] (Prowler v5.35.0) + +### 🚀 Added + +- `excluded_checks` and `excluded_services` in scan configurations to narrow the execution scope [(#12028)](https://github.com/prowler-cloud/prowler/pull/12028) + +### 🔐 Security + +- Jira tenant information requests validate site names and do not follow redirects [(#12012)](https://github.com/prowler-cloud/prowler/pull/12012) + +--- + ## [5.34.0] (Prowler v5.34.0) ### 🚀 Added diff --git a/prowler/changelog.d/alibabacloud-security-group-policy-case.fixed.md b/prowler/changelog.d/alibabacloud-security-group-policy-case.fixed.md new file mode 100644 index 0000000000..cef4ff076c --- /dev/null +++ b/prowler/changelog.d/alibabacloud-security-group-policy-case.fixed.md @@ -0,0 +1 @@ +Alibaba Cloud SSH and RDP security group checks no longer produce false negatives when allowed rules use capitalized `Policy="Accept"` values diff --git a/prowler/changelog.d/bucket-validation-syntaxwarning.fixed.md b/prowler/changelog.d/bucket-validation-syntaxwarning.fixed.md new file mode 100644 index 0000000000..ceba09541c --- /dev/null +++ b/prowler/changelog.d/bucket-validation-syntaxwarning.fixed.md @@ -0,0 +1 @@ +Fix invalid escape sequence `SyntaxWarning` raised on startup by the S3 bucket name validation regex diff --git a/prowler/changelog.d/grouped-jira-dispatch.changed.md b/prowler/changelog.d/grouped-jira-dispatch.changed.md new file mode 100644 index 0000000000..8dd2e43ab5 --- /dev/null +++ b/prowler/changelog.d/grouped-jira-dispatch.changed.md @@ -0,0 +1 @@ +Jira output rendering supports grouped Finding Group issues with caller-provided links and capped or uncapped finding copy diff --git a/prowler/changelog.d/jira-tenant-info-request.security.md b/prowler/changelog.d/jira-tenant-info-request.security.md deleted file mode 100644 index 270ba01bfc..0000000000 --- a/prowler/changelog.d/jira-tenant-info-request.security.md +++ /dev/null @@ -1 +0,0 @@ -Jira tenant information requests validate site names and do not follow redirects diff --git a/prowler/changelog.d/sagemaker-notebook-no-secrets.added.md b/prowler/changelog.d/sagemaker-notebook-no-secrets.added.md new file mode 100644 index 0000000000..4789aa6aaf --- /dev/null +++ b/prowler/changelog.d/sagemaker-notebook-no-secrets.added.md @@ -0,0 +1 @@ +`sagemaker_notebook_instance_no_secrets` check for AWS provider, scanning SageMaker notebook instance lifecycle configuration scripts (`OnCreate` and `OnStart`) for hardcoded secrets such as API keys, passwords, tokens, and connection strings diff --git a/prowler/config/config.py b/prowler/config/config.py index c76f79910a..bee2984db3 100644 --- a/prowler/config/config.py +++ b/prowler/config/config.py @@ -49,7 +49,7 @@ class _MutableTimestamp: timestamp = _MutableTimestamp(datetime.today()) timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc)) -prowler_version = "5.35.0" +prowler_version = "5.36.0" html_logo_url = "https://github.com/prowler-cloud/prowler/" square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png" aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png" diff --git a/prowler/config/scan_config_schema.py b/prowler/config/scan_config_schema.py index ac00250c78..431c4cec60 100644 --- a/prowler/config/scan_config_schema.py +++ b/prowler/config/scan_config_schema.py @@ -9,21 +9,49 @@ The Prowler App, however, needs to surface those errors to the user when they save a Scan Config from the UI, and to expose the schema as JSON so the UI can validate live with `ajv`. This module provides: -- `validate_scan_config(payload)` — STRICT: returns a list of - `{path, message}` errors without silently dropping anything. The DRF - serializer (`api/.../v1/serializers.py:validate_scan_config_payload`) - turns each entry into a `ValidationError`. +- `validate_and_normalize_scan_config(payload)` — STRICT: returns + ``(normalized, errors)``. When ``errors`` is non-empty the normalized + dictionary is empty so callers never persist a partially validated + configuration. On success the normalized payload is JSON-serializable + (`model_dump(mode="json", exclude_unset=True)`), so the API can store + it directly in a Django ``JSONField`` and consume it at scan time + without re-running schema validation. + +- `validate_scan_config(payload)` — thin backward-compatible wrapper that + returns only the validation errors, preserved for callers that don't + need the normalized payload. - `SCAN_CONFIG_SCHEMA` — aggregated JSON Schema derived from the Pydantic models via `model_json_schema()`. Served by the `/scan-configs/schema` endpoint and consumed by the UI editor for in-editor live validation. """ +import json +from functools import lru_cache from typing import Any from pydantic import ValidationError from prowler.config.schema.registry import SCHEMAS +from prowler.lib.check.check import list_services +from prowler.lib.check.models import CheckMetadata + +# Pydantic v2 prefixes messages emitted from a ``field_validator`` that +# raises ``ValueError`` with this string. Strip it so the message that +# reaches the UI is the one the validator actually wrote. +_PYDANTIC_VALUE_ERROR_PREFIX = "Value error, " + + +@lru_cache(maxsize=None) +def _get_provider_check_ids(provider: str) -> frozenset[str]: + """Return cached check identifiers for a provider.""" + return frozenset(CheckMetadata.get_bulk(provider)) + + +@lru_cache(maxsize=None) +def _get_provider_services(provider: str) -> frozenset[str]: + """Return cached service identifiers for a provider.""" + return frozenset(list_services(provider)) def _format_loc(loc: tuple) -> str: @@ -50,48 +78,145 @@ def _format_loc(loc: tuple) -> str: return ".".join(parts) if parts else "" -def validate_scan_config(payload: Any) -> list[dict]: - """Validate a scan config payload against the registered provider schemas. +def validate_and_normalize_scan_config( + payload: Any, +) -> tuple[dict, list[dict[str, str]]]: + """Strict validation and normalization of a scan configuration payload. - Strict by design: every Pydantic violation surfaces as a `{path, message}` - entry so the caller can decide how to present it. Unknown provider - sections are accepted (consistent with `additionalProperties: True` at - the top level — the SDK simply has no opinion on them). + Returns ``(normalized, errors)``: + + - ``normalized`` is a JSON-serializable dict that mirrors the layout of + ``prowler/config/config.yaml`` (keyed by provider type). Registered + provider sections are dumped from their Pydantic models with + ``mode="json"`` (so the API can persist the result in a Django + ``JSONField``) and ``exclude_unset=True`` (so omitted defaults are + not injected into pre-existing configurations). Unknown provider + sections and unknown keys inside registered sections are preserved + untouched for forward compatibility with plugin-provided keys. + - ``errors`` is a list of ``{"path": , "message": }`` + entries, one per schema or exclusion-catalog violation. When any error + is present the normalized dictionary is returned empty so the caller + never persists a partially validated configuration. + + The input payload is never mutated. """ if not isinstance(payload, dict): - return [ + return {}, [ { "path": "", "message": "Scan config must be a mapping with provider sections.", } ] - errors: list[dict] = [] + errors: list[dict[str, str]] = [] + normalized: dict[str, Any] = {} + for provider, section in payload.items(): - schema_cls = SCHEMAS.get(provider) + # Reject non-string provider keys so distinct entries like ``123`` + # and ``"123"`` don't collide after ``str()`` in the normalized dict. + # YAML always produces string keys at this level; anything else + # comes from a hand-built payload and is a caller bug. + if not isinstance(provider, str): + errors.append( + { + "path": repr(provider), + "message": "provider keys must be strings.", + } + ) + continue + + provider_key = provider + schema_cls = SCHEMAS.get(provider_key) if schema_cls is None: - # Unknown provider type: tolerated. The SDK will simply ignore it. + # Unknown provider type: tolerated, but only when its contents + # are already JSON-serializable. The API persists the returned + # payload in a Django ``JSONField`` and would blow up at write + # time if we let a ``set()`` or similar through here. + try: + json.dumps(section) + except (TypeError, ValueError) as exc: + errors.append( + { + "path": provider_key, + "message": ( + "unknown provider section is not JSON-serializable: " + f"{exc}" + ), + } + ) + continue + normalized[provider_key] = section continue if not isinstance(section, dict): errors.append( { - "path": str(provider), + "path": provider_key, "message": "section must be a mapping.", } ) continue try: - schema_cls.model_validate(section) + model = schema_cls.model_validate(section) except ValidationError as exc: for err in exc.errors(): loc = err.get("loc") or () - path = _format_loc((str(provider), *loc)) - errors.append( - { - "path": path, - "message": err.get("msg", "validation error"), - } - ) + path = _format_loc((provider_key, *loc)) + message = err.get("msg", "validation error") + # Only strip on the specific error type that pydantic + # prefixes — a legitimate future message that happens to + # start with "Value error, " keeps its text intact. + if err.get("type") == "value_error" and message.startswith( + _PYDANTIC_VALUE_ERROR_PREFIX + ): + message = message[len(_PYDANTIC_VALUE_ERROR_PREFIX) :] + errors.append({"path": path, "message": message}) + continue + + if model.excluded_checks: + available_checks = _get_provider_check_ids(provider_key) + for index, check in enumerate(model.excluded_checks): + if check not in available_checks: + errors.append( + { + "path": f"{provider_key}.excluded_checks[{index}]", + "message": ( + f"Unknown check '{check}' for provider " + f"'{provider_key}'." + ), + } + ) + + if model.excluded_services: + available_services = _get_provider_services(provider_key) + for index, service in enumerate(model.excluded_services): + if service not in available_services: + errors.append( + { + "path": f"{provider_key}.excluded_services[{index}]", + "message": ( + f"Unknown service '{service}' for provider " + f"'{provider_key}'." + ), + } + ) + + normalized[provider_key] = model.model_dump(mode="json", exclude_unset=True) + + if errors: + return {}, errors + return normalized, [] + + +def validate_scan_config(payload: Any) -> list[dict]: + """Backward-compatible wrapper returning only validation errors. + + Preserved for callers that only need the strict-validation error list + (e.g. the DRF serializer that turns each entry into a + ``ValidationError``). New callers should prefer + :func:`validate_and_normalize_scan_config` to also receive the + normalized payload. + """ + _, errors = validate_and_normalize_scan_config(payload) return errors diff --git a/prowler/config/schema/base.py b/prowler/config/schema/base.py index cc473a4545..fc5a76af43 100644 --- a/prowler/config/schema/base.py +++ b/prowler/config/schema/base.py @@ -1,4 +1,12 @@ -from pydantic import BaseModel, ConfigDict +from typing import Annotated + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints, field_validator + +# Item type for excluded_checks / excluded_services list entries. Item +# whitespace is stripped via ``str_strip_whitespace`` on the base +# ``model_config`` (no second stripping implementation added here), so +# ``min_length=1`` catches "", " ", and any all-whitespace input uniformly. +NonEmptyScopeIdentifier = Annotated[str, StringConstraints(min_length=1)] class ProviderConfigBase(BaseModel): @@ -15,3 +23,28 @@ class ProviderConfigBase(BaseModel): str_strip_whitespace=True, validate_assignment=False, ) + + excluded_checks: list[NonEmptyScopeIdentifier] = Field( + default_factory=list, + description="Check identifiers to exclude from the scan scope.", + json_schema_extra={"default": [], "uniqueItems": True}, + ) + excluded_services: list[NonEmptyScopeIdentifier] = Field( + default_factory=list, + description="Service identifiers to exclude from the scan scope.", + json_schema_extra={"default": [], "uniqueItems": True}, + ) + + @field_validator("excluded_checks", "excluded_services") + @classmethod + def _reject_duplicates(cls, value: list[str]) -> list[str]: + seen: set[str] = set() + duplicates: set[str] = set() + for item in value: + if item in seen: + duplicates.add(item) + else: + seen.add(item) + if duplicates: + raise ValueError(f"duplicate values are not allowed: {sorted(duplicates)}") + return value diff --git a/prowler/lib/outputs/jira/jira.py b/prowler/lib/outputs/jira/jira.py index 32ee8af4eb..601120cdfc 100644 --- a/prowler/lib/outputs/jira/jira.py +++ b/prowler/lib/outputs/jira/jira.py @@ -417,6 +417,19 @@ class Jira: message=init_error, file=os.path.basename(__file__) ) + @staticmethod + def _sanitize_summary(summary: str) -> str: + """Normalize and truncate a Jira issue summary. + + Args: + summary: Raw summary text. + + Returns: + The summary collapsed to one line and limited to Jira's 255-character + summary maximum. + """ + return " ".join(summary.split())[:255] + @staticmethod def _build_code_block_content(code_value: str) -> Optional[Dict]: if not code_value: @@ -1155,6 +1168,101 @@ class Jira: return "#0000FF" return "#000000" # Default black color for unknown severities + @staticmethod + def _adf_colored_strong_marks(color_mark_type: str, color: str) -> list[dict]: + """Build ADF marks for bold text with a Jira color mark. + + Args: + color_mark_type: Jira ADF color mark type, such as textColor or + backgroundColor. + color: Hex color value for the mark. + + Returns: + ADF marks for strong colored text. + """ + return [ + {"type": "strong"}, + {"type": color_mark_type, "attrs": {"color": color}}, + ] + + def _adf_severity_marks( + self, severity: str = "", severity_color: str | None = None + ) -> list[dict]: + """Build ADF marks for severity text. + + Args: + severity: Finding severity used to derive a color when severity_color + is not provided. + severity_color: Optional explicit severity color. + + Returns: + ADF marks for highlighted severity text. + """ + color = severity_color or self.get_severity_color(str(severity).lower()) + return self._adf_colored_strong_marks("backgroundColor", color) + + def _adf_status_marks( + self, status: str = "", status_color: str | None = None + ) -> list[dict]: + """Build ADF marks for status text. + + Args: + status: Finding status used to derive a color when status_color is + not provided. + status_color: Optional explicit status color. + + Returns: + ADF marks for colored status text. + """ + color = status_color or self.get_color_from_status(str(status).upper()) + return self._adf_colored_strong_marks("textColor", color) + + @staticmethod + def _adf_text_node(text: str, marks: list[dict] | None = None) -> dict: + """Build an ADF text node. + + Args: + text: Text content for the node. + marks: Optional ADF marks to apply to the text. + + Returns: + ADF text node with optional marks. + """ + node = {"type": "text", "text": text} + if marks: + node["marks"] = marks + return node + + def _adf_severity_text_node( + self, severity: str = "", severity_color: str | None = None + ) -> dict: + """Build an ADF text node for severity. + + Args: + severity: Severity text to render. + severity_color: Optional explicit severity color. + + Returns: + ADF text node with severity marks. + """ + return self._adf_text_node( + severity, self._adf_severity_marks(severity, severity_color) + ) + + def _adf_status_text_node( + self, status: str = "", status_color: str | None = None + ) -> dict: + """Build an ADF text node for status. + + Args: + status: Status text to render. + status_color: Optional explicit status color. + + Returns: + ADF text node with status marks. + """ + return self._adf_text_node(status, self._adf_status_marks(status, status_color)) + def get_adf_description( self, check_id: str = "", @@ -1293,19 +1401,9 @@ class Jira: { "type": "paragraph", "content": [ - { - "type": "text", - "text": severity, - "marks": [ - {"type": "strong"}, - { - "type": "backgroundColor", - "attrs": { - "color": severity_color, - }, - }, - ], - } + self._adf_severity_text_node( + severity, severity_color + ) ], } ], @@ -1338,17 +1436,7 @@ class Jira: { "type": "paragraph", "content": [ - { - "type": "text", - "text": status, - "marks": [ - {"type": "strong"}, - { - "type": "textColor", - "attrs": {"color": status_color}, - }, - ], - } + self._adf_status_text_node(status, status_color) ], } ], @@ -1872,6 +1960,239 @@ class Jira: ], } + def get_grouped_adf_description( + self, + check_id: str = "", + check_title: str = "", + check_description: str = "", + severity: str = "", + status: str = "", + provider: str = "", + service: str = "", + affected_failing_resources: int = 0, + last_seen: str = "", + failing_for: str = "", + grouped_resources: list[dict] | None = None, + resources_total: int = 0, + resources_shown: int = 0, + finding_group_url: str = "", + finding_group_link_text: str = "", + risk: str = "", + recommendation_text: str = "", + recommendation_url: str = "", + ) -> dict: + """Build a Jira ADF description for a grouped finding issue. + + Args: + check_id: Finding check ID. + check_title: Finding check title. + check_description: Finding check description. + severity: Finding group severity. + status: Finding group status. + provider: Cloud provider name. + service: Provider service name. + affected_failing_resources: Number of failing resources in the group. + last_seen: Last time the finding group was seen. + failing_for: Duration the finding group has been failing. + grouped_resources: Resource rows to include in the grouped issue. + resources_total: Total number of resources in the group. + resources_shown: Number of resources rendered in this Jira issue. + finding_group_url: Optional URL for the full finding group. + finding_group_link_text: Optional link text for finding_group_url. + risk: Risk description for the check. + recommendation_text: Remediation recommendation text. + recommendation_url: Optional remediation recommendation URL. + + Returns: + Jira ADF document describing the finding group. + """ + + def _safe(value) -> str: + return str(value) if value not in (None, "") else "-" + + def _text(value, marks: list[dict] | None = None) -> dict: + node = {"type": "text", "text": _safe(value)} + if marks: + node["marks"] = marks + return node + + def _paragraph(value, marks: list[dict] | None = None) -> dict: + return {"type": "paragraph", "content": [_text(value, marks)]} + + def _cell(value, marks: list[dict] | None = None) -> dict: + return {"type": "tableCell", "content": [_paragraph(value, marks)]} + + def _content_cell(content: list[dict]) -> dict: + return {"type": "tableCell", "content": content} + + def _append_link(content: list[dict], url: str) -> list[dict]: + if not url: + return content + + link_node = { + "type": "text", + "text": url, + "marks": [{"type": "link", "attrs": {"href": url}}], + } + if content and content[-1].get("type") == "paragraph": + paragraph_content = content[-1].setdefault("content", []) + if paragraph_content: + last_inline = paragraph_content[-1] + if last_inline.get("type") != "text" or not last_inline.get( + "text", "" + ).endswith(" "): + paragraph_content.append({"type": "text", "text": " "}) + paragraph_content.append(link_node) + else: + content.append({"type": "paragraph", "content": [link_node]}) + return content + + def _row(cells: list[dict]) -> dict: + return {"type": "tableRow", "content": cells} + + strong = [{"type": "strong"}] + code = [{"type": "code"}] + severity_marks = self._adf_severity_marks(severity) + status_marks = self._adf_status_marks(status) + recommendation_content = _append_link( + self._markdown_converter.convert(_safe(recommendation_text)), + recommendation_url, + ) + main_rows = [ + _row([_cell("Check Id", strong), _cell(check_id, code)]), + _row([_cell("Check Title", strong), _cell(check_title)]), + _row([_cell("Severity", strong), _cell(severity, severity_marks)]), + _row([_cell("Status", strong), _cell(status, status_marks)]), + _row([_cell("Provider", strong), _cell(provider, code)]), + _row([_cell("Service", strong), _cell(service, code)]), + _row( + [ + _cell("Affected Failing Resources", strong), + _cell(affected_failing_resources, strong), + ] + ), + _row([_cell("Last Seen", strong), _cell(last_seen)]), + _row([_cell("Failing For", strong), _cell(failing_for)]), + _row( + [ + _cell("Risk", strong), + _content_cell(self._markdown_converter.convert(_safe(risk))), + ] + ), + _row( + [ + _cell("Recommendation", strong), + _content_cell(recommendation_content), + ] + ), + ] + + resource_rows = [ + _row( + [ + _cell("Resource", strong), + _cell("Resource UID", strong), + _cell("Provider", strong), + _cell("Service", strong), + _cell("Account / Tenant", strong), + _cell("Status", strong), + _cell("Severity", strong), + _cell("Region", strong), + _cell("Last Seen", strong), + _cell("Failing For", strong), + _cell("Triage", strong), + ] + ) + ] + for resource in grouped_resources or []: + resource_status = resource.get("status") + resource_severity = str(resource.get("severity", "")).upper() + resource_status_marks = self._adf_status_marks(resource_status) + resource_severity_marks = self._adf_severity_marks(resource_severity) + resource_rows.append( + _row( + [ + _cell(resource.get("resource_name"), code), + _cell(resource.get("resource_uid"), code), + _cell(resource.get("provider"), code), + _cell(resource.get("service"), code), + _cell(resource.get("provider_account"), code), + _cell(resource_status, resource_status_marks), + _cell(resource_severity, resource_severity_marks), + _cell(resource.get("region"), code), + _cell(resource.get("last_seen")), + _cell(resource.get("failing_for")), + _cell(resource.get("triage")), + ] + ) + ) + + content = [ + _paragraph("Prowler has discovered the following Finding Group:"), + {"type": "table", "attrs": {"layout": "full-width"}, "content": main_rows}, + ] + + content.extend( + [ + { + "type": "heading", + "attrs": {"level": 2}, + "content": [_text("Affected failing resources")], + }, + { + "type": "table", + "attrs": {"layout": "full-width"}, + "content": resource_rows, + }, + ] + ) + + if resources_total > resources_shown: + remaining_content = [ + _text(f"Showing {resources_shown} of {resources_total} Findings.") + ] + if finding_group_url and finding_group_link_text: + remaining_content = [ + _text( + f"Showing {resources_shown} of {resources_total} Findings " + "in this Jira issue. " + ), + _text( + finding_group_link_text, + [ + { + "type": "link", + "attrs": {"href": finding_group_url}, + } + ], + ), + ] + content.append( + { + "type": "paragraph", + "content": remaining_content, + } + ) + elif finding_group_url and finding_group_link_text: + content.append( + { + "type": "paragraph", + "content": [ + _text( + finding_group_link_text, + [ + { + "type": "link", + "attrs": {"href": finding_group_url}, + } + ], + ), + ], + } + ) + + return {"type": "doc", "version": 1, "content": content} + def send_findings( self, findings: list[Finding] = None, @@ -1965,7 +2286,7 @@ class Jira: summary_parts.append(finding.resource_uid) summary = " - ".join(summary_parts[1:]) - summary = f"{summary_parts[0]} {summary}"[:255] + summary = self._sanitize_summary(f"{summary_parts[0]} {summary}") payload = { "fields": { @@ -2048,11 +2369,13 @@ class Jira: self, check_id: str = "", check_title: str = "", + check_description: str = "", severity: str = "", status: str = "", status_extended: str = "", provider: str = "", region: str = "", + service: str = "", resource_uid: str = "", resource_name: str = "", risk: str = "", @@ -2069,6 +2392,14 @@ class Jira: issue_labels: list[str] = "", finding_url: str = "", tenant_info: str = "", + affected_failing_resources: int = 0, + grouped_resources: list[dict] | None = None, + resources_total: int = 0, + resources_shown: int = 0, + last_seen: str = "", + failing_for: str = "", + finding_group_url: str = "", + finding_group_link_text: str = "", ) -> bool: """ Send the finding to Jira @@ -2076,11 +2407,13 @@ class Jira: Args: - check_id: The check ID - check_title: The check title + - check_description: The check description - severity: The severity - status: The status - status_extended: The status extended - provider: The provider - region: The region + - service: The service - resource_uid: The resource UID - resource_name: The resource name - risk: The risk @@ -2097,6 +2430,15 @@ class Jira: - issue_labels: The issue labels - finding_url: The finding URL - tenant_info: The tenant info + - affected_failing_resources: The number of affected failing resources + - grouped_resources: The grouped resources to render, or None for a + single finding issue + - resources_total: The total resources in the finding group + - resources_shown: The resources shown in the Jira issue + - last_seen: The last time the finding group was seen + - failing_for: The duration the finding group has been failing + - finding_group_url: The finding group URL + - finding_group_link_text: The link text for the finding group URL Raises: - JiraRefreshTokenError: Failed to refresh the access token @@ -2140,40 +2482,66 @@ class Jira: status_color = self.get_color_from_status(status) severity_color = self.get_severity_color(severity.lower()) - adf_description = self.get_adf_description( - check_id=check_id, - check_title=check_title, - severity=severity.upper(), - severity_color=severity_color, - status=status, - status_color=status_color, - status_extended=status_extended, - provider=provider, - region=region, - resource_uid=resource_uid, - resource_name=resource_name, - risk=risk, - recommendation_text=recommendation_text, - recommendation_url=recommendation_url, - remediation_code_native_iac=remediation_code_native_iac, - remediation_code_terraform=remediation_code_terraform, - remediation_code_cli=remediation_code_cli, - remediation_code_other=remediation_code_other, - resource_tags=resource_tags, - compliance=compliance, - finding_url=finding_url, - tenant_info=tenant_info, - ) + if grouped_resources is not None: + adf_description = self.get_grouped_adf_description( + check_id=check_id, + check_title=check_title, + check_description=check_description, + severity=severity.upper(), + status=status, + provider=provider, + service=service, + affected_failing_resources=affected_failing_resources, + last_seen=last_seen, + failing_for=failing_for, + grouped_resources=grouped_resources, + resources_total=resources_total, + resources_shown=resources_shown, + finding_group_url=finding_group_url, + finding_group_link_text=finding_group_link_text, + risk=risk, + recommendation_text=recommendation_text, + recommendation_url=recommendation_url, + ) + else: + adf_description = self.get_adf_description( + check_id=check_id, + check_title=check_title, + severity=severity.upper(), + severity_color=severity_color, + status=status, + status_color=status_color, + status_extended=status_extended, + provider=provider, + region=region, + resource_uid=resource_uid, + resource_name=resource_name, + risk=risk, + recommendation_text=recommendation_text, + recommendation_url=recommendation_url, + remediation_code_native_iac=remediation_code_native_iac, + remediation_code_terraform=remediation_code_terraform, + remediation_code_cli=remediation_code_cli, + remediation_code_other=remediation_code_other, + resource_tags=resource_tags, + compliance=compliance, + finding_url=finding_url, + tenant_info=tenant_info, + ) summary_parts = ["[Prowler]"] if severity: summary_parts.append(severity.upper()) if check_id: summary_parts.append(check_id) - if resource_uid: + if grouped_resources is not None: + summary_parts.append( + f"{affected_failing_resources} affected failing resources" + ) + elif resource_uid: summary_parts.append(resource_uid) summary = " - ".join(summary_parts[1:]) - summary = f"{summary_parts[0]} {summary}"[:255] + summary = self._sanitize_summary(f"{summary_parts[0]} {summary}") payload = { "fields": { diff --git a/prowler/lib/scan/scan.py b/prowler/lib/scan/scan.py index 4bef660d33..b87cfcdf1d 100644 --- a/prowler/lib/scan/scan.py +++ b/prowler/lib/scan/scan.py @@ -178,25 +178,58 @@ class Scan: ) ) - # Exclude checks + # Validate excluded checks against the FULL provider catalog — not + # just the selected scope — so a global config can exclude a valid + # check even when that check is not part of a particular scoped run. + excluded_check_set: set[str] = set() if excluded_checks: - for check in excluded_checks: - if check in self._checks_to_execute: - self._checks_to_execute.remove(check) - else: - raise ScanInvalidCheckError( - f"Invalid check provided: {check}. Check does not exist in the provider." - ) + excluded_check_set = set(excluded_checks) + if len(excluded_check_set) != len(excluded_checks): + raise ScanInvalidCheckError( + "Duplicate excluded checks are not allowed." + ) + unknown_checks = excluded_check_set.difference(self._bulk_checks_metadata) + if unknown_checks: + raise ScanInvalidCheckError( + f"Invalid excluded check(s) provided: {sorted(unknown_checks)}." + ) - # Exclude services + # Validate excluded services against the provider service catalog. + # Only resolve the catalog when there is something to check to avoid + # walking the provider package tree unnecessarily. + excluded_service_set: set[str] = set() if excluded_services: - for check in self._checks_to_execute: - if get_service_name_from_check_name(check) in excluded_services: - self._checks_to_execute.remove(check) - else: - raise ScanInvalidServiceError( - f"Invalid service provided: {check}. Service does not exist in the provider." - ) + excluded_service_set = set(excluded_services) + if len(excluded_service_set) != len(excluded_services): + raise ScanInvalidServiceError( + "Duplicate excluded services are not allowed." + ) + unknown_services = excluded_service_set.difference( + list_services(provider.type) + ) + if unknown_services: + raise ScanInvalidServiceError( + f"Invalid excluded service(s) provided: {sorted(unknown_services)}." + ) + + if excluded_check_set or excluded_service_set: + previous_scope = self._checks_to_execute + selected_checks = { + check + for check in previous_scope + if check not in excluded_check_set + and get_service_name_from_check_name(check) not in excluded_service_set + } + # Only complain when exclusions actually emptied a non-empty + # scope. If the scope was already empty (e.g. a severity or + # category filter matched nothing) the exclusions did not + # cause the emptiness and the misleading error would obscure + # the real reason. + if previous_scope and not selected_checks: + raise ScanInvalidCheckError( + "The scan configuration excludes every selected check." + ) + self._checks_to_execute = sorted(selected_checks) self._number_of_checks_to_execute = len(self._checks_to_execute) diff --git a/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet.py b/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet.py index 72e579bbc6..45cfa81bda 100644 --- a/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet.py +++ b/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet.py @@ -26,7 +26,7 @@ class ecs_securitygroup_restrict_rdp_internet(Check): for ingress_rule in security_group.ingress_rules: # Check if rule allows traffic (policy == "accept") - if ingress_rule.get("policy", "accept") != "accept": + if str(ingress_rule.get("policy", "accept")).lower() != "accept": continue # Check protocol (tcp for RDP) diff --git a/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet.py b/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet.py index 9dfdd182e1..0315a69207 100644 --- a/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet.py +++ b/prowler/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet.py @@ -26,7 +26,7 @@ class ecs_securitygroup_restrict_ssh_internet(Check): for ingress_rule in security_group.ingress_rules: # Check if rule allows traffic (policy == "accept") - if ingress_rule.get("policy", "accept") != "accept": + if str(ingress_rule.get("policy", "accept")).lower() != "accept": continue # Check protocol (tcp for SSH) diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index 50f4665b2d..2d1632422b 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -235,7 +235,7 @@ def validate_arguments(arguments: Namespace) -> tuple[bool, str]: def validate_bucket(bucket_name: str) -> str: """validate_bucket validates that the input bucket_name is valid""" if search( - "^(?!^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$)(?!.*\.{2})(?!.*\.-)(?!.*-\.)(?!^xn--)(?!^sthree-)(?!^amzn-s3-demo-)(?!.*--table-s3$)[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", + r"^(?!^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$)(?!.*\.{2})(?!.*\.-)(?!.*-\.)(?!^xn--)(?!^sthree-)(?!^amzn-s3-demo-)(?!.*--table-s3$)[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", bucket_name, ): return bucket_name diff --git a/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/__init__.py b/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets.metadata.json b/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets.metadata.json new file mode 100644 index 0000000000..161abe5d30 --- /dev/null +++ b/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets.metadata.json @@ -0,0 +1,41 @@ +{ + "Provider": "aws", + "CheckID": "sagemaker_notebook_instance_no_secrets", + "CheckTitle": "SageMaker notebook instance lifecycle configuration contains no hardcoded secrets", + "CheckType": [ + "Software and Configuration Checks/AWS Security Best Practices", + "Sensitive Data Identifications/Passwords", + "Effects/Data Exposure" + ], + "ServiceName": "sagemaker", + "SubServiceName": "", + "ResourceIdTemplate": "", + "Severity": "high", + "ResourceType": "AwsSageMakerNotebookInstance", + "ResourceGroup": "ai_ml", + "Description": "**SageMaker notebook instance lifecycle configuration scripts** (`OnCreate` and `OnStart`) are analyzed for **embedded secrets**, detecting patterns like API keys, passwords, tokens, and connection strings. Findings reference the lifecycle hook and line numbers where potential secrets appear.", + "Risk": "**Hardcoded secrets** in lifecycle configuration scripts can be read by anyone with SageMaker access to the notebook instance, letting attackers reuse the credentials to access databases, APIs, or cloud resources, enabling data exfiltration and unauthorized changes.\n\nRotation is harder, increasing dwell time and blast radius of compromises.", + "RelatedUrl": "", + "AdditionalURLs": [ + "https://docs.aws.amazon.com/sagemaker/latest/dg/notebook-lifecycle-config.html" + ], + "Remediation": { + "Code": { + "CLI": "aws sagemaker update-notebook-instance-lifecycle-config --notebook-instance-lifecycle-config-name --on-start Content=", + "NativeIaC": "", + "Other": "1. Create a secret in AWS Secrets Manager for the hardcoded value.\n2. Update the notebook instance IAM role to allow secretsmanager:GetSecretValue on that secret.\n3. Edit the lifecycle script to fetch the secret at runtime instead of hardcoding it.\n4. Update the notebook instance lifecycle configuration.", + "Terraform": "" + }, + "Recommendation": { + "Text": "Use AWS Secrets Manager or Parameter Store to store secrets and retrieve them at runtime in lifecycle scripts; never hardcode them.", + "Url": "https://hub.prowler.com/check/sagemaker_notebook_instance_no_secrets" + } + }, + "Categories": [ + "secrets", + "gen-ai" + ], + "DependsOn": [], + "RelatedTo": [], + "Notes": "" +} diff --git a/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets.py b/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets.py new file mode 100644 index 0000000000..9b975596e4 --- /dev/null +++ b/prowler/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets.py @@ -0,0 +1,131 @@ +from prowler.lib.check.models import Check, Check_Report_AWS +from prowler.lib.utils.utils import ( + SecretsScanError, + annotate_verified_secrets, + detect_secrets_scan_batch, +) +from prowler.providers.aws.services.sagemaker.sagemaker_client import ( + sagemaker_client, +) + + +class sagemaker_notebook_instance_no_secrets(Check): + """Check for hardcoded secrets in SageMaker notebook instance lifecycle scripts. + + Scans the OnCreate and OnStart lifecycle configuration scripts of each + SageMaker notebook instance for hardcoded secrets such as API keys, + passwords, tokens, and connection strings. The scripts are fetched and + decoded by the SageMaker service; this check only consumes that data. + """ + + def execute(self): + """Execute the sagemaker_notebook_instance_no_secrets check. + + Returns: + list[Check_Report_AWS]: One report per SageMaker notebook + instance, with status PASS, FAIL, or MANUAL. + """ + findings = [] + notebook_instances = sagemaker_client.sagemaker_notebook_instances + if not notebook_instances: + return findings + + secrets_ignore_patterns = sagemaker_client.audit_config.get( + "secrets_ignore_patterns", [] + ) + validate = sagemaker_client.audit_config.get("secrets_validate", False) + + # Instances that actually contribute a script to the batch. Only these + # (plus instances whose describe/decode failed) may be marked MANUAL on + # a batch scan failure; instances with nothing to scan must PASS. + scanned_resources = { + notebook_instance.arn + for notebook_instance in notebook_instances + if notebook_instance.lifecycle_scripts + } + + def payloads(): + for notebook_instance in notebook_instances: + for fragment, script in notebook_instance.lifecycle_scripts.items(): + yield (notebook_instance.arn, fragment), script + + scan_error = None + try: + batch_results = detect_secrets_scan_batch( + payloads(), + excluded_secrets=secrets_ignore_patterns, + validate=validate, + ) + except SecretsScanError as error: + batch_results = {} + scan_error = error + + findings_by_instance = {} + for ( + resource_id, + fragment, + ), fragment_findings in batch_results.items(): + findings_by_instance.setdefault(resource_id, {})[ + fragment + ] = fragment_findings + + for notebook_instance in notebook_instances: + report = Check_Report_AWS( + metadata=self.metadata(), resource=notebook_instance + ) + + # MANUAL when the instance could not be fully scanned: either the + # lifecycle config describe/decode failed, or the batch scan failed + # for an instance that actually had scripts queued for scanning. + batch_failed = ( + scan_error is not None and notebook_instance.arn in scanned_resources + ) + if notebook_instance.lifecycle_scan_failed or batch_failed: + report.status = "MANUAL" + report.status_extended = ( + f"Could not fully scan SageMaker notebook instance " + f"{notebook_instance.name} lifecycle configuration for " + f"secrets; manual review is required." + ) + findings.append(report) + continue + + report.status = "PASS" + if not notebook_instance.lifecycle_config_name: + report.status_extended = ( + f"SageMaker notebook instance {notebook_instance.name} " + f"does not have a lifecycle configuration." + ) + else: + report.status_extended = ( + f"No secrets found in SageMaker notebook instance " + f"{notebook_instance.name} lifecycle configuration." + ) + + fragments_with_secrets = findings_by_instance.get(notebook_instance.arn) + + if fragments_with_secrets: + all_secrets = [] + secrets_findings = [] + + for fragment, fragment_findings in fragments_with_secrets.items(): + all_secrets.extend(fragment_findings) + secrets_string = ", ".join( + f"{secret['type']} on line {secret['line_number']}" + for secret in fragment_findings + ) + secrets_findings.append(f"{fragment}: {secrets_string}") + + final_output_string = "; ".join(secrets_findings) + report.status = "FAIL" + report.status_extended = ( + f"Potential {'secrets' if len(secrets_findings) > 1 else 'secret'} " + f"found in SageMaker notebook instance " + f"{notebook_instance.name} lifecycle configuration -> " + f"{final_output_string}." + ) + annotate_verified_secrets(report, all_secrets) + + findings.append(report) + + return findings diff --git a/prowler/providers/aws/services/sagemaker/sagemaker_service.py b/prowler/providers/aws/services/sagemaker/sagemaker_service.py index 20ea4c0280..6303ebbbf2 100644 --- a/prowler/providers/aws/services/sagemaker/sagemaker_service.py +++ b/prowler/providers/aws/services/sagemaker/sagemaker_service.py @@ -1,3 +1,4 @@ +import base64 from typing import Optional from botocore.client import ClientError @@ -37,6 +38,11 @@ class SageMaker(AWSService): self.__threading_call__( self._describe_notebook_instance, self.sagemaker_notebook_instances ) + # Runs after _describe_notebook_instance so lifecycle_config_name is set. + self.__threading_call__( + self._describe_notebook_instance_lifecycle_config, + self.sagemaker_notebook_instances, + ) self.__threading_call__( self._describe_training_job, self.sagemaker_training_jobs ) @@ -224,11 +230,61 @@ class SageMaker(AWSService): notebook_instance.direct_internet_access = True if "KmsKeyId" in describe_notebook_instance: notebook_instance.kms_key_id = describe_notebook_instance["KmsKeyId"] + if "NotebookInstanceLifecycleConfigName" in describe_notebook_instance: + notebook_instance.lifecycle_config_name = describe_notebook_instance[ + "NotebookInstanceLifecycleConfigName" + ] except Exception as error: logger.error( f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) + def _describe_notebook_instance_lifecycle_config(self, notebook_instance): + """Fetch and decode a notebook instance's lifecycle scripts. + + Reads the ``OnCreate`` and ``OnStart`` scripts from + ``DescribeNotebookInstanceLifecycleConfig`` and stores the base64-decoded + content on ``notebook_instance.lifecycle_scripts`` keyed by + ``"[]"``. Instances without a lifecycle configuration are + skipped. Any describe or decode failure sets + ``notebook_instance.lifecycle_scan_failed`` to True so the consuming + check can report ``MANUAL`` instead of a false ``PASS``. + + Args: + notebook_instance: NotebookInstance model to enrich in-place. + """ + if not notebook_instance.lifecycle_config_name: + return + logger.info("SageMaker - describing notebook instance lifecycle config...") + try: + regional_client = self.regional_clients[notebook_instance.region] + lifecycle_config = regional_client.describe_notebook_instance_lifecycle_config( + NotebookInstanceLifecycleConfigName=notebook_instance.lifecycle_config_name + ) + except Exception as error: + notebook_instance.lifecycle_scan_failed = True + logger.error( + f"{notebook_instance.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + return + + scripts = {} + for hook_name in ("OnCreate", "OnStart"): + for script_index, script in enumerate(lifecycle_config.get(hook_name, [])): + content_b64 = script.get("Content") + if not content_b64: + continue + try: + scripts[f"{hook_name}[{script_index}]"] = base64.b64decode( + content_b64 + ).decode("utf-8", errors="ignore") + except Exception as error: + notebook_instance.lifecycle_scan_failed = True + logger.error( + f"{notebook_instance.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + notebook_instance.lifecycle_scripts = scripts + def _describe_model(self, model): logger.info("SageMaker - describing models...") try: @@ -497,6 +553,13 @@ class NotebookInstance(BaseModel): subnet_id: str = None direct_internet_access: bool = None kms_key_id: str = None + lifecycle_config_name: str = None + # Decoded lifecycle scripts keyed by "[]" (e.g. "OnStart[0]"), + # populated by _describe_notebook_instance_lifecycle_config. + lifecycle_scripts: dict = {} + # True if the lifecycle configuration could not be fully described/decoded, + # so the secrets check reports MANUAL instead of a false PASS. + lifecycle_scan_failed: bool = False tags: Optional[list] = [] diff --git a/pyproject.toml b/pyproject.toml index 2edbcc428b..64affc8556 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"} name = "prowler" readme = "README.md" requires-python = ">=3.10,<3.14" -version = "5.35.0" +version = "5.36.0" [project.scripts] prowler = "prowler.__main__:prowler" diff --git a/skills/prowler-compliance/SKILL.md b/skills/prowler-compliance/SKILL.md index f119c7fa9b..747cb6ab64 100644 --- a/skills/prowler-compliance/SKILL.md +++ b/skills/prowler-compliance/SKILL.md @@ -2,23 +2,29 @@ name: prowler-compliance description: > Creates, syncs, audits and manages Prowler compliance frameworks end-to-end. - Covers the four-layer architecture (SDK models → JSON catalogs → output - formatters → API/UI), upstream sync workflows, cloud-auditor check-mapping - reviews, output formatter creation, and framework-specific attribute models. - Trigger: When working with compliance frameworks (CIS, NIST, PCI-DSS, SOC2, - GDPR, ISO27001, ENS, MITRE ATT&CK, CCC, C5, CSA CCM, KISA ISMS-P, - Prowler ThreatScore, FedRAMP, HIPAA), syncing with upstream catalogs, - auditing check-to-requirement mappings, adding output formatters, or fixing - compliance JSON bugs (duplicate IDs, empty Version, wrong Section, stale - check refs). + Covers the two supported JSON schemas (universal multi-provider and legacy + per-provider), the SDK model tree (legacy attribute classes, universal + ComplianceFramework, ConfigRequirements guardrails), output formatters + (legacy per-framework + universal data-driven), API/UI consumption, upstream + sync workflows, and cloud-auditor check-mapping reviews. + Trigger: When working with compliance frameworks (CIS, CIS Controls, NIST, + PCI-DSS, SOC2, GDPR, ISO27001, ENS, MITRE ATT&CK, CCC, C5, CSA CCM, DORA, + KISA ISMS-P, ASD Essential Eight, DISA STIG, CISA SCuBA, SecNumCloud, + FedRAMP, HIPAA, NIS2, Prowler ThreatScore), creating a universal + multi-provider framework, adding ConfigRequirements guardrails, syncing with + upstream catalogs, auditing check-to-requirement mappings, adding output + formatters, or fixing compliance JSON bugs (duplicate IDs, empty Version, + wrong Section, stale check refs). license: Apache-2.0 metadata: author: prowler-cloud - version: "1.2" + version: "2.0" scope: [root, sdk] auto_invoke: - "Creating/updating compliance frameworks" + - "Creating a universal (multi-provider) compliance framework" - "Mapping checks to compliance controls" + - "Adding ConfigRequirements guardrails to compliance requirements" - "Syncing compliance framework with upstream catalog" - "Auditing check-to-requirement mappings as a cloud auditor" - "Adding a compliance output formatter (per-provider class + table dispatcher)" @@ -29,527 +35,673 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task ## When to Use Use this skill when: -- Creating a new compliance framework for any provider + +- Creating a new compliance framework for any provider — **decide universal vs legacy first** (see below) - **Syncing an existing framework with an upstream source of truth** (CIS, FINOS CCC, CSA CCM, NIST, ENS, etc.) -- Adding requirements to existing frameworks +- Adding requirements to existing frameworks, or extending a universal framework to a new provider - Mapping checks to compliance controls -- **Auditing existing check mappings as a cloud auditor** (user asks "are these mappings correct?", "which checks apply to this requirement?", "review the mappings") -- **Adding a new output formatter** (new framework needs a table dispatcher + per-provider classes + CSV models) +- **Adding `ConfigRequirements` guardrails** so configurable checks can't silently satisfy a requirement with a loosened config +- **Auditing existing check mappings as a cloud auditor** ("are these mappings correct?", "which checks apply?", "review the mappings") +- **Adding a new legacy output formatter** (table dispatcher + per-provider classes + CSV models) - **Fixing JSON bugs**: duplicate IDs, empty Version, wrong Section, stale check refs, inconsistent FamilyName, padded tangential check mappings -- **Registering a framework in the CLI table dispatcher or API export map** - Investigating why a finding/check isn't showing under the expected compliance framework in the UI - Understanding compliance framework structures and attributes -## Four-Layer Architecture (Mental Model) +The authoritative contributor doc is `docs/developer-guide/security-compliance-framework.mdx` — +keep this skill and that doc consistent when either changes. For **reviewing** +a compliance PR, use the sister skill +[prowler-compliance-review](../prowler-compliance-review/SKILL.md) instead. -Prowler compliance is a **four-layer system** hanging off one Pydantic model tree. Bugs usually happen where one layer doesn't match another, so know all four before touching anything. +## Universal vs Legacy: The First Decision + +Prowler supports **two JSON schemas**. Choosing wrong means unnecessary Python +code, so decide this before anything else. At load time both converge: legacy +files are adapted into the universal `ComplianceFramework` model +(`adapt_legacy_to_universal()`), so the difference is about **authoring cost +and capabilities**, not about what the rest of Prowler sees. + +### Side-by-side comparison + +| | Universal (recommended for new frameworks) | Legacy provider-specific | +|---|---|---| +| File location | `prowler/compliance/.json` (top level) | `prowler/compliance//__.json` | +| Providers | Any number, one file (`checks` dict keyed by provider) | Exactly one provider per file (one file per provider to multi-cover) | +| Key style | lowercase (`framework`, `requirements`, `checks`) | Capitalized (`Framework`, `Requirements`, `Checks`) | +| Attribute schema | Declared **in the JSON itself** via `attributes_metadata`, validated at load | Pydantic class per framework family in `compliance_models.py` (code change for new shapes) | +| Attributes per requirement | One flat dict (`attributes: {...}`) | List of objects (`Attributes: [{...}]`) — only `Attributes[0]` is used downstream | +| Table/CSV/OCSF output | Data-driven from `outputs.table_config` — **zero Python changes** | Formatter package + registrations in `compliance.py`, `__main__.py`, `export.py` | +| Guardrails field | `config_requirements` (+ mandatory `Provider` per constraint) | `ConfigRequirements` (`Provider` omitted) | +| Loader behavior on error | Lenient: logs + skips file (`load_compliance_framework_universal`) | Fail-fast: `sys.exit(1)` (`load_compliance_framework`) | +| Loaded by | Only `get_bulk_compliance_frameworks_universal()` | Both loaders (`Compliance.get_bulk()` + universal, via adapter) | +| Shipped examples | `cis_controls_8.1.json`, `csa_ccm_4.0.json`, `dora_2022_2554.json` | Everything else (~105 files across 11 providers) | + +### When to use which + +**Use universal when** (any of these): + +- The framework is **new to Prowler** — no existing attribute class, no + existing formatter. This is the default: zero Python changes needed. +- The framework spans (or will span) **more than one provider** — DORA, CSA + CCM, CIS Controls. One file covers all providers; extending to a new + provider is a one-line `checks` edit. +- The attribute shape is **unique to this framework** — declare it in + `attributes_metadata` instead of adding a Pydantic class to the Union. + +**Use legacy only when extending an existing legacy family**: + +- A new **version** of a shipped legacy framework (CIS 8.0 for AWS → new + `cis_8.0_aws.json`, same `CIS_Requirement_Attribute`, same `cis/` formatter). +- An existing legacy framework for a **new provider** (ENS for m365 → new + `ens_rd2022_m365.json` + `ens_m365.py` transformer). +- Consistency with the family matters more than the universal benefits — a + lone `cis_8.0_aws` in universal format while 20+ CIS files stay legacy + would fragment the family. + +**Never**: start a brand-new single-provider framework as legacy "because it's +only AWS today". Universal handles single-provider fine (the `checks` dict +just has one key) and you skip 3 output files + 3 registrations. + +### The same requirement in both schemas + +Universal (`prowler/compliance/my_framework_1.0.json`): + +```json +{ + "framework": "My-Framework", + "name": "My Framework 1.0", + "version": "1.0", + "description": "...", + "attributes_metadata": [ + {"key": "Section", "type": "str", "required": true}, + {"key": "Service", "type": "str"} + ], + "outputs": {"table_config": {"group_by": "Section"}}, + "requirements": [ + { + "id": "MF-1.1", + "name": "Root MFA", + "description": "Root account must have MFA enabled.", + "attributes": {"Section": "IAM", "Service": "iam"}, + "checks": { + "aws": ["iam_root_mfa_enabled"], + "azure": [] + } + } + ] +} +``` + +Legacy (`prowler/compliance/aws/my_framework_1.0_aws.json` — plus a second +file per extra provider, plus formatter + registrations): + +```json +{ + "Framework": "My-Framework", + "Name": "My Framework 1.0 for AWS", + "Version": "1.0", + "Provider": "AWS", + "Description": "...", + "Requirements": [ + { + "Id": "MF-1.1", + "Name": "Root MFA", + "Description": "Root account must have MFA enabled.", + "Attributes": [ + {"ItemId": "MF-1.1", "Section": "IAM", "Service": "iam"} + ], + "Checks": ["iam_root_mfa_enabled"] + } + ] +} +``` + +Same control, but the universal file already covers Azure, validates its own +attribute schema, and renders table/CSV/OCSF with no code. Field-by-field +references for each schema follow below. + +## Architecture (Mental Model) + +Prowler compliance is a four-layer system. Bugs usually happen where one layer +doesn't match another, so know all four before touching anything. ### Layer 1: SDK / Core Models — `prowler/lib/check/` -- **`compliance_models.py`** — Pydantic **v1** model tree (`from pydantic.v1 import`). One `*_Requirement_Attribute` class per framework type + `Generic_Compliance_Requirement_Attribute` as fallback. -- `Compliance_Requirement.Attributes: list[Union[...]]` — **`Generic_Compliance_Requirement_Attribute` MUST be LAST** in the Union or every framework-specific attribute falls through to Generic (Pydantic v1 tries union members in order). -- **`compliance.py`** — runtime linker. `get_check_compliance()` builds the key as `f"{Framework}-{Version}"` **only if `Version` is non-empty**. An empty Version makes the key just `"{Framework}"` — this breaks downstream filters and tests that expect the versioned key. -- `Compliance.get_bulk(provider)` walks `prowler/compliance/{provider}/` and parses every `.json` file. No central index — just directory scan. +All in **Pydantic v1** (`from pydantic.v1 import ...`). Three model groups live +in `compliance_models.py`: -### Layer 2: JSON Frameworks — `prowler/compliance/{provider}/` +**Legacy tree** — `Compliance` → `Compliance_Requirement` / `Mitre_Requirement`: -See "Compliance Framework Location" and "Framework-Specific Attribute Structures" sections below. +- One `*_Requirement_Attribute` class per framework family. Registered today (Union order matters): + `ASDEssentialEight`, `CIS`, `ENS`, `ISO27001_2013`, `AWS_Well_Architected`, + `KISA_ISMSP`, `Prowler_ThreatScore`, `CCC`, `C5Germany`, `CSA_CCM`, `STIG` + (Okta IDaaS), and `Generic_Compliance_Requirement_Attribute` as fallback. +- **Generic MUST stay LAST** in `Compliance_Requirement.Attributes: list[Union[...]]` — + Pydantic v1 tries union members in order; Generic first would swallow every + framework-specific attribute. NIST 800-53/CSF, PCI DSS, GDPR, HIPAA, SOC2, + FedRAMP, SecNumCloud etc. intentionally use Generic. +- A `root_validator` rejects empty `Framework`, `Provider` or `Name`. +- MITRE uses the separate `Mitre_Requirement` model (`Tactics`, `SubTechniques`, + `Platforms`, `TechniqueURL` at requirement top level, per-provider + `Mitre_Requirement_Attribute_{AWS,Azure,GCP}`). -### Layer 3: Output Formatters — `prowler/lib/outputs/compliance/{framework}/` +**Universal tree** — `ComplianceFramework` → `UniversalComplianceRequirement`: -**Every framework directory follows this exact convention** — do not deviate: +- Flat `attributes: dict` per requirement, schema declared in + `attributes_metadata` (key, label, type, enum, required, `enum_display`, + `enum_order`, `output_formats`). A `root_validator` rejects missing required + keys, unknown keys (drift guard), enum violations, and int/float/bool type + mismatches. If `attributes_metadata` is omitted, **no validation runs**. +- `checks: dict[provider, list[check_id]]` — the provider list of the framework + is **derived** from these keys (`get_providers()` / `supports_provider()`); + the top-level `provider` field is only a fallback. +- `outputs.table_config` (group_by, split_by, scoring, labels) drives the CLI + table; `outputs.pdf_config` exists in the model but **is not consumed by the + API PDF pipeline yet** (see Layer 4). + +**Guardrails** — `Compliance_Requirement_ConfigConstraint`: + +- Fields `Check`, `ConfigKey`, `Operator` (`lte|gte|eq|in|subset|superset`), + `Value`, optional `Provider` (required in universal multi-provider files). +- A `root_validator` rejects Value/Operator type mismatches at load time. +- Evaluation is centralized in `prowler/lib/check/compliance_config_eval.py` + (`evaluate_config_constraints`, `apply_config_status`, `get_effective_status`, + `CONFIG_NOT_VALID_PREFIX = "Configuration not valid for this requirement."`), + shared by CSV/OCSF/table outputs **and** the API backend. A violated + constraint forces the requirement to FAIL and prepends the reason to + `status_extended`. Constraints whose `ConfigKey` is absent from + `audit_config` are skipped (defaults assumed compliant). + +**Loaders**: + +- `Compliance.get_bulk(provider)` — legacy: scans only + `prowler/compliance/{provider}/` (+ external JSONs via the + `prowler.compliance` entry-point group). Does NOT see top-level universal files. +- `get_bulk_compliance_frameworks_universal(provider)` — scans **both** the + top-level `prowler/compliance/` and every provider subdirectory, adapting + legacy files via `adapt_legacy_to_universal()` (flattens `Attributes[0]` to a + dict, wraps `Checks` as `{provider: [...]}`, infers `attributes_metadata`). + Also loads external universal frameworks via the + `prowler.compliance.universal` entry-point group (built-ins win collisions). +- `get_check_compliance(finding, provider_type, bulk_checks_metadata)` lives in + **`prowler/lib/outputs/compliance/compliance_check.py`** (not in + `lib/check/compliance.py`). It builds the per-finding dict keyed + `f"{Framework}-{Version}"` **only when Version is non-empty** — an empty + Version silently produces the key `"{Framework}"` and breaks downstream + filters and tests. +- `prowler/lib/check/compliance.py` now contains only + `update_checks_metadata_with_compliance()`. + +### Layer 2: JSON Catalogs — `prowler/compliance/` + +See "Compliance Catalog Coverage" below. + +### Layer 3: Output Formatters — `prowler/lib/outputs/compliance/` + +**Universal path** (no Python needed per framework): + +- `universal/universal_table.py` — `get_universal_table()`, renders the CLI + table from `outputs.table_config` + `attributes_metadata`. +- `universal/universal_output.py` — `UniversalComplianceOutput`, builds the CSV + Pydantic model **dynamically** from `attributes_metadata`. +- `universal/ocsf_compliance.py` — `OCSFComplianceOutput`; OCSF output is + **always generated** for universal frameworks regardless of `--output-formats`. +- Orchestrated by `process_universal_compliance_frameworks()` in + `compliance.py`, which runs **before** any legacy dispatch and removes the + processed frameworks from the set. + +**Legacy path** — per-framework directory, usually: ```text {framework}/ ├── __init__.py -├── {framework}.py # ONLY get_{framework}_table() — NO function docstring -├── {framework}_{provider}.py # One class per provider (e.g., CCC_AWS, CCC_Azure, CCC_GCP) -└── models.py # One Pydantic v2 BaseModel per provider (CSV columns) +├── {framework}.py # get_{framework}_table() summary-table function +├── {framework}_{provider}.py # One ComplianceOutput subclass per provider +└── models.py # One Pydantic CSV row model per provider ``` -- **`{framework}.py`** holds the **table dispatcher function** `get_{framework}_table()`. It prints the pass/fail/muted summary table. **Must NOT import `Finding` or `ComplianceOutput`** — doing so creates a circular import with `prowler/lib/outputs/compliance/compliance.py`. Only imports: `colorama`, `tabulate`, `prowler.config.config.orange_color`. -- **`{framework}_{provider}.py`** holds a per-provider class like `CCC_AWS(ComplianceOutput)` with a `transform()` method that walks findings and emits rows. This file IS allowed to import `Finding` because it's not on the dispatcher import chain. -- **`models.py`** holds one Pydantic v2 `BaseModel` per provider. Field names become CSV column headers (**public API** — renaming breaks downstream consumers). -- **Never collapse per-provider files into a unified parameterized class**, even when DRY-tempting. Every framework in Prowler follows the per-provider file pattern and reviewers will reject the refactor. CSV columns differ per provider (`AccountId`/`Region` vs `SubscriptionId`/`Location` vs `ProjectId`/`Location`) — three classes is the convention. -- **No function docstring on `get_{framework}_table()`** — no other framework has one; stay consistent. -- Register in `prowler/lib/outputs/compliance/compliance.py` → `display_compliance_table()` with an `elif compliance_framework.startswith("{framework}_"):` branch. Import the table function at the top of the file. +Directories today: `asd_essential_eight`, `aws_well_architected`, `c5`, `ccc`, +`cis`, `cisa_scuba`, `ens`, `generic`, `iso27001`, `kisa_ismsp`, +`mitre_attack`, `okta_idaas_stig`, `prowler_threatscore`, `universal`. +Known deviations (don't "fix" them without a reason): `iso27001/` has no table +file (falls to the generic table), `aws_well_architected/` has no per-provider +files, `cisa_scuba/` only ships googleworkspace. + +- CSV writers emit `;`-delimited files with UPPERCASE headers + (`ComplianceOutput.batch_write_data_to_file`). Field names in `models.py` + are **public API** — renaming breaks downstream consumers. +- **Circular import rule**: the table file (`{framework}.py`) must not import + `Finding` directly or transitively (`compliance.compliance` → table module → + `ComplianceOutput` → `Finding` → `get_check_compliance` → cycle). Keep table + files bare (`colorama`, `tabulate`, `prowler.config.config`); when a module + genuinely needs both, use `if TYPE_CHECKING:` or function-local imports (see + `universal_output.py` / `process_universal_compliance_frameworks`). +- Legacy table functions have no docstrings; the universal ones do. Match the + style of the file family you're touching. +- Dispatcher `display_compliance_table()` in `compliance.py` order: + universal (`table_config`) first → `cis_` → `ens_` → `mitre_attack` → + `kisa` → `prowler_threatscore_` → `c5_` → `ccc_` → `asd_essential_eight` + (substring) → `okta_idaas_stig` → else provider hook + (`provider.display_compliance_table()`, may raise `NotImplementedError`) → + `get_generic_compliance_table()`. iso27001, aws_well_architected and + cisa_scuba ride the fallback on purpose. ### Layer 4: API / UI -- **API table dispatcher**: `api/src/backend/tasks/jobs/export.py` → `COMPLIANCE_CLASS_MAP` keyed by provider. Uses `startswith` predicates: `(lambda name: name.startswith("ccc_"), CCC_AWS)`. **Never use exact match** (`name == "ccc_aws"`) — it's inconsistent and breaks versioning. -- **API lazy loader**: `api/src/backend/api/compliance.py` — `LazyComplianceTemplate` and `LazyChecksMapping` load compliance per provider on first access. -- **UI mapper routing**: `ui/lib/compliance/compliance-mapper.ts` routes framework names → per-framework mapper. -- **UI per-framework mapper**: `ui/lib/compliance/{framework}.tsx` flattens `Requirements` into a 3-level tree (Framework → Category → Control → Requirement) for the accordion view. Groups by `Attributes[0].FamilyName` and `Attributes[0].Section`. -- **UI detail panel**: `ui/components/compliance/compliance-custom-details/{framework}-details.tsx`. -- **UI types**: `ui/types/compliance.ts` — TypeScript mirrors of the attribute metadata. +- **API lazy loaders**: `api/src/backend/api/compliance.py` — + `LazyComplianceTemplate` / `LazyChecksMapping` (per-provider lazy caches over + `get_bulk_compliance_frameworks_universal`, with Gunicorn background warm-up). +- **API CSV export dispatch**: `COMPLIANCE_CLASS_MAP` in + `api/src/backend/tasks/jobs/export.py`, consumed from `tasks/tasks.py`. It is + a dict `provider → [(predicate, exporter_class)]` with `GenericCompliance` as + fallback. Predicates mix **`startswith` for multi-version families** + (`cis_`, `ens_`, `iso27001_`, `ccc_`, `cisa_scuba_`, ...) and **exact + `name == ...` for true singletons** (`mitre_attack_aws`, + `prowler_threatscore_*`, `asd_essential_eight_aws` — and inconsistently + `c5_azure`/`c5_gcp`, while aws uses `startswith("c5_")`). Rule of thumb: if + the framework can ever grow versions or variants, use `startswith`. +- **API overview ingestion**: `create_compliance_requirements()` in + `api/src/backend/tasks/jobs/scan.py` builds per-region rows from the lazy + template and persists `ComplianceRequirementOverview` (COPY with bulk-create + fallback) plus `ComplianceOverviewSummary`. +- **API PDF reports**: `api/src/backend/tasks/jobs/reports/` — hardcoded + `FRAMEWORK_REGISTRY` (own `FrameworkConfig` dataclass, NOT the SDK + `PDFConfig`) with one generator class per framework. Only + `prowler_threatscore`, `ens`, `nis2`, `csa_ccm` and `cis` have PDFs today; + adding one means a generator class + registry entry + wiring in `report.py`. +- **UI mapper routing**: `ui/lib/compliance/compliance-mapper.ts` — + `getComplianceMappers()` keyed by the JSON's `framework` value + (e.g. `"CIS"`, `"CIS-Controls"`, `"DORA"`, `"Okta-IDaaS-STIG"`). Unregistered + frameworks **fall back to the generic mapper + `GenericCustomDetails` + automatically** — a dedicated mapper/detail panel is a first-class upgrade, + not a requirement to render. +- **UI grouping varies per mapper**: generic/cis group by + `Section`/`SubSection`, iso by `Category`, ccc by `FamilyName`. All read + `attributes[0]` — inconsistent values within one JSON become separate tree + branches, so normalize before shipping. +- **UI types**: `ui/types/compliance.ts` — one `*AttributesMetadata` interface + per framework, added to the `AttributesItemData` metadata union. +- **UI icons**: `ui/components/icons/compliance/` + `IconCompliance.tsx`. + Registration is an ordered substring match (`COMPLIANCE_LOGOS`): put + framework-specific keywords **before** generic ones (`nist` before `nis2`, + `cisa` before `cis`; `aws` deliberately last). ### The CLI Pipeline (end-to-end) ```text -prowler aws --compliance ccc_aws +prowler aws --compliance cis_7.0_aws # framework key = JSON basename ↓ -Compliance.get_bulk("aws") → parses prowler/compliance/aws/*.json +Compliance.get_bulk("aws") # legacy frameworks +get_bulk_compliance_frameworks_universal("aws") # legacy (adapted) + universal ↓ -update_checks_metadata_with_compliance() → attaches compliance info to CheckMetadata +update_checks_metadata_with_compliance() # attaches compliance to CheckMetadata ↓ -execute_checks() → runs checks, produces Finding objects +execute_checks() → Finding objects ↓ -get_check_compliance(finding, "aws", bulk_checks_metadata) - → dict "{Framework}-{Version}" → [requirement_ids] +get_check_compliance(finding, "aws", bulk) # dict "{Framework}-{Version}" → [req_ids] ↓ -CCC_AWS(findings, compliance).transform() → per-provider class builds CSV rows +process_universal_compliance_frameworks() # universal: CSV + OCSF, then removed from set +per-provider elif branches in __main__.py # legacy: AWSCIS(...).batch_write_data_to_file() ↓ -batch_write_data_to_file() → writes {output_filename}_ccc_aws.csv - ↓ -display_compliance_table() → get_ccc_table() → prints stdout summary +display_compliance_table() # universal table first, then legacy elifs, + # then generic fallback ``` --- -## Compliance Framework Location +## Compliance Catalog Coverage -Frameworks are JSON files located in: `prowler/compliance/{provider}/{framework_name}_{provider}.json` +Counts as of 2026-07 (109 JSON files). Regenerate before trusting them: -**Supported Providers:** -- `aws` - Amazon Web Services -- `azure` - Microsoft Azure -- `gcp` - Google Cloud Platform -- `kubernetes` - Kubernetes -- `github` - GitHub -- `m365` - Microsoft 365 -- `alibabacloud` - Alibaba Cloud -- `cloudflare` - Cloudflare -- `oraclecloud` - Oracle Cloud -- `oci` - Oracle Cloud Infrastructure -- `nhn` - NHN Cloud -- `mongodbatlas` - MongoDB Atlas -- `iac` - Infrastructure as Code -- `llm` - Large Language Models +```bash +for d in prowler/compliance/*/; do printf "%s: %s\n" "$(basename $d)" "$(ls $d*.json 2>/dev/null | wc -l)"; done +ls prowler/compliance/*.json # universal, top-level +``` -## Base Framework Structure +**Universal (top-level, multi-provider)**: `cis_controls_8.1.json` (18 +providers), `csa_ccm_4.0.json` (aws/azure/gcp/alibabacloud/oraclecloud), +`dora_2022_2554.json` (aws/azure/gcp/alibabacloud/cloudflare). -All compliance frameworks share this base structure: +**Legacy per-provider** (families, not exhaustive versions): + +| Provider | # | Framework families | +|---|---|---| +| aws | 45 | CIS 1.4–7.0, NIST 800-53 r4/r5, NIST 800-171 r2, NIST CSF 1.1/2.0, PCI 3.2.1/4.0, ISO 27001 2013/2022, HIPAA, GDPR, SOC2, FedRAMP low/moderate r4 + 20x KSI low, ENS RD2022, MITRE ATT&CK, C5, CCC, CISA, FFIEC, RBI, Well-Architected (security/reliability), FTR, FSBP, AWS AI Security Framework, AWS Account Security Onboarding, Audit Manager Control Tower, GxP 21 CFR 11 / EU Annex 11, KISA ISMS-P 2023 (en+ko), NIS2, ASD Essential Eight, SecNumCloud 3.2, Prowler ThreatScore | +| azure | 19 | CIS 2.0–6.0, ISO 27001 2022, ENS RD2022, MITRE ATT&CK, PCI 4.0, HIPAA, SOC2, NIS2, RBI, C5, CCC, FedRAMP 20x KSI low, SecNumCloud 3.2, Prowler ThreatScore | +| gcp | 17 | CIS 2.0–5.0, ISO 27001 2022, ENS RD2022, MITRE ATT&CK, PCI 4.0, HIPAA, SOC2, NIS2, RBI, C5, CCC, FedRAMP 20x KSI low, SecNumCloud 3.2, Prowler ThreatScore | +| kubernetes | 8 | CIS 1.8–2.0.1, ISO 27001 2022, PCI 4.0, Prowler ThreatScore | +| m365 | 5 | CIS 4.0/6.0/7.0, ISO 27001 2022, Prowler ThreatScore | +| alibabacloud | 3 | CIS 2.0, SecNumCloud 3.2, Prowler ThreatScore | +| oraclecloud | 3 | CIS 3.0/3.1, SecNumCloud 3.2 | +| github | 2 | CIS 1.0/1.2.0 | +| googleworkspace | 2 | CIS 1.3, CISA SCuBA 0.6 | +| okta | 1 | Okta IDaaS STIG V1R2 | +| nhn | 1 | ISO 27001 2022 | + +Providers with a compliance directory but no frameworks yet: cloudflare, iac, +linode, llm, mongodbatlas, openstack, stackit. Provider keys inside universal +`checks` dicts must match directory names under `prowler/providers/` (lowercase). + +--- + +## Universal Schema Reference + +Full spec in `docs/developer-guide/security-compliance-framework.mdx`. Skeleton: + +```json +{ + "framework": "DORA", + "name": "Digital Operational Resilience Act (DORA) 2022/2554", + "version": "2022/2554", + "description": "Shown in --list-compliance and PDF reports.", + "icon": "dora", + "attributes_metadata": [ + {"key": "Pillar", "label": "Pillar", "type": "str", "required": true, + "enum": ["ICT Risk Management", "..."], + "output_formats": {"csv": true, "ocsf": true}}, + {"key": "Article", "type": "str", "required": true} + ], + "outputs": { + "table_config": {"group_by": "Pillar"}, + "pdf_config": {"group_by_field": "Pillar", "charts": ["..."]} + }, + "requirements": [ + { + "id": "DORA-Art5", + "name": "Governance and organisation", + "description": "Requirement text verbatim from the source.", + "attributes": {"Pillar": "ICT Risk Management", "Article": "Article 5"}, + "checks": { + "aws": ["iam_no_root_access_key"], + "azure": [], + "gcp": [] + }, + "config_requirements": [ + {"Check": "iam_user_accesskey_unused", "Provider": "aws", + "ConfigKey": "max_unused_access_keys_days", "Operator": "lte", "Value": 45} + ] + } + ] +} +``` + +### Universal fields, top level (`ComplianceFramework`) + +| Field | Type | Required | Notes | +|---|---|---|---| +| `framework` | string | Yes | Short identifier (`DORA`, `CSA-CCM`, `CIS-Controls`). This is the key the UI mapper routes on. | +| `name` | string | Yes | Human-readable full name. | +| `version` | string | No (never leave empty) | Framework version/edition (`8.1`, `2022/2554`). | +| `description` | string | Yes | Shown in `--list-compliance` and PDF reports. | +| `provider` | string | No | Fallback only — the effective provider list is derived from `checks` keys across requirements (`get_providers()`). | +| `icon` | string | No | Short icon slug. | +| `attributes_metadata` | array | No (strongly recommended) | Declares the schema of every `attributes` key. **If omitted, no attribute validation runs at all.** | +| `outputs` | object | No | `table_config` (CLI table) + `pdf_config` (modeled, not yet consumed by the API). | +| `requirements` | array | Yes | List of requirement objects (below). | + +### Universal fields, per requirement (`UniversalComplianceRequirement`) + +| Field | Type | Required | Notes | +|---|---|---|---| +| `id` | string | Yes | Unique within the framework. | +| `description` | string | Yes | Requirement text verbatim from the source. | +| `name` | string | No | Short title. | +| `attributes` | dict | No (default `{}`) | Flat dict; every key must be declared in `attributes_metadata` (unknown keys are rejected at load when metadata exists). | +| `checks` | dict | No (default `{}`) | `{provider: [check_ids]}`, lowercase keys matching `prowler/providers/` dirs. Empty list = manual requirement for that provider. | +| `config_requirements` | array | No | Guardrails; each constraint **must** carry `Provider`. | +| `tactics`, `sub_techniques`, `platforms`, `technique_url` | — | No | MITRE-style extras (auto-populated when adapting legacy MITRE files). | + +### `attributes_metadata` entry fields (`AttributeMetadata`) + +| Field | Type | Notes | +|---|---|---| +| `key` | string (required) | Attribute name as used in `requirement.attributes`. | +| `label` | string | Human-readable label for CSV headers / PDF. | +| `type` | string | `str` (default), `int`, `float`, `bool`, `list_str`, `list_dict`. Only int/float/bool are enforced at load; the rest are documentation. | +| `enum` | list | Allowed values — enforced at load. Use it whenever the value set is closed. | +| `required` | bool | Enforced at load: every requirement must carry the key non-null. | +| `enum_display` / `enum_order` | dict / list | Per-enum-value visual metadata (label, abbreviation, color, icon) and ordering for PDF rendering. | +| `chart_label` | string | Axis label when the attribute is used in charts. | +| `output_formats` | object | `{"csv": bool, "ocsf": bool}`, both default `true` — toggles inclusion per output. | + +Key rules: + +- `--compliance` key = JSON basename without `.json` (`dora_2022_2554`). +- Auto-discovered: no `__init__.py`, no formatter, no dispatcher registration. +- `table_config.group_by`, `pdf_config.group_by_field` and every + `charts[].group_by` must reference a key declared in `attributes_metadata`. +- Runtime type validation only covers `int`/`float`/`bool`; `str`/`list_str`/ + `list_dict` are documentation-only. +- Extending to a new provider = adding a key to `requirement.checks`. Nothing else. +- **No automatic check-existence validation at load time** — a typo'd check id + silently produces a requirement with no findings. Always run the + check-existence cross-check (see Validation). +- In universal files, always set `Provider` on every config constraint so a + guardrail authored for an AWS check never affects Azure/GCP scans of the + same requirement. + +## Legacy Schema Reference + +Base legacy file structure: ```json { "Framework": "FRAMEWORK_NAME", "Name": "Full Framework Name with Version", "Version": "X.X", - "Provider": "PROVIDER", + "Provider": "AWS", "Description": "Framework description...", "Requirements": [ { "Id": "requirement_id", - "Description": "Requirement description", "Name": "Optional requirement name", - "Attributes": [...], - "Checks": ["check_name_1", "check_name_2"] + "Description": "Requirement description", + "Attributes": [ ... ], + "Checks": ["check_name_1"], + "ConfigRequirements": [ ... ] } ] } ``` -## Framework-Specific Attribute Structures +### Legacy fields, top level (`Compliance`) -Each framework type has its own attribute model. Below are the exact structures used by Prowler: +| Field | Type | Required | Notes | +|---|---|---|---| +| `Framework` | string | Yes (non-empty, validated) | Canonical identifier (`CIS`, `ENS`, `NIST-800-53-Revision-5`). | +| `Name` | string | Yes (non-empty, validated) | Human-readable name with version. | +| `Version` | string | Optional in the model — **never leave it empty in practice** | Empty Version silently degrades the `get_check_compliance()` key to `"{Framework}"` (gotcha #4). Must match the version substring in the filename. | +| `Provider` | string | Yes (non-empty, validated) | Upper-cased single provider (`AWS`, `AZURE`, `GCP`, `M365`, ...). One file = one provider. | +| `Description` | string | Yes | Framework scope and purpose. | +| `Requirements` | array | Yes | Requirement objects (below), or `Mitre_Requirement` objects for MITRE files. | -### CIS (Center for Internet Security) +### Legacy fields, per requirement (`Compliance_Requirement`) -**Framework ID format:** `cis_{version}_{provider}` (e.g., `cis_5.0_aws`) +| Field | Type | Required | Notes | +|---|---|---|---| +| `Id` | string | Yes | Unique within the framework; follow the source numbering exactly (`1.1`, `A.5.1`, `CCC.Core.CN01.AR01`). | +| `Description` | string | Yes | Verbatim from the source catalog. | +| `Name` | string | No | Optional short title (NIST-style catalogs use it). | +| `Attributes` | array of objects | Yes | Parsed against the Union of attribute classes below; only `Attributes[0]` survives the universal adaptation and drives UI grouping. | +| `Checks` | array of strings | Yes | Check ids automating the requirement; `[]` = manual. | +| `ConfigRequirements` | array | No | Guardrails; `Provider` is omitted (the file is single-provider). | + +MITRE files use `Mitre_Requirement` instead, which adds `Tactics`, +`SubTechniques`, `Platforms`, `TechniqueURL` at the requirement top level. + +### Attribute shapes per framework family + +Unlike universal (schema in-file), a legacy requirement's `Attributes` must +match one of the Pydantic classes registered in +`Compliance_Requirement.Attributes` — a shape matching no class **silently +falls through to Generic**, dropping its specific fields. The most common +shapes (full field sets in `compliance_models.py`): + +### CIS — `cis_{version}_{provider}` ```json { - "Id": "1.1", - "Description": "Maintain current contact details", - "Checks": ["account_maintain_current_contact_details"], - "Attributes": [ - { - "Section": "1 Identity and Access Management", - "SubSection": "Optional subsection", - "Profile": "Level 1", - "AssessmentStatus": "Automated", - "Description": "Detailed attribute description", - "RationaleStatement": "Why this control matters", - "ImpactStatement": "Impact of implementing this control", - "RemediationProcedure": "Steps to fix the issue", - "AuditProcedure": "Steps to verify compliance", - "AdditionalInformation": "Extra notes", - "DefaultValue": "Default configuration value", - "References": "https://docs.example.com/reference" - } - ] + "Section": "1 Identity and Access Management", + "SubSection": "Optional subsection", + "Profile": "Level 1", + "AssessmentStatus": "Automated", + "Description": "...", "RationaleStatement": "...", "ImpactStatement": "...", + "RemediationProcedure": "...", "AuditProcedure": "...", + "AdditionalInformation": "...", "DefaultValue": "...", "References": "https://..." } ``` -**Profile values:** `Level 1`, `Level 2`, `E3 Level 1`, `E3 Level 2`, `E5 Level 1`, `E5 Level 2` -**AssessmentStatus values:** `Automated`, `Manual` +`Profile`: `Level 1|Level 2|E3 Level 1|E3 Level 2|E5 Level 1|E5 Level 2`. +`AssessmentStatus`: `Automated|Manual`. ---- - -### ISO 27001 - -**Framework ID format:** `iso27001_{year}_{provider}` (e.g., `iso27001_2022_aws`) +### ENS — `ens_rd2022_{provider}` ```json { - "Id": "A.5.1", - "Description": "Policies for information security should be defined...", - "Name": "Policies for information security", - "Checks": ["securityhub_enabled"], - "Attributes": [ - { - "Category": "A.5 Organizational controls", - "Objetive_ID": "A.5.1", - "Objetive_Name": "Policies for information security", - "Check_Summary": "Summary of what is being checked" - } - ] + "IdGrupoControl": "op.acc.1", "Marco": "operacional", + "Categoria": "control de acceso", "DescripcionControl": "...", + "Nivel": "alto", "Tipo": "requisito", + "Dimensiones": ["trazabilidad", "autenticidad"], + "ModoEjecucion": "automatico", "Dependencias": [] } ``` -**Note:** `Objetive_ID` and `Objetive_Name` use this exact spelling (not "Objective"). +`Nivel`: `opcional|bajo|medio|alto`. `Tipo`: `refuerzo|requisito|recomendacion|medida`. +`Dimensiones`: `confidencialidad|integridad|trazabilidad|autenticidad|disponibilidad`. ---- - -### ENS (Esquema Nacional de Seguridad - Spain) - -**Framework ID format:** `ens_rd2022_{provider}` (e.g., `ens_rd2022_aws`) +### ISO 27001 — `iso27001_{year}_{provider}` ```json { - "Id": "op.acc.1.aws.iam.2", - "Description": "Proveedor de identidad centralizado", - "Checks": ["iam_check_saml_providers_sts"], - "Attributes": [ - { - "IdGrupoControl": "op.acc.1", - "Marco": "operacional", - "Categoria": "control de acceso", - "DescripcionControl": "Detailed control description in Spanish", - "Nivel": "alto", - "Tipo": "requisito", - "Dimensiones": ["trazabilidad", "autenticidad"], - "ModoEjecucion": "automatico", - "Dependencias": [] - } - ] + "Category": "A.5 Organizational controls", + "Objetive_ID": "A.5.1", "Objetive_Name": "Policies for information security", + "Check_Summary": "Summary of what is being checked" } ``` -**Nivel values:** `opcional`, `bajo`, `medio`, `alto` -**Tipo values:** `refuerzo`, `requisito`, `recomendacion`, `medida` -**Dimensiones values:** `confidencialidad`, `integridad`, `trazabilidad`, `autenticidad`, `disponibilidad` +Note: `Objetive_ID` / `Objetive_Name` use this exact (mis)spelling. ---- - -### MITRE ATT&CK - -**Framework ID format:** `mitre_attack_{provider}` (e.g., `mitre_attack_aws`) - -MITRE uses a different requirement structure: +### MITRE ATT&CK — `mitre_attack_{provider}` (separate requirement model) ```json { - "Name": "Exploit Public-Facing Application", - "Id": "T1190", - "Tactics": ["Initial Access"], - "SubTechniques": [], - "Platforms": ["Containers", "IaaS", "Linux", "Network", "Windows", "macOS"], - "Description": "Adversaries may attempt to exploit a weakness...", + "Name": "Exploit Public-Facing Application", "Id": "T1190", + "Tactics": ["Initial Access"], "SubTechniques": [], + "Platforms": ["IaaS"], "Description": "...", "TechniqueURL": "https://attack.mitre.org/techniques/T1190/", - "Checks": ["guardduty_is_enabled", "inspector2_is_enabled"], + "Checks": ["guardduty_is_enabled"], "Attributes": [ - { - "AWSService": "Amazon GuardDuty", - "Category": "Detect", - "Value": "Minimal", - "Comment": "Explanation of how this service helps..." - } + {"AWSService": "Amazon GuardDuty", "Category": "Detect", + "Value": "Minimal", "Comment": "..."} ] } ``` -**For Azure:** Use `AzureService` instead of `AWSService` -**For GCP:** Use `GCPService` instead of `AWSService` -**Category values:** `Detect`, `Protect`, `Respond` -**Value values:** `Minimal`, `Partial`, `Significant` +`AzureService`/`GCPService` for the other providers. `Category`: +`Detect|Protect|Respond`. `Value`: `Minimal|Partial|Significant`. ---- - -### NIST 800-53 - -**Framework ID format:** `nist_800_53_revision_{version}_{provider}` (e.g., `nist_800_53_revision_5_aws`) +### CCC — `ccc_{provider}` ```json { - "Id": "ac_2_1", - "Name": "AC-2(1) Automated System Account Management", - "Description": "Support the management of system accounts...", - "Checks": ["iam_password_policy_minimum_length_14"], - "Attributes": [ - { - "ItemId": "ac_2_1", - "Section": "Access Control (AC)", - "SubSection": "Account Management (AC-2)", - "SubGroup": "AC-2(3) Disable Accounts", - "Service": "iam" - } - ] + "FamilyName": "Data", "FamilyDescription": "...", + "Section": "CCC.Core.CN01 Encrypt Data for Transmission", "SubSection": "", + "SubSectionObjective": "...", + "Applicability": ["tlp-green", "tlp-amber", "tlp-red"], + "Recommendation": "...", + "SectionThreatMappings": [{"ReferenceId": "CCC", "Identifiers": ["CCC.Core.TH02"]}], + "SectionGuidelineMappings": [{"ReferenceId": "NIST-CSF", "Identifiers": ["PR.DS-02"]}] } ``` ---- +`Applicability` holds TLP tags (`tlp-clear|tlp-green|tlp-amber|tlp-red`). -### Generic Compliance (Fallback) - -For frameworks without specific attribute models: +### ASD Essential Eight — `asd_essential_eight_aws` ```json { - "Id": "requirement_id", - "Description": "Requirement description", - "Name": "Optional name", - "Checks": ["check_name"], - "Attributes": [ - { - "ItemId": "item_id", - "Section": "Section name", - "SubSection": "Subsection name", - "SubGroup": "Subgroup name", - "Service": "service_name", - "Type": "type" - } - ] + "Section": "Patch applications", "MaturityLevel": "ML1", + "AssessmentStatus": "Automated", "CloudApplicability": "partial", + "MitigatedThreats": ["..."], "Description": "...", + "RationaleStatement": "...", "ImpactStatement": "...", + "RemediationProcedure": "...", "AuditProcedure": "...", + "AdditionalInformation": "...", "References": "..." } ``` ---- +`MaturityLevel`: `ML1|ML2|ML3`. `CloudApplicability`: `full|partial|limited|non-applicable`. -### AWS Well-Architected Framework - -**Framework ID format:** `aws_well_architected_framework_{pillar}_pillar_aws` +### DISA STIG — `okta_idaas_stig_v1r2_okta` ```json { - "Id": "SEC01-BP01", - "Description": "Establish common guardrails...", - "Name": "Establish common guardrails", - "Checks": ["account_part_of_organizations"], - "Attributes": [ - { - "Name": "Establish common guardrails", - "WellArchitectedQuestionId": "securely-operate", - "WellArchitectedPracticeId": "sec_securely_operate_multi_accounts", - "Section": "Security", - "SubSection": "Security foundations", - "LevelOfRisk": "High", - "AssessmentMethod": "Automated", - "Description": "Detailed description", - "ImplementationGuidanceUrl": "https://docs.aws.amazon.com/..." - } - ] + "Section": "...", "Severity": "high", "RuleID": "...", "StigID": "...", + "CCI": ["CCI-000015"], "CheckText": "...", "FixText": "..." } ``` ---- +`Severity`: `high|medium|low` (maps to CAT I/II/III). -### KISA ISMS-P (Korea) +### Other registered shapes -**Framework ID format:** `kisa_isms_p_{year}_{provider}` (e.g., `kisa_isms_p_2023_aws`) +- **AWS Well-Architected** (`aws_well_architected_framework_{pillar}_pillar_aws`): + `Name`, `WellArchitectedQuestionId`, `WellArchitectedPracticeId`, `Section`, + `SubSection`, `LevelOfRisk`, `AssessmentMethod`, `Description`, + `ImplementationGuidanceUrl`. +- **KISA ISMS-P** (`kisa_isms_p_2023_{provider}`): `Domain`, `Subdomain`, + `Section`, `AuditChecklist`, `RelatedRegulations`, `AuditEvidence`, + `NonComplianceCases`. +- **C5** (`c5_{provider}`): `Section`, `SubSection`, `Type`, `AboutCriteria`, + `ComplementaryCriteria`. +- **CSA CCM** (legacy shape; the shipped CSA CCM 4.0 is universal): `Section`, + `CCMLite`, `IaaS`, `PaaS`, `SaaS`, `ScopeApplicability`. +- **Prowler ThreatScore** (`prowler_threatscore_{provider}`): `Title`, + `Section`, `SubSection`, `AttributeDescription`, `AdditionalInformation`, + `LevelOfRisk` (1–5), `Weight` (1/8/10/100/1000). Pillars: 1 IAM, 2 Attack + Surface, 3 Logging and Monitoring, 4 Encryption. Available for aws, + azure, gcp, kubernetes, m365, alibabacloud. +- **Generic (fallback)**: `ItemId`, `Section`, `SubSection`, `SubGroup`, + `Service`, `Type`, `Comment` — all optional. Used by NIST, PCI, GDPR, + HIPAA, SOC2, FedRAMP, CISA, FFIEC, RBI, NIS2, GxP, SecNumCloud, etc. + +## Config Guardrails (`ConfigRequirements`) + +Requirements backed by [configurable checks](https://docs.prowler.com/developer-guide/configurable-checks) +can be silently "satisfied" by a loosened `audit_config` (e.g. CIS demands +45-day unused credentials but the scan ran with `max_unused_access_keys_days: 120`). +Guardrails force such requirements to FAIL: ```json -{ - "Id": "1.1.1", - "Description": "Requirement description", - "Name": "Requirement name", - "Checks": ["check_name"], - "Attributes": [ - { - "Domain": "1. Management System", - "Subdomain": "1.1 Management System Establishment", - "Section": "1.1.1 Section Name", - "AuditChecklist": ["Checklist item 1", "Checklist item 2"], - "RelatedRegulations": ["Regulation 1"], - "AuditEvidence": ["Evidence type 1"], - "NonComplianceCases": ["Non-compliance example"] - } - ] -} +"ConfigRequirements": [ + {"Check": "iam_user_accesskey_unused", + "ConfigKey": "max_unused_access_keys_days", "Operator": "lte", "Value": 45} +] ``` ---- - -### C5 (Germany Cloud Computing Compliance Criteria Catalogue) - -**Framework ID format:** `c5_{provider}` (e.g., `c5_aws`) - -```json -{ - "Id": "BCM-01", - "Description": "Requirement description", - "Name": "Requirement name", - "Checks": ["check_name"], - "Attributes": [ - { - "Section": "BCM Business Continuity Management", - "SubSection": "BCM-01", - "Type": "Basic Criteria", - "AboutCriteria": "Description of criteria", - "ComplementaryCriteria": "Additional criteria" - } - ] -} -``` +- Operators: `lte`/`gte` (numeric thresholds), `eq` (toggles/exact — use JSON + booleans, not 0/1), `in` (scalar in allowed set), `subset` (allowlists — + widening breaks it), `superset` (denylists — removing an entry breaks it). +- `Value` must be the **strictest** setting the control text tolerates. +- `ConfigKey` must be spelled exactly as the check reads it; unknown keys are + silently skipped (defaults assumed OK). +- Guardrails only tighten (PASS→FAIL), never relax. +- Universal files: lowercase `config_requirements` + mandatory `Provider` per + constraint. +- Tests: `tests/lib/check/compliance_config_eval_test.py`, + `compliance_config_constraint_model_test.py`, + `compliance_config_requirements_data_test.py`, plus per-output tests under + `tests/lib/outputs/compliance/`. --- -### CCC (Cloud Computing Compliance) - -**Framework ID format:** `ccc_{provider}` (e.g., `ccc_aws`) - -```json -{ - "Id": "CCC.C01", - "Description": "Requirement description", - "Name": "Requirement name", - "Checks": ["check_name"], - "Attributes": [ - { - "FamilyName": "Cryptography & Key Management", - "FamilyDescription": "Family description", - "Section": "CCC.C01", - "SubSection": "Key Management", - "SubSectionObjective": "Objective description", - "Applicability": ["IaaS", "PaaS", "SaaS"], - "Recommendation": "Recommended action", - "SectionThreatMappings": [{"threat": "T1190"}], - "SectionGuidelineMappings": [{"guideline": "NIST"}] - } - ] -} -``` - ---- - -### Prowler ThreatScore - -**Framework ID format:** `prowler_threatscore_{provider}` (e.g., `prowler_threatscore_aws`) - -Prowler ThreatScore is a custom security scoring framework developed by Prowler that evaluates AWS account security based on **four main pillars**: - -| Pillar | Description | -|--------|-------------| -| **1. IAM** | Identity and Access Management controls (authentication, authorization, credentials) | -| **2. Attack Surface** | Network exposure, public resources, security group rules | -| **3. Logging and Monitoring** | Audit logging, threat detection, forensic readiness | -| **4. Encryption** | Data at rest and in transit encryption | - -**Scoring System:** -- **LevelOfRisk** (1-5): Severity of the security issue - - `5` = Critical (e.g., root MFA, public S3 buckets) - - `4` = High (e.g., user MFA, public EC2) - - `3` = Medium (e.g., password policies, encryption) - - `2` = Low - - `1` = Informational -- **Weight**: Impact multiplier for score calculation - - `1000` = Critical controls (root security, public exposure) - - `100` = High-impact controls (user authentication, monitoring) - - `10` = Standard controls (password policies, encryption) - - `1` = Low-impact controls (best practices) - -```json -{ - "Id": "1.1.1", - "Description": "Ensure MFA is enabled for the 'root' user account", - "Checks": ["iam_root_mfa_enabled"], - "Attributes": [ - { - "Title": "MFA enabled for 'root'", - "Section": "1. IAM", - "SubSection": "1.1 Authentication", - "AttributeDescription": "The root user account holds the highest level of privileges within an AWS account. Enabling MFA enhances security by adding an additional layer of protection.", - "AdditionalInformation": "Enabling MFA enhances console security by requiring the authenticating user to both possess a time-sensitive key-generating device and have knowledge of their credentials.", - "LevelOfRisk": 5, - "Weight": 1000 - } - ] -} -``` - -**Available for providers:** AWS, Kubernetes, M365 - ---- - -## Available Compliance Frameworks - -### AWS (41 frameworks) - -| Framework | File Name | -|-----------|-----------| -| CIS 1.4, 1.5, 2.0, 3.0, 4.0, 5.0 | `cis_{version}_aws.json` | -| ISO 27001:2013, 2022 | `iso27001_{year}_aws.json` | -| NIST 800-53 Rev 4, 5 | `nist_800_53_revision_{version}_aws.json` | -| NIST 800-171 Rev 2 | `nist_800_171_revision_2_aws.json` | -| NIST CSF 1.1, 2.0 | `nist_csf_{version}_aws.json` | -| PCI DSS 3.2.1, 4.0 | `pci_{version}_aws.json` | -| HIPAA | `hipaa_aws.json` | -| GDPR | `gdpr_aws.json` | -| SOC 2 | `soc2_aws.json` | -| FedRAMP Low/Moderate | `fedramp_{level}_revision_4_aws.json` | -| ENS RD2022 | `ens_rd2022_aws.json` | -| MITRE ATT&CK | `mitre_attack_aws.json` | -| C5 Germany | `c5_aws.json` | -| CISA | `cisa_aws.json` | -| FFIEC | `ffiec_aws.json` | -| RBI Cyber Security | `rbi_cyber_security_framework_aws.json` | -| AWS Well-Architected | `aws_well_architected_framework_{pillar}_pillar_aws.json` | -| AWS FTR | `aws_foundational_technical_review_aws.json` | -| GxP 21 CFR Part 11, EU Annex 11 | `gxp_{standard}_aws.json` | -| KISA ISMS-P 2023 | `kisa_isms_p_2023_aws.json` | -| NIS2 | `nis2_aws.json` | - -### Azure (15+ frameworks) - -| Framework | File Name | -|-----------|-----------| -| CIS 2.0, 2.1, 3.0, 4.0 | `cis_{version}_azure.json` | -| ISO 27001:2022 | `iso27001_2022_azure.json` | -| ENS RD2022 | `ens_rd2022_azure.json` | -| MITRE ATT&CK | `mitre_attack_azure.json` | -| PCI DSS 4.0 | `pci_4.0_azure.json` | -| NIST CSF 2.0 | `nist_csf_2.0_azure.json` | - -### GCP (15+ frameworks) - -| Framework | File Name | -|-----------|-----------| -| CIS 2.0, 3.0, 4.0 | `cis_{version}_gcp.json` | -| ISO 27001:2022 | `iso27001_2022_gcp.json` | -| HIPAA | `hipaa_gcp.json` | -| MITRE ATT&CK | `mitre_attack_gcp.json` | -| PCI DSS 4.0 | `pci_4.0_gcp.json` | -| NIST CSF 2.0 | `nist_csf_2.0_gcp.json` | - -### Kubernetes (6 frameworks) - -| Framework | File Name | -|-----------|-----------| -| CIS 1.8, 1.10, 1.11 | `cis_{version}_kubernetes.json` | -| ISO 27001:2022 | `iso27001_2022_kubernetes.json` | -| PCI DSS 4.0 | `pci_4.0_kubernetes.json` | - -### Other Providers -- **GitHub:** `cis_1.0_github.json` -- **M365:** `cis_4.0_m365.json`, `iso27001_2022_m365.json` -- **NHN:** `iso27001_2022_nhn.json` - ## Workflow A: Sync a Framework With an Upstream Catalog -Use when the framework is maintained upstream (CIS Benchmarks, FINOS CCC, CSA CCM, NIST, ENS, etc.) and Prowler needs to catch up. +Use when the framework is maintained upstream (CIS Benchmarks, FINOS CCC, CSA +CCM, NIST, ENS, etc.) and Prowler needs to catch up. ### Step 1 — Cache the upstream source -Download every upstream file to a local cache so subsequent iterations don't hit the network. For FINOS CCC: +Download every upstream file to a local cache so iterations don't hit the +network. For FINOS CCC: ```bash mkdir -p /tmp/ccc_upstream @@ -563,492 +715,480 @@ done ### Step 2 — Run the generic sync runner against a framework config -The sync tooling is split into three layers so adding a new framework only takes a YAML config (and optionally a new parser module for an unfamiliar upstream format): +The sync tooling is three layers, so adding a framework only takes a YAML +config (plus a parser module for an unfamiliar upstream format): ```text skills/prowler-compliance/assets/ ├── sync_framework.py # generic runner — works for any framework -├── configs/ -│ └── ccc.yaml # per-framework config (canonical example) -└── parsers/ - ├── __init__.py - └── finos_ccc.py # parser module for FINOS CCC YAML +├── configs/ccc.yaml # per-framework config (canonical example) +└── parsers/finos_ccc.py # parser module for FINOS CCC YAML ``` -**For frameworks that already have a config + parser** (today: FINOS CCC), run: - ```bash python skills/prowler-compliance/assets/sync_framework.py \ skills/prowler-compliance/assets/configs/ccc.yaml ``` -The runner loads the config, validates it, dynamically imports the parser declared in `parser.module`, calls `parser.parse_upstream(config) -> list[dict]`, then applies generic post-processing (id uniqueness safety net, `FamilyName` normalization, legacy check-mapping preservation) and writes the provider JSONs. +The runner loads the config, dynamically imports `parser.module`, calls +`parse_upstream(config) -> list[dict]`, then applies generic post-processing +(id-uniqueness safety net, `FamilyName` normalization, legacy check-mapping +preservation with config-driven fallback keys) and writes the provider JSONs +with Pydantic post-validation. **To add a new framework sync**: -1. **Write a config file** at `skills/prowler-compliance/assets/configs/{framework}.yaml`. See `configs/ccc.yaml` as the canonical example. Required top-level sections: - - `framework` — `name`, `display_name`, `version` (**never empty** — empty Version silently breaks `get_check_compliance()` key construction, so the runner refuses to start), `description_template` (accepts `{provider_display}`, `{provider_key}`, `{framework_name}`, `{framework_display}`, `{version}` placeholders). - - `providers` — list of `{key, display}` pairs, one per Prowler provider the framework targets. - - `output.path_template` — supports `{provider}`, `{framework}`, `{version}` placeholders. Examples: `"prowler/compliance/{provider}/ccc_{provider}.json"` for unversioned file names, `"prowler/compliance/{provider}/cis_{version}_{provider}.json"` for versioned ones. - - `upstream.dir` — local cache directory (populate via Step 1). - - `parser.module` — name of the module under `parsers/` to load (without `.py`). Everything else under `parser.` is opaque to the runner and passed to the parser as config. - - `post_processing.check_preservation.primary_key` — top-level field name for the primary legacy-mapping lookup (almost always `Id`). - - `post_processing.check_preservation.fallback_keys` — **config-driven fallback keys** for preserving check mappings when ids change. Each entry is a list of `Attributes[0]` field names composed into a tuple. Examples: - - CCC: `- [Section, Applicability]` (because `Applicability` is a CCC-only attribute, verified in `compliance_models.py:213`). - - CIS would use `- [Section, Profile]`. - - NIST would use `- [ItemId]`. - - List-valued fields (like `Applicability`) are automatically frozen to `frozenset` so the tuple is hashable. - - `post_processing.family_name_normalization` (optional) — map of raw → canonical `FamilyName` values. The UI groups by `Attributes[0].FamilyName` exactly, so inconsistent upstream variants otherwise become separate tree branches. +1. Write `assets/configs/{framework}.yaml` (see `ccc.yaml`). Required sections: + - `framework` — `name`, `display_name`, `version` (**never empty** — the + runner refuses to start, because empty Version breaks the + `get_check_compliance()` key), `description_template`. + - `providers` — list of `{key, display}` pairs. + - `output.path_template` — e.g. + `"prowler/compliance/{provider}/cis_{version}_{provider}.json"`. + - `upstream.dir` — local cache (Step 1). + - `parser.module` — module under `parsers/`; the rest of `parser.` is + passed through opaque. + - `post_processing.check_preservation.primary_key` (almost always `Id`) and + `fallback_keys` — lists of `Attributes[0]` field names composed into + tuples for recovering mappings when ids change. CCC: + `- [Section, Applicability]`; CIS: `- [Section, Profile]`; NIST: + `- [ItemId]`. List-valued fields are frozen to `frozenset` automatically. + - `post_processing.family_name_normalization` (optional) — raw → canonical + map; the UI groups by the exact attribute value, so upstream variants + otherwise become separate tree branches. +2. Reuse an existing parser or write `parsers/{name}.py` implementing + `parse_upstream(config) -> list[dict]` returning Prowler-format + requirements with **guaranteed-unique ids**. The runner raises on + duplicates — it never silently renumbers, because mutating a canonical + upstream id (CIS `1.1.1`, NIST `AC-2(1)`) would be catastrophic. The parser + owns all upstream quirks: foreign-prefix rewriting, genuine collision + renumbering, multi-shape handling. -2. **Reuse an existing parser** if the upstream format matches one (currently only `finos_ccc` exists). Otherwise, **write a new parser** at `parsers/{name}.py` implementing: +**Gotchas the runner already handles** (from the FINOS CCC v2025.10 sync): - ```python - def parse_upstream(config: dict) -> list[dict]: - """Return Prowler-format requirements {Id, Description, Attributes: [...], Checks: []}. - - Ids MUST be unique in the returned list. The runner raises ValueError - on duplicates — it does NOT silently renumber, because mutating a - canonical upstream id (e.g. CIS '1.1.1' or NIST 'AC-2(1)') would be - catastrophic. The parser owns all upstream-format quirks: foreign-prefix - rewriting, genuine collision renumbering, shape handling. - """ - ``` - - The parser reads its own settings from `config['upstream']` and `config['parser']`. It does NOT load existing Prowler JSONs (the runner does that for check preservation) and does NOT write output (the runner does that too). - -**Gotchas the runner already handles for you** (learned from the FINOS CCC v2025.10 sync — they're documented here so you don't re-discover them): - -- **Multiple upstream YAML shapes**. Most FINOS CCC catalogs use `control-families: [...]`, but `storage/object` uses a top-level `controls: [...]` with a `family: "CCC.X.Y"` reference id and no human-readable family name. A parser that only handles shape 1 silently drops the shape-2 catalog — this exact bug dropped ObjStor from Prowler for a full iteration. `parsers/finos_ccc.py` handles both shapes; if you write a new parser for a similar format, test with at least one file of each shape. -- **Whitespace collapse**. Upstream YAML multi-line block scalars (`|`) preserve newlines. Prowler stores descriptions single-line. Collapse with `" ".join(value.split())` before emitting (see `parsers/finos_ccc.py::clean()`). -- **Foreign-prefix AR id rewriting**. Upstream sometimes aliases requirements across catalogs by keeping the original prefix (e.g., `CCC.AuditLog.CN08.AR01` appears nested under `CCC.Logging.CN03`). Rewrite the foreign id to fit its parent control: `CCC.Logging.CN03.AR01`. This logic is parser-specific because the id structure varies per framework (CCC uses 3-dot depth; CIS uses numeric dots; NIST uses `AC-2(1)`). -- **Genuine upstream collision renumbering**. Sometimes upstream has a real typo where two different requirements share the same id (e.g., `CCC.Core.CN14.AR02` defined twice for 30-day and 14-day backup variants). Renumber the second copy to the next free AR number (`.AR03`). The parser handles this; the runner asserts the final list has unique ids as a safety net. -- **Existing check mapping preservation**. The runner uses the `primary_key` + `fallback_keys` declared in config to look up the old `Checks` list for each requirement. For CCC this means primary index by `Id` plus fallback index by `(Section, frozenset(Applicability))` — the fallback recovers mappings for requirements whose ids were rewritten or renumbered by the parser. -- **FamilyName normalization**. Configured via `post_processing.family_name_normalization` — no code changes needed to collapse upstream variants like `"Logging & Monitoring"` → `"Logging and Monitoring"`. -- **Populate `Version`**. The runner refuses to start on empty `framework.version` — fail-fast replaces the silent bug where `get_check_compliance()` would build the key as just `"{Framework}"`. +- **Multiple upstream YAML shapes.** Most FINOS CCC catalogs use + `control-families: [...]` but `storage/object` uses top-level + `controls: [...]`. A single-shape parser silently drops entire catalogs — + this exact bug dropped ObjStor for a full iteration. Test with one file of + each shape. +- **Whitespace collapse.** Upstream `|` block scalars keep newlines; Prowler + stores single-line. Collapse with `" ".join(value.split())`. +- **Foreign-prefix id rewriting.** Upstream aliases requirements across + catalogs keeping the original prefix (`CCC.AuditLog.CN08.AR01` nested under + `CCC.Logging.CN03`) — rewrite to fit the parent (`CCC.Logging.CN03.AR01`). +- **Genuine upstream collisions.** Two different requirements sharing one id + (upstream typo): renumber the second to the next free number; check-mapping + preservation recovers by the fallback keys. +- **Populate `Version`** — fail-fast beats the silent broken-key bug. ### Step 3 — Validate before committing -```python -from prowler.lib.check.compliance_models import Compliance -for prov in ['aws', 'azure', 'gcp']: - c = Compliance.parse_file(f"prowler/compliance/{prov}/ccc_{prov}.json") - print(f"{prov}: {len(c.Requirements)} reqs, version={c.Version}") -``` +Run the full Validation section below (universal loader + check existence + +CLI smoke + pytest). -Any `ValidationError` means the Attribute fields don't match the `*_Requirement_Attribute` model. Either fix the JSON or extend the model in `compliance_models.py` (remember: Generic stays last). +### Step 4 — Add an attribute model if needed -### Step 4 — Verify every check id exists - -```python -import json -from pathlib import Path -for prov in ['aws', 'azure', 'gcp']: - existing = {p.stem.replace('.metadata','') - for p in Path(f'prowler/providers/{prov}/services').rglob('*.metadata.json')} - with open(f'prowler/compliance/{prov}/ccc_{prov}.json') as f: - data = json.load(f) - refs = {c for r in data['Requirements'] for c in r['Checks']} - missing = refs - existing - assert not missing, f"{prov} missing: {missing}" -``` - -A stale check id silently becomes dead weight — no finding will ever map to it. This pre-validation **must run on every write**; bake it into the generator script. - -### Step 5 — Add an attribute model if needed - -Only if the framework has fields beyond `Generic_Compliance_Requirement_Attribute`. Add the class to `prowler/lib/check/compliance_models.py` and register it in `Compliance_Requirement.Attributes: list[Union[...]]`. **Generic stays last.** +Only if the framework has fields beyond +`Generic_Compliance_Requirement_Attribute` and must stay legacy. Add the class +to `compliance_models.py` and register it in the +`Compliance_Requirement.Attributes` Union **before Generic** (Generic stays +last). For new frameworks, prefer universal `attributes_metadata` instead. --- ## Workflow B: Audit Check Mappings as a Cloud Auditor -Use when the user asks to review existing mappings ("are these correct?", "verify that the checks apply", "audit the CCC mappings"). This is the highest-value compliance task — it surfaces padded mappings with zero actual coverage and missing mappings for legitimate coverage. +Use when the user asks to review existing mappings. This is the +highest-value compliance task — it surfaces padded mappings with zero actual +coverage and missing mappings for legitimate coverage. ### The golden rule -> A Prowler check's title/risk MUST **literally describe what the requirement text says**. "Related" is not enough. If no check actually addresses the requirement, leave `Checks: []` (MANUAL) — **honest MANUAL is worth more than padded coverage**. +> A Prowler check's title/risk MUST **literally describe what the requirement +> text says**. "Related" is not enough. If no check actually addresses the +> requirement, leave the checks list empty (MANUAL) — **honest MANUAL is worth +> more than padded coverage**. ### Audit process -**Step 1 — Build a per-provider check inventory** (cache in `/tmp/`): +1. **Build a per-provider check inventory** — `assets/build_inventory.py` + (writes `/tmp/checks_{provider}.json` for every provider discovered under + `prowler/providers/`). +2. **Query it** — `assets/query_checks.py` (run from the repository root): -```python -import json -from pathlib import Path -for provider in ['aws', 'azure', 'gcp']: - inv = {} - for meta in Path(f'prowler/providers/{provider}/services').rglob('*.metadata.json'): - with open(meta) as f: - d = json.load(f) - cid = d.get('CheckID') or meta.stem.replace('.metadata','') - inv[cid] = { - 'service': d.get('ServiceName', ''), - 'title': d.get('CheckTitle', ''), - 'risk': d.get('Risk', ''), - 'description': d.get('Description', ''), - } - with open(f'/tmp/checks_{provider}.json', 'w') as f: - json.dump(inv, f, indent=2) -``` + ```bash + python skills/prowler-compliance/assets/query_checks.py aws encryption transit # keyword AND-search + python skills/prowler-compliance/assets/query_checks.py aws --service iam # all iam checks + python skills/prowler-compliance/assets/query_checks.py aws --id kms_cmk_rotation_enabled + ``` -**Step 2 — Keyword/service query helper** — see [assets/query_checks.py](assets/query_checks.py): +3. **Dump a framework section with current mappings** — `assets/dump_section.py`: -```bash -python assets/query_checks.py aws encryption transit # keyword AND-search -python assets/query_checks.py aws --service iam # all iam checks -python assets/query_checks.py aws --id kms_cmk_rotation_enabled # full metadata -``` + ```bash + python skills/prowler-compliance/assets/dump_section.py ccc "CCC.Core." + python skills/prowler-compliance/assets/dump_section.py cis_5.0_aws "1." + ``` -**Step 3 — Dump a framework section with current mappings** — see [assets/dump_section.py](assets/dump_section.py): +4. **Encode explicit REPLACE decisions** — `assets/audit_framework_template.py`: -```bash -python assets/dump_section.py ccc "CCC.Core." # all Core ARs across 3 providers -python assets/dump_section.py ccc "CCC.AuditLog." # all AuditLog ARs -``` + ```python + DECISIONS = {} + DECISIONS["CCC.Core.CN01.AR01"] = { + "aws": ["cloudfront_distributions_https_enabled", ...], + "azure": ["storage_secure_transfer_required_is_enabled", ...], + "gcp": ["cloudsql_instance_ssl_connections"], + # Missing provider key = leave the legacy mapping untouched + } + # Empty list = EXPLICITLY MANUAL (overwrites legacy) + DECISIONS["CCC.Core.CN01.AR07"] = {"aws": [], "azure": [], "gcp": []} + ``` -**Step 4 — Encode explicit REPLACE decisions** — see [assets/audit_framework_template.py](assets/audit_framework_template.py). Structure: + **REPLACE, not PATCH.** Full lists make the audit reproducible and surface + hidden assumptions in the legacy data. +5. **Pre-validate** every check id against the inventory; the script MUST + abort with stderr listing typos (real audits caught + `storage_secure_transfer_required_enabled` → + `storage_secure_transfer_required_is_enabled`, + `sqlserver_minimum_tls_version_12` → + `sqlserver_recommended_minimal_tls_version`, and several checks that + simply don't exist). +6. **Apply + validate + test**: -```python -DECISIONS = {} + ```bash + python /path/to/audit_script.py + uv run pytest -n auto tests/lib/outputs/compliance/ tests/lib/check/ -q + ``` -DECISIONS["CCC.Core.CN01.AR01"] = { - "aws": [ - "cloudfront_distributions_https_enabled", - "cloudfront_distributions_origin_traffic_encrypted", - # ... - ], - "azure": [ - "storage_secure_transfer_required_is_enabled", - "app_minimum_tls_version_12", - # ... - ], - "gcp": [ - "cloudsql_instance_ssl_connections", - ], - # Missing provider key = leave the legacy mapping untouched -} - -# Empty list = EXPLICITLY MANUAL (overwrites legacy) -DECISIONS["CCC.Core.CN01.AR07"] = { - "aws": [], # Prowler has no IANA port/protocol check - "azure": [], - "gcp": [], -} -``` - -**REPLACE, not PATCH.** Encoding every mapping as a full list (not add/remove delta) makes the audit reproducible and surfaces hidden assumptions from the legacy data. - -**Step 5 — Pre-validation**. The audit script MUST validate every check id against the inventory and **abort with stderr listing typos**. Common typos caught during a real audit: - -- `fsx_file_system_encryption_at_rest_using_kms` (doesn't exist) -- `cosmosdb_account_encryption_at_rest_with_cmk` (doesn't exist) -- `sqlserver_geo_replication` (doesn't exist) -- `redshift_cluster_audit_logging` (should be `redshift_cluster_encrypted_at_rest`) -- `postgresql_flexible_server_require_secure_transport` (should be `postgresql_flexible_server_enforce_ssl_enabled`) -- `storage_secure_transfer_required_enabled` (should be `storage_secure_transfer_required_is_enabled`) -- `sqlserver_minimum_tls_version_12` (should be `sqlserver_recommended_minimal_tls_version`) - -**Step 6 — Apply + validate + test**: - -```bash -python /path/to/audit_script.py # applies decisions, pre-validates -python -m pytest tests/lib/outputs/compliance/ tests/lib/check/ -q -``` - -### Audit Reference Table: Requirement Text → Prowler Checks - -Use this table to map CCC-style / NIST-style / ISO-style requirements to the checks that actually verify them. Built from a real audit of 172 CCC ARs × 3 providers. - -| Requirement text | AWS checks | Azure checks | GCP checks | -|---|---|---|---| -| **TLS in transit enforced** | `cloudfront_distributions_https_enabled`, `s3_bucket_secure_transport_policy`, `elbv2_ssl_listeners`, `elbv2_insecure_ssl_ciphers`, `elb_ssl_listeners`, `elb_insecure_ssl_ciphers`, `opensearch_service_domains_https_communications_enforced`, `rds_instance_transport_encrypted`, `redshift_cluster_in_transit_encryption_enabled`, `elasticache_redis_cluster_in_transit_encryption_enabled`, `dynamodb_accelerator_cluster_in_transit_encryption_enabled`, `dms_endpoint_ssl_enabled`, `kafka_cluster_in_transit_encryption_enabled`, `transfer_server_in_transit_encryption_enabled`, `glue_database_connections_ssl_enabled`, `sns_subscription_not_using_http_endpoints` | `storage_secure_transfer_required_is_enabled`, `storage_ensure_minimum_tls_version_12`, `postgresql_flexible_server_enforce_ssl_enabled`, `mysql_flexible_server_ssl_connection_enabled`, `mysql_flexible_server_minimum_tls_version_12`, `sqlserver_recommended_minimal_tls_version`, `app_minimum_tls_version_12`, `app_ensure_http_is_redirected_to_https`, `app_ftp_deployment_disabled` | `cloudsql_instance_ssl_connections` (almost only option) | -| **TLS 1.3 specifically** | Partial: `cloudfront_distributions_using_deprecated_ssl_protocols`, `elb*_insecure_ssl_ciphers`, `*_minimum_tls_version_12` | Partial: `*_minimum_tls_version_12` checks | None — accept as MANUAL | -| **SSH / port 22 hardening** | `ec2_instance_port_ssh_exposed_to_internet`, `ec2_securitygroup_allow_ingress_from_internet_to_tcp_port_22`, `ec2_networkacl_allow_ingress_tcp_port_22` | `network_ssh_internet_access_restricted`, `vm_linux_enforce_ssh_authentication` | `compute_firewall_ssh_access_from_the_internet_allowed`, `compute_instance_block_project_wide_ssh_keys_disabled`, `compute_project_os_login_enabled`, `compute_project_os_login_2fa_enabled` | -| **mTLS (mutual TLS)** | `kafka_cluster_mutual_tls_authentication_enabled`, `apigateway_restapi_client_certificate_enabled` | `app_client_certificates_on` | None — MANUAL | -| **Data at rest encrypted** | `s3_bucket_default_encryption`, `s3_bucket_kms_encryption`, `ec2_ebs_default_encryption`, `ec2_ebs_volume_encryption`, `rds_instance_storage_encrypted`, `rds_cluster_storage_encrypted`, `rds_snapshots_encrypted`, `dynamodb_tables_kms_cmk_encryption_enabled`, `redshift_cluster_encrypted_at_rest`, `neptune_cluster_storage_encrypted`, `documentdb_cluster_storage_encrypted`, `opensearch_service_domains_encryption_at_rest_enabled`, `kinesis_stream_encrypted_at_rest`, `firehose_stream_encrypted_at_rest`, `sns_topics_kms_encryption_at_rest_enabled`, `sqs_queues_server_side_encryption_enabled`, `efs_encryption_at_rest_enabled`, `athena_workgroup_encryption`, `glue_data_catalogs_metadata_encryption_enabled`, `backup_vaults_encrypted`, `backup_recovery_point_encrypted`, `cloudtrail_kms_encryption_enabled`, `cloudwatch_log_group_kms_encryption_enabled`, `eks_cluster_kms_cmk_encryption_in_secrets_enabled`, `sagemaker_notebook_instance_encryption_enabled`, `apigateway_restapi_cache_encrypted`, `kafka_cluster_encryption_at_rest_uses_cmk`, `dynamodb_accelerator_cluster_encryption_enabled`, `storagegateway_fileshare_encryption_enabled` | `storage_infrastructure_encryption_is_enabled`, `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encryption_enabled`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled`, `monitor_storage_account_with_activity_logs_cmk_encrypted` | `compute_instance_encryption_with_csek_enabled`, `dataproc_encrypted_with_cmks_disabled`, `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption` | -| **CMEK required (customer-managed keys)** | `kms_cmk_are_used` | `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled` | `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption`, `dataproc_encrypted_with_cmks_disabled`, `compute_instance_encryption_with_csek_enabled` | -| **Key rotation enabled** | `kms_cmk_rotation_enabled` | `keyvault_key_rotation_enabled`, `storage_key_rotation_90_days` | `kms_key_rotation_enabled` | -| **MFA for UI access** | `iam_root_mfa_enabled`, `iam_root_hardware_mfa_enabled`, `iam_user_mfa_enabled_console_access`, `iam_user_hardware_mfa_enabled`, `iam_administrator_access_with_mfa`, `cognito_user_pool_mfa_enabled` | `entra_privileged_user_has_mfa`, `entra_non_privileged_user_has_mfa`, `entra_user_with_vm_access_has_mfa`, `entra_security_defaults_enabled` | `compute_project_os_login_2fa_enabled` | -| **API access / credentials** | `iam_no_root_access_key`, `iam_user_no_setup_initial_access_key`, `apigateway_restapi_authorizers_enabled`, `apigateway_restapi_public_with_authorizer`, `apigatewayv2_api_authorizers_enabled` | `entra_conditional_access_policy_require_mfa_for_management_api`, `app_function_access_keys_configured`, `app_function_identity_is_configured` | `apikeys_api_restrictions_configured`, `apikeys_key_exists`, `apikeys_key_rotated_in_90_days` | -| **Log all admin/config changes** | `cloudtrail_multi_region_enabled`, `cloudtrail_multi_region_enabled_logging_management_events`, `cloudtrail_cloudwatch_logging_enabled`, `cloudtrail_log_file_validation_enabled`, `cloudwatch_log_metric_filter_*`, `cloudwatch_changes_to_*_alarm_configured`, `config_recorder_all_regions_enabled` | `monitor_diagnostic_settings_exists`, `monitor_diagnostic_setting_with_appropriate_categories`, `monitor_alert_*` | `iam_audit_logs_enabled`, `logging_log_metric_filter_and_alert_for_*`, `logging_sink_created` | -| **Log integrity (digital signatures)** | `cloudtrail_log_file_validation_enabled` (exact) | None | None | -| **Public access denied** | `s3_bucket_public_access`, `s3_bucket_public_list_acl`, `s3_bucket_public_write_acl`, `s3_account_level_public_access_blocks`, `apigateway_restapi_public`, `awslambda_function_url_public`, `awslambda_function_not_publicly_accessible`, `rds_instance_no_public_access`, `rds_snapshots_public_access`, `ec2_securitygroup_allow_ingress_from_internet_to_all_ports`, `sns_topics_not_publicly_accessible`, `sqs_queues_not_publicly_accessible` | `storage_blob_public_access_level_is_disabled`, `storage_ensure_private_endpoints_in_storage_accounts`, `containerregistry_not_publicly_accessible`, `keyvault_private_endpoints`, `app_function_not_publicly_accessible`, `aks_clusters_public_access_disabled`, `network_http_internet_access_restricted` | `cloudstorage_bucket_public_access`, `compute_instance_public_ip`, `cloudsql_instance_public_ip`, `compute_firewall_*_access_from_the_internet_allowed` | -| **IAM least privilege** | `iam_*_no_administrative_privileges`, `iam_policy_allows_privilege_escalation`, `iam_inline_policy_allows_privilege_escalation`, `iam_role_administratoraccess_policy`, `iam_group_administrator_access_policy`, `iam_user_administrator_access_policy`, `iam_policy_attached_only_to_group_or_roles`, `iam_role_cross_service_confused_deputy_prevention` | `iam_role_user_access_admin_restricted`, `iam_subscription_roles_owner_custom_not_created`, `iam_custom_role_has_permissions_to_administer_resource_locks` | `iam_sa_no_administrative_privileges`, `iam_no_service_roles_at_project_level`, `iam_role_kms_enforce_separation_of_duties`, `iam_role_sa_enforce_separation_of_duties` | -| **Password policy** | `iam_password_policy_minimum_length_14`, `iam_password_policy_uppercase`, `iam_password_policy_lowercase`, `iam_password_policy_symbol`, `iam_password_policy_number`, `iam_password_policy_expires_passwords_within_90_days_or_less`, `iam_password_policy_reuse_24` | None | None | -| **Credential rotation / unused** | `iam_rotate_access_key_90_days`, `iam_user_accesskey_unused`, `iam_user_console_access_unused` | None | `iam_sa_user_managed_key_rotate_90_days`, `iam_sa_user_managed_key_unused`, `iam_service_account_unused` | -| **VPC / flow logs** | `vpc_flow_logs_enabled` | `network_flow_log_captured_sent`, `network_watcher_enabled`, `network_flow_log_more_than_90_days` | `compute_subnet_flow_logs_enabled` | -| **Backup / DR / Multi-AZ** | `backup_vaults_exist`, `backup_plans_exist`, `backup_reportplans_exist`, `rds_instance_backup_enabled`, `rds_*_protected_by_backup_plan`, `rds_cluster_multi_az`, `neptune_cluster_backup_enabled`, `documentdb_cluster_backup_enabled`, `efs_have_backup_enabled`, `s3_bucket_cross_region_replication`, `dynamodb_table_protected_by_backup_plan` | `vm_backup_enabled`, `vm_sufficient_daily_backup_retention_period`, `storage_geo_redundant_enabled` | `cloudsql_instance_automated_backups`, `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_sufficient_retention_period` | -| **Access analysis / discovery** | `accessanalyzer_enabled`, `accessanalyzer_enabled_without_findings` | None specific | `iam_account_access_approval_enabled`, `iam_cloud_asset_inventory_enabled` | -| **Object lock / retention** | `s3_bucket_object_lock`, `s3_bucket_object_versioning`, `s3_bucket_lifecycle_enabled`, `cloudtrail_bucket_requires_mfa_delete`, `s3_bucket_no_mfa_delete` | `storage_ensure_soft_delete_is_enabled`, `storage_blob_versioning_is_enabled`, `storage_ensure_file_shares_soft_delete_is_enabled` | `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_soft_delete_enabled`, `cloudstorage_bucket_versioning_enabled`, `cloudstorage_bucket_sufficient_retention_period` | -| **Uniform bucket-level access** | `s3_bucket_acl_prohibited` | `storage_account_key_access_disabled`, `storage_default_to_entra_authorization_enabled` | `cloudstorage_bucket_uniform_bucket_level_access` | -| **Container vulnerability scanning** | `ecr_registry_scan_images_on_push_enabled`, `ecr_repositories_scan_vulnerabilities_in_latest_image` | `defender_container_images_scan_enabled`, `defender_container_images_resolved_vulnerabilities` | `artifacts_container_analysis_enabled`, `gcr_container_scanning_enabled` | -| **WAF / rate limiting** | `wafv2_webacl_with_rules`, `waf_*_webacl_with_rules`, `wafv2_webacl_logging_enabled`, `waf_global_webacl_logging_enabled` | None | None | -| **Deployment region restriction** | `organizations_scp_check_deny_regions` | None | None | -| **Secrets automatic rotation** | `secretsmanager_automatic_rotation_enabled`, `secretsmanager_secret_rotated_periodically` | `keyvault_rbac_secret_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | -| **Certificate management** | `acm_certificates_expiration_check`, `acm_certificates_with_secure_key_algorithms`, `acm_certificates_transparency_logs_enabled` | `keyvault_key_expiration_set_in_non_rbac`, `keyvault_rbac_key_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | -| **GenAI guardrails / input/output filtering** | `bedrock_guardrail_prompt_attack_filter_enabled`, `bedrock_guardrail_sensitive_information_filter_enabled`, `bedrock_agent_guardrail_enabled`, `bedrock_model_invocation_logging_enabled`, `bedrock_api_key_no_administrative_privileges`, `bedrock_api_key_no_long_term_credentials` | None | None | -| **ML dev environment security** | `sagemaker_notebook_instance_root_access_disabled`, `sagemaker_notebook_instance_without_direct_internet_access_configured`, `sagemaker_notebook_instance_vpc_settings_configured`, `sagemaker_models_vpc_settings_configured`, `sagemaker_training_jobs_vpc_settings_configured`, `sagemaker_training_jobs_network_isolation_enabled`, `sagemaker_training_jobs_volume_and_output_encryption_enabled` | None | None | -| **Threat detection / anomalous behavior** | `cloudtrail_threat_detection_enumeration`, `cloudtrail_threat_detection_privilege_escalation`, `cloudtrail_threat_detection_llm_jacking`, `guardduty_is_enabled`, `guardduty_no_high_severity_findings` | None | None | -| **Serverless private access** | `awslambda_function_inside_vpc`, `awslambda_function_not_publicly_accessible`, `awslambda_function_url_public` | `app_function_not_publicly_accessible` | None | - -### What Prowler Does NOT Cover (accept MANUAL honestly) - -Don't pad mappings for these — mark `Checks: []` and move on: - -- **TLS 1.3 version specifically** — Prowler verifies TLS is enforced, not always the exact version -- **IANA port-protocol consistency** — no check for "protocol running on its assigned port" -- **mTLS on most Azure/GCP services** — limited to App Service client certs on Azure, nothing on GCP -- **Rate limiting** on monitoring endpoints, load balancers, serverless invocations, vector ingestion -- **Session cookie expiry** (LB stickiness) -- **HTTP header scrubbing** (Server, X-Powered-By) -- **Certificate transparency verification for imports** -- **Model version pinning, red teaming, AI quality review** -- **Vector embedding validation, dimensional constraints, ANN vs exact search** -- **Secret region replication** (cross-region residency) -- **Lifecycle cleanup policies on container registries** -- **Row-level / column-level security in data warehouses** -- **Deployment region restriction on Azure/GCP** (AWS has `organizations_scp_check_deny_regions`, others don't) -- **Cross-tenant alert silencing permissions** -- **Field-level masking in logs** -- **Managed view enforcement for database access** -- **Automatic MFA delete on all S3 buckets** (only CloudTrail bucket variant exists for some frameworks — AWS has the generic `s3_bucket_no_mfa_delete` though) +For the curated mapping table (requirement text → AWS/Azure/GCP checks) and +the list of controls Prowler genuinely cannot verify, see +[references/check-mapping-reference.md](references/check-mapping-reference.md). --- -## Workflow C: Add a New Output Formatter +## Workflow C: Add a New Universal Framework -Use when a new framework needs its own CSV columns or terminal table. Follow the c5/csa/ens layout exactly: +1. Author `prowler/compliance/{framework}_{version}.json` following the + Universal Schema Reference above (use `dora_2022_2554.json` or + `csa_ccm_4.0.json` as template). +2. Declare every attribute in `attributes_metadata` (with `required`/`enum` + where possible — that's your load-time validation) and a + `outputs.table_config.group_by`. +3. Map checks per provider; add `config_requirements` (with `Provider`) for + configurable checks; leave empty lists for manual requirements — **include + every requirement of the source catalog** (coverage percentages depend on + the full denominator). +4. Validate (section below). No Python registration of any kind is needed for + CLI table/CSV/OCSF. +5. Optional first-class UI: mapper in `ui/lib/compliance/{framework}.tsx`, + registration in `getComplianceMappers()` under the JSON's `framework` value, + detail panel, `*AttributesMetadata` type, and icon (ordered keyword!). Until + then the generic mapper renders it. +6. Optional API extras: CSV exporter entry in `COMPLIANCE_CLASS_MAP`; PDF + generator + `FRAMEWORK_REGISTRY` entry if a PDF is required. +7. Tests: extend `tests/lib/check/universal_compliance_models_test.py` with a + case loading the new JSON. The parametrized `test_loads_as_universal` + already picks the file up automatically. +8. Changelog fragment `prowler/changelog.d/.added.md` + user-guide + tutorial under `docs/user-guide/compliance/tutorials/` for high-profile + frameworks. -```bash -mkdir -p prowler/lib/outputs/compliance/{framework} -touch prowler/lib/outputs/compliance/{framework}/__init__.py -``` +## Workflow D: Add a New Legacy Output Formatter -### Step 1 — Create `{framework}.py` (table dispatcher ONLY) +Only for new members of an existing legacy family. Follow the `c5/` or `ccc/` +layout exactly: -Copy from `prowler/lib/outputs/compliance/c5/c5.py` and change the function name + framework string. The `diff` between your file and `c5.py` should be just those two lines. **No function docstring** — other frameworks don't have one, stay consistent. +1. `mkdir prowler/lib/outputs/compliance/{framework}` with `__init__.py`. +2. `{framework}.py` — copy `c5/c5.py`, change function name + framework + string; the diff should be just those lines. No docstring (legacy style). +3. `models.py` — one Pydantic CSV row model per provider. Column sets differ + per provider (`AccountId`/`Region` vs `SubscriptionId`/`Location` vs + `ProjectId`/`Location`); per-provider files are the convention — don't + collapse them into a parameterized class, reviewers will reject it. +4. `{framework}_{provider}.py` — `{Framework}_{Provider}(ComplianceOutput)` + with `transform()`; this file may import `Finding`. +5. Register: + - `compliance.py` → `display_compliance_table()` `elif` branch (+ top import). + - `prowler/__main__.py` → per-provider `elif compliance_name.startswith(...)` + branches instantiating the writer classes. + - `api/src/backend/tasks/jobs/export.py` → `COMPLIANCE_CLASS_MAP` entries + (`startswith` for families, exact match only for true singletons). +6. Tests under `tests/lib/outputs/compliance/{framework}/` + fixtures in + `tests/lib/outputs/compliance/fixtures.py` (1 evaluated + 1 manual + requirement to exercise both `transform()` paths). -### Step 2 — Create `models.py` +**Circular import warning**: the table file must not import `Finding` directly +or transitively (cycle: `compliance.compliance` → table → `ComplianceOutput` → +`Finding` → `get_check_compliance` → `compliance.compliance`). Keep it bare; +use `TYPE_CHECKING`/function-local imports where both are genuinely needed. -One Pydantic v2 `BaseModel` per provider. Field names become CSV column headers (public API — don't rename later without a migration). +--- -```python -from typing import Optional -from pydantic import BaseModel +## Validation (run before every commit) -class {Framework}_AWSModel(BaseModel): - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Id: str - Requirements_Description: str - # ... provider-specific columns - Status: str - StatusExtended: str - ResourceId: str - ResourceName: str - CheckId: str - Muted: bool -``` +1. **Schema load (both formats)**: -### Step 3 — Create `{framework}_{provider}.py` for each provider + ```python + from prowler.lib.check.compliance_models import ( + load_compliance_framework_universal, + get_bulk_compliance_frameworks_universal, + ) + fw = load_compliance_framework_universal("prowler/compliance/.json") + assert fw is not None, "check logs for the ValidationError" + print(fw.framework, len(fw.requirements), fw.get_providers()) + assert "" in get_bulk_compliance_frameworks_universal("aws") + ``` -Copy from `prowler/lib/outputs/compliance/c5/c5_aws.py` etc. Contains the `{Framework}_AWS(ComplianceOutput)` class with `transform()` that walks findings and emits model rows. This file IS allowed to import `Finding`. + Remember: the universal loader is lenient (skips broken files with a log + line) — an `assert fw is not None` is mandatory, a green scan is not proof. -### Step 4 — Register everywhere +2. **Check existence** — no loader validates this; a stale id is silent dead + weight: -**`prowler/lib/outputs/compliance/compliance.py`** (CLI table dispatcher): -```python -from prowler.lib.outputs.compliance.{framework}.{framework} import get_{framework}_table + ```python + import json + from pathlib import Path + for prov in ["aws", "azure", "gcp"]: + real = {p.stem.replace(".metadata", "") + for p in Path(f"prowler/providers/{prov}/services").rglob("*.metadata.json")} + data = json.load(open(f"prowler/compliance/{prov}/.json")) + refs = {c for r in data["Requirements"] for c in r["Checks"]} + missing = refs - real + assert not missing, f"{prov} missing: {missing}" + ``` -def display_compliance_table(...): - ... - elif compliance_framework.startswith("{framework}_"): - get_{framework}_table(findings, bulk_checks_metadata, - compliance_framework, output_filename, - output_directory, compliance_overview) -``` + (For universal files use `r.get("checks", {}).get(prov, [])` instead — + requirements may legitimately omit a provider key.) -**`prowler/__main__.py`** (CLI output writer per provider): -Add imports at the top: -```python -from prowler.lib.outputs.compliance.{framework}.{framework}_aws import {Framework}_AWS -from prowler.lib.outputs.compliance.{framework}.{framework}_azure import {Framework}_Azure -from prowler.lib.outputs.compliance.{framework}.{framework}_gcp import {Framework}_GCP -``` -Add provider-specific `elif compliance_name.startswith("{framework}_"):` branches that instantiate the class and call `batch_write_data_to_file()`. +3. **CLI smoke test**: -**`api/src/backend/tasks/jobs/export.py`** (API export dispatcher): -```python -from prowler.lib.outputs.compliance.{framework}.{framework}_aws import {Framework}_AWS -# ... azure, gcp + ```bash + uv run python prowler-cli.py --list-compliance # appears? + uv run python prowler-cli.py --compliance --log-level ERROR + ``` -COMPLIANCE_CLASS_MAP = { - "aws": [ - # ... - (lambda name: name.startswith("{framework}_"), {Framework}_AWS), - ], - # ... azure, gcp -} -``` + Verify the CSV under `output/compliance/`, the summary table sections, and + the findings roll-up. -**Always use `startswith`**, never `name == "framework_aws"`. Exact match is a regression. +4. **Tests**: -### Step 5 — Add tests + ```bash + uv run pytest -n auto tests/lib/check/universal_compliance_models_test.py \ + tests/lib/outputs/compliance/ + ``` -Create `tests/lib/outputs/compliance/{framework}/` with `{framework}_aws_test.py`, `{framework}_azure_test.py`, `{framework}_gcp_test.py`. See the test template in [references/test_template.md](references/test_template.md). + `test_loads_as_universal` is parametrized over **every** JSON in + `prowler/compliance/` (top-level + subdirectories) — a malformed file fails + CI here even if you never wrote a dedicated test. -Add fixtures to `tests/lib/outputs/compliance/fixtures.py`: one `Compliance` object per provider with 1 evaluated + 1 manual requirement to exercise both code paths in `transform()`. +5. **What CI/pre-commit do and don't cover**: pre-commit only guarantees + well-formed/pretty JSON (`check-json`, `pretty-format-json`) — no semantic + validation. The workflow `.github/workflows/pr-check-compliance-mapping.yml` + flags PRs adding new checks without mapping them to any framework (label + `needs-compliance-review`; skip with label `no-compliance-check`). Semantic + validation happens in the pytest suite above and manually via + `skills/prowler-compliance-review/assets/validate_compliance.py` (note: + that validator assumes the **legacy** schema). -### Circular import warning - -**The table dispatcher file (`{framework}.py`) MUST NOT import `Finding`** (directly or transitively). The cycle is: - -```text -compliance.compliance imports get_{framework}_table - → {framework}.py imports ComplianceOutput - → compliance_output imports Finding - → finding imports get_check_compliance from compliance.compliance - → CIRCULAR -``` - -Keep `{framework}.py` bare — only `colorama`, `tabulate`, `prowler.config.config`. Put anything that imports `Finding` in the per-provider `{framework}_{provider}.py` files. +6. **Prowler Local Server**: `docker compose up` and confirm the compliance + page renders requirements, sections and widgets. --- ## Conventions and Hard-Won Gotchas -These are lessons from the FINOS CCC v2025.10 sync + 172-AR audit pass (April 2026). Learn them once; save days of debugging. - -1. **Per-provider files are non-negotiable.** Never collapse `{framework}_aws.py`, `{framework}_azure.py`, `{framework}_gcp.py` into a single parameterized class, no matter how DRY-tempting. Every other framework in the codebase follows the per-provider pattern and reviewers will reject the refactor. The CSV column names differ per provider — three classes is the convention. -2. **`{framework}.py` has NO function docstring.** Other frameworks don't have them. Don't add one to be "helpful". -3. **Circular import protection**: the table dispatcher file MUST NOT import `Finding` (directly or transitively). Split the code so `{framework}.py` only has `get_{framework}_table()` with bare imports, and `{framework}_{provider}.py` holds the class that needs `Finding`. -4. **`Generic_Compliance_Requirement_Attribute` is the fallback** — in the `Compliance_Requirement.Attributes` Union in `compliance_models.py`, Generic MUST be LAST because Pydantic v1 tries union members in order. Putting Generic first means every framework-specific attribute falls through to Generic and the specific model is never used. -5. **Pydantic v1 imports.** `from pydantic.v1 import BaseModel` in `compliance_models.py` — not v2. Mixing causes validation errors. Pydantic v2 is used in the CSV models (`models.py`) — that's fine because they're separate trees. -6. **`get_check_compliance()` key format** is `f"{Framework}-{Version}"` ONLY if Version is set. Empty Version → key is `"{Framework}"` (no version suffix). Tests that mock compliance dicts must match this exact format — when a framework ships with `Version: ""`, downstream code and tests break silently. -7. **CSV column names from `models.py` are public API.** Don't rename a field without migrating downstream consumers — CSV headers change. -8. **Upstream YAML multi-line scalars** (`|` block scalars) preserve newlines. Collapse to single-line with `" ".join(value.split())` before writing to JSON. -9. **Upstream catalogs can use multiple shapes.** FINOS CCC uses `control-families: [...]` in most catalogs but `controls: [...]` at the top level in `storage/object`. Any sync script must handle both or silently drop entire catalogs. -10. **Foreign-prefix AR ids.** Upstream sometimes "imports" requirements from one catalog into another by keeping the original id prefix (e.g., `CCC.AuditLog.CN08.AR01` appearing under `CCC.Logging.CN03`). Prowler's compliance model requires unique ids within a catalog — rewrite the foreign id to fit the parent control: `CCC.AuditLog.CN08.AR01` (inside `CCC.Logging.CN03`) → `CCC.Logging.CN03.AR01`. -11. **Genuine upstream id collisions.** Sometimes upstream has a real typo where two different requirements share the same id (e.g., `CCC.Core.CN14.AR02` defined twice for 30-day and 14-day backup variants). Renumber the second copy to the next free AR number. Preserve check mappings by matching on `(Section, frozenset(Applicability))` since the renumbered id won't match by id. -12. **`COMPLIANCE_CLASS_MAP` in `export.py` uses `startswith` predicates** for all modern frameworks. Exact match (`name == "ccc_aws"`) is an anti-pattern — it was present for CCC until April 2026 and was the reason CCC couldn't have versioned variants. -13. **Pre-validate every check id** against the per-provider inventory before writing the JSON. A typo silently creates an unreferenced check that will fail when findings try to map to it. The audit script MUST abort with stderr listing typos, not swallow them. -14. **REPLACE is better than PATCH** for audit decisions. Encoding every mapping explicitly makes the audit reproducible and surfaces hidden assumptions from the legacy data. A PATCH system that adds/removes is too easy to forget. -15. **When no check applies, MANUAL is correct.** Do not pad mappings with tangential checks "just in case". Prowler's compliance reports are meant to be actionable — padding them with noise breaks that. Honest manual reqs can be mapped later when new checks land. -16. **UI groups by `Attributes[0].FamilyName` and `Attributes[0].Section`.** If FamilyName has inconsistent variants within the same JSON (e.g., "Logging & Monitoring" vs "Logging and Monitoring"), the UI renders them as separate categories. Section empty → the requirement falls into an orphan control with label "". Normalize before shipping. -17. **Provider coverage is asymmetric.** AWS has dense coverage (~586 checks across 80+ services): in-transit encryption, IAM, database encryption, backup. Azure (~167 checks) and GCP (~102 checks) are thinner especially for in-transit encryption, mTLS, and ML/AI. Accept the asymmetry in mappings — don't force GCP parity where Prowler genuinely can't verify. +1. **Universal first.** A new framework that starts as legacy needs 3 output + files + 3 registrations; the same framework as universal needs zero. Only + extend legacy families. +2. **`Generic_Compliance_Requirement_Attribute` stays LAST** in the legacy + Attributes Union — Pydantic v1 tries members in order; Generic first + silently swallows every specific shape. +3. **Pydantic v1 everywhere in `compliance_models.py`** + (`from pydantic.v1 import ...`). Don't mix in v2. +4. **`get_check_compliance()` lives in + `prowler/lib/outputs/compliance/compliance_check.py`** and keys the dict + `f"{Framework}-{Version}"` only when Version is non-empty. Never ship + `Version: ""` — the key silently degrades to `"{Framework}"` and breaks + filters, tests and `--compliance`. For legacy files the filename version + substring must match `Version` (the CLI reads + `compliance_framework.split("_")[1]`). +5. **`Compliance.get_bulk()` does not see top-level universal files** — only + `get_bulk_compliance_frameworks_universal()` does. Wire new code paths + against the universal loader. +6. **Loader leniency differs**: legacy loader exits the process on a broken + JSON; universal loader logs and skips. A missing framework after your edit + usually means the universal loader dropped it — check the logs. +7. **Circular import protection**: legacy table dispatcher files must not + import `Finding` (directly or transitively). Use `TYPE_CHECKING` or + function-local imports when a module needs both sides (that's how the + universal formatter does it). +8. **Per-provider formatter files are the legacy convention** — but know the + exceptions before flagging them (iso27001 has no table file, + aws_well_architected has no per-provider files, cisa_scuba is + googleworkspace-only). CSV model field names are public API. +9. **CSV output**: `;` delimiter, UPPERCASE headers. OCSF compliance output is + always generated for universal frameworks regardless of `--output-formats`. +10. **`COMPLIANCE_CLASS_MAP` mixes predicate styles**: `startswith` for + multi-version families, exact `==` for singletons. When in doubt use + `startswith` — exact match blocked versioned CCC variants until 2026. +11. **UI grouping is per-mapper, always on `attributes[0]`**: generic/cis → + `Section`/`SubSection`, iso → `Category`, ccc → `FamilyName`. Inconsistent + values (or empty Section) create orphan/duplicate tree branches — normalize + before shipping. +12. **UI has a generic fallback** — an unregistered framework still renders. + A dedicated mapper/panel/icon is an upgrade, not a prerequisite. +13. **Icon registration is ordered substring matching** in + `IconCompliance.tsx` — specific keywords before generic (`nist` before + `nis2`, `cisa` before `cis`, `aws` last). +14. **API PDF pipeline is not `PDFConfig`-driven yet** — it has its own + `FRAMEWORK_REGISTRY` (5 frameworks). Don't assume adding `pdf_config` to a + JSON produces a PDF in Prowler App. +15. **Pre-validate every check id** against the per-provider inventory before + writing JSON. No loader will catch a typo; the requirement just never + matches a finding. +16. **REPLACE beats PATCH** for audit decisions — full explicit lists are + reproducible and surface legacy assumptions. +17. **When no check applies, MANUAL is correct.** Don't pad mappings with + tangential checks; compliance reports must stay actionable. +18. **Include every requirement of the source catalog**, automated or not — + compliance percentages use the full requirement count as denominator. +19. **Provider coverage is asymmetric** (AWS dense; Azure/GCP thinner; new + providers minimal). Accept it — don't force parity Prowler can't verify. +20. **Guardrail authoring**: strictest tolerated `Value`, exact `ConfigKey` + spelling, `Provider` mandatory in universal files, booleans as JSON + booleans. Malformed constraints are treated as satisfied — validate with + the config tests, don't trust silence. --- ## Useful One-Liners ```bash -# Count requirements per service prefix (CCC, CIS sections, etc.) -jq -r '.Requirements[].Id | split(".")[1]' prowler/compliance/aws/ccc_aws.json | sort | uniq -c - -# Find duplicate requirement IDs +# Find duplicate requirement IDs (legacy | universal) jq -r '.Requirements[].Id' file.json | sort | uniq -d +jq -r '.requirements[].id' file.json | sort | uniq -d -# Count manual requirements (no checks) +# Count manual requirements (legacy | universal, per provider) jq '[.Requirements[] | select((.Checks | length) == 0)] | length' file.json +jq '[.requirements[] | select((.checks.aws // [] | length) == 0)] | length' file.json -# List all unique check references in a framework +# List unique check references (legacy | universal) jq -r '.Requirements[].Checks[]' file.json | sort -u +jq -r '.requirements[].checks[]? | .[]' file.json | sort -u -# List all unique Sections (to spot inconsistency) +# Providers covered by a universal framework +jq '[.requirements[].checks | keys[]] | unique' file.json + +# Spot inconsistent grouping values (UI tree branches) jq '[.Requirements[].Attributes[0].Section] | unique' file.json - -# List all unique FamilyNames (to spot inconsistency) jq '[.Requirements[].Attributes[0].FamilyName] | unique' file.json -# Diff requirement ids between two versions of the same framework +# Requirements with config guardrails (empty arrays are truthy in jq — check length) +jq '[.Requirements[] | select((.ConfigRequirements // []) | length > 0)] | length' file.json + +# Diff requirement ids between two versions diff <(jq -r '.Requirements[].Id' a.json | sort) <(jq -r '.Requirements[].Id' b.json | sort) -# Find where a check id is used across all frameworks +# Where is a check mapped across all frameworks? grep -rl "my_check_name" prowler/compliance/ -# Check if a Prowler check exists +# Does a check exist? find prowler/providers/aws/services -name "{check_id}.metadata.json" -# Validate a JSON with Pydantic -python -c "from prowler.lib.check.compliance_models import Compliance; print(Compliance.parse_file('prowler/compliance/aws/ccc_aws.json').Framework)" +# Validate one file with the universal loader +python -c "from prowler.lib.check.compliance_models import load_compliance_framework_universal as l; fw=l('prowler/compliance/aws/cis_7.0_aws.json'); print(fw.framework, len(fw.requirements))" ``` ---- - -## Best Practices - -1. **Requirement IDs**: Follow the original framework numbering exactly (e.g., "1.1", "A.5.1", "T1190", "ac_2_1") -2. **Check Mapping**: Map to existing checks when possible. Use `Checks: []` for manual-only requirements — honest MANUAL beats padded coverage -3. **Completeness**: Include all framework requirements, even those without automated checks -4. **Version Control**: Include framework version in `Name` and `Version` fields. **Never leave `Version: ""`** — it breaks `get_check_compliance()` key format -5. **File Naming**: Use format `{framework}_{version}_{provider}.json` -6. **Validation**: Prowler validates JSON against Pydantic models at startup — invalid JSON will cause errors -7. **Pre-validate check ids** against the provider's `*.metadata.json` inventory before every commit -8. **Normalize FamilyName and Section** to avoid inconsistent UI tree branches -9. **Register everywhere**: SDK model (if needed) → `compliance.py` dispatcher → `__main__.py` CLI writer → `export.py` API map → UI mapper. Skipping any layer results in silent failures -10. **Audit, don't pad**: when reviewing mappings, apply the golden rule — the check's title/risk MUST literally describe what the requirement text says. Tangential relation doesn't count - ## Commands ```bash -# List available frameworks for a provider prowler {provider} --list-compliance - -# Run scan with specific compliance framework -prowler aws --compliance cis_5.0_aws - -# Run scan with multiple frameworks -prowler aws --compliance cis_5.0_aws pci_4.0_aws - -# Output compliance report in multiple formats -prowler aws --compliance cis_5.0_aws -M csv json html +prowler {provider} --compliance cis_7.0_aws +prowler aws --compliance cis_7.0_aws pci_4.0_aws +prowler aws --compliance dora_2022_2554 # universal key = file basename +prowler aws --list-compliance-requirements cis_7.0_aws +prowler aws --compliance cis_7.0_aws -M csv json html ``` ## Code References ### Layer 1 — SDK / Core -- **Compliance Models:** `prowler/lib/check/compliance_models.py` (Pydantic v1 model tree) -- **Compliance Processing / Linker:** `prowler/lib/check/compliance.py` (`get_check_compliance`, `update_checks_metadata_with_compliance`) -- **Check Utils:** `prowler/lib/check/utils.py` (`list_compliance_modules`) + +- `prowler/lib/check/compliance_models.py` — legacy + universal model trees, + `Compliance_Requirement_ConfigConstraint`, all loaders and the + legacy→universal adapter +- `prowler/lib/check/compliance.py` — `update_checks_metadata_with_compliance` +- `prowler/lib/check/compliance_config_eval.py` — guardrail evaluation + (shared with the API) +- `prowler/lib/outputs/compliance/compliance_check.py` — `get_check_compliance` +- `prowler/lib/check/utils.py` — `list_compliance_modules` ### Layer 2 — JSON Catalogs -- **Framework JSONs:** `prowler/compliance/{provider}/` (auto-discovered via directory walk) + +- `prowler/compliance/*.json` — universal, multi-provider (auto-discovered) +- `prowler/compliance/{provider}/` — legacy, per-provider (auto-discovered) ### Layer 3 — Output Formatters -- **Per-framework folders:** `prowler/lib/outputs/compliance/{framework}/` -- **Shared base class:** `prowler/lib/outputs/compliance/compliance_output.py` (`ComplianceOutput` + `batch_write_data_to_file`) -- **CLI table dispatcher:** `prowler/lib/outputs/compliance/compliance.py` (`display_compliance_table`) -- **Finding model:** `prowler/lib/outputs/finding.py` (**do not import transitively from table dispatcher files — circular import**) -- **CLI writer:** `prowler/__main__.py` (per-provider `elif compliance_name.startswith(...)` branches that instantiate per-provider classes) + +- `prowler/lib/outputs/compliance/universal/` — `universal_table.py`, + `universal_output.py`, `ocsf_compliance.py` +- `prowler/lib/outputs/compliance/{framework}/` — legacy per-framework packages +- `prowler/lib/outputs/compliance/compliance.py` — + `process_universal_compliance_frameworks`, `display_compliance_table` +- `prowler/lib/outputs/compliance/compliance_output.py` — `ComplianceOutput` + base + CSV writer +- `prowler/__main__.py` — universal processing + per-provider legacy writer + branches ### Layer 4 — API / UI -- **API lazy loader:** `api/src/backend/api/compliance.py` (`LazyComplianceTemplate`, `LazyChecksMapping`) -- **API export dispatcher:** `api/src/backend/tasks/jobs/export.py` (`COMPLIANCE_CLASS_MAP` with `startswith` predicates) -- **UI framework router:** `ui/lib/compliance/compliance-mapper.ts` -- **UI per-framework mapper:** `ui/lib/compliance/{framework}.tsx` -- **UI detail panel:** `ui/components/compliance/compliance-custom-details/{framework}-details.tsx` -- **UI types:** `ui/types/compliance.ts` -- **UI icon:** `ui/components/icons/compliance/{framework}.svg` + registration in `IconCompliance.tsx` + +- `api/src/backend/api/compliance.py` — `LazyComplianceTemplate`, + `LazyChecksMapping`, cache warm-up +- `api/src/backend/tasks/jobs/export.py` — `COMPLIANCE_CLASS_MAP` +- `api/src/backend/tasks/jobs/scan.py` — `create_compliance_requirements` + (overview ingestion) +- `api/src/backend/tasks/jobs/reports/` — PDF generators + `FRAMEWORK_REGISTRY` +- `ui/lib/compliance/compliance-mapper.ts` — mapper routing + generic fallback +- `ui/lib/compliance/{framework}.tsx` — per-framework mappers +- `ui/components/compliance/compliance-custom-details/` — detail panels +- `ui/types/compliance.ts` — attribute metadata types +- `ui/components/icons/compliance/` + `IconCompliance.tsx` — icons (ordered) ### Tests -- **Output formatter tests:** `tests/lib/outputs/compliance/{framework}/{framework}_{provider}_test.py` -- **Shared fixtures:** `tests/lib/outputs/compliance/fixtures.py` + +- `tests/lib/check/universal_compliance_models_test.py` — includes the + parametrized `test_loads_as_universal` over every shipped JSON +- `tests/lib/check/compliance_check_test.py`, + `compliance_config_eval_test.py`, `compliance_config_constraint_model_test.py`, + `compliance_config_requirements_data_test.py`, `mitre_config_requirements_test.py` +- `tests/lib/outputs/compliance/` — per-framework + universal + dispatcher + + config-status coverage tests; shared `fixtures.py` ## Resources -- **JSON Templates:** See [assets/](assets/) for framework JSON templates (cis, ens, iso27001, mitre_attack, prowler_threatscore, generic) -- **Config-driven compliance sync** (any upstream-backed framework): - - [assets/sync_framework.py](assets/sync_framework.py) — generic runner. Loads a YAML config, dynamically imports the declared parser, applies generic post-processing (id uniqueness safety net, `FamilyName` normalization, legacy check-mapping preservation with config-driven fallback keys), and writes the provider JSONs with Pydantic post-validation. Framework-agnostic — works for any compliance framework. - - [assets/configs/ccc.yaml](assets/configs/ccc.yaml) — canonical config example (FINOS CCC v2025.10). Copy and adapt for new frameworks. - - [assets/parsers/finos_ccc.py](assets/parsers/finos_ccc.py) — FINOS CCC YAML parser. Handles both upstream shapes (`control-families` and top-level `controls`), foreign-prefix AR rewriting, and genuine collision renumbering. Exposes `parse_upstream(config) -> list[dict]`. - - [assets/parsers/](assets/parsers/) — add new parser modules here for unfamiliar upstream formats (NIST OSCAL JSON, MITRE STIX, CIS Benchmarks, etc.). Each parser is a `{name}.py` file implementing `parse_upstream(config) -> list[dict]` with guaranteed-unique ids. -- **Reusable audit tooling** (added April 2026 after the FINOS CCC v2025.10 sync): - - [assets/audit_framework_template.py](assets/audit_framework_template.py) — explicit REPLACE decision ledger with pre-validation against the per-provider inventory. Drop-in template for auditing any framework. - - [assets/query_checks.py](assets/query_checks.py) — keyword/service/id query helper over `/tmp/checks_{provider}.json`. - - [assets/dump_section.py](assets/dump_section.py) — dumps every AR for a given id prefix across all 3 providers with current check mappings. - - [assets/build_inventory.py](assets/build_inventory.py) — generates `/tmp/checks_{provider}.json` from `*.metadata.json` files. -- **Documentation:** See [references/compliance-docs.md](references/compliance-docs.md) for additional resources -- **Related skill:** [prowler-compliance-review](../prowler-compliance-review/SKILL.md) — PR review checklist and validator script for compliance framework PRs +- **Docs (source of truth for contributors)**: + `docs/developer-guide/security-compliance-framework.mdx` (both schemas, + guardrails, validation, PR process), + `docs/user-guide/compliance/tutorials/compliance.mdx`, + `docs/user-guide/compliance/tutorials/cross-provider-compliance.mdx` +- **Repo tooling** (`util/compliance/`): CSV→JSON generators + (`generate_json_from_csv/`), `ccc/from_yaml_to_json.py`, + `compliance_mapper/`, `threatscore/` +- **Skill assets** ([assets/](assets/)): + - `sync_framework.py` + `configs/ccc.yaml` + `parsers/finos_ccc.py` — + config-driven upstream sync (Workflow A) + - `build_inventory.py`, `query_checks.py`, `dump_section.py`, + `audit_framework_template.py` — audit tooling (Workflow B) + - Legacy JSON templates: `cis_framework.json`, `ens_framework.json`, + `iso27001_framework.json`, `mitre_attack_framework.json`, + `prowler_threatscore_framework.json`, `generic_framework.json` +- **References**: + [references/compliance-docs.md](references/compliance-docs.md) — model/loader + quick reference; + [references/check-mapping-reference.md](references/check-mapping-reference.md) + — curated requirement-text → checks mapping table + honest-MANUAL list +- **Sister skill**: + [prowler-compliance-review](../prowler-compliance-review/SKILL.md) — PR + review checklist + `validate_compliance.py` (legacy-schema validator) +- After editing this skill's frontmatter, run + `./skills/skill-sync/assets/sync.sh` to regenerate the AGENTS.md auto-invoke + tables. diff --git a/skills/prowler-compliance/references/check-mapping-reference.md b/skills/prowler-compliance/references/check-mapping-reference.md new file mode 100644 index 0000000000..cae9701ae7 --- /dev/null +++ b/skills/prowler-compliance/references/check-mapping-reference.md @@ -0,0 +1,78 @@ +# Audit Reference: Requirement Text → Prowler Checks + +Built from a real audit of 172 CCC ARs × 3 providers (April 2026). Use it to map +CCC-style / NIST-style / ISO-style requirement text to the checks that actually +verify them. Always re-validate every check id against the current inventory +(`assets/build_inventory.py` + `assets/query_checks.py`) before using a row — +checks get renamed and added over time. + +**Entries containing `*` are glob patterns, NOT literal check ids** (e.g. +`iam_*_no_administrative_privileges`, `cloudwatch_log_metric_filter_*`, +`*_minimum_tls_version_12`). Copied verbatim into a compliance JSON they map +nothing — expand each pattern to the concrete check ids via +`python skills/prowler-compliance/assets/query_checks.py ` +before writing any mapping. + +| Requirement text | AWS checks | Azure checks | GCP checks | +|---|---|---|---| +| **TLS in transit enforced** | `cloudfront_distributions_https_enabled`, `s3_bucket_secure_transport_policy`, `elbv2_ssl_listeners`, `elbv2_insecure_ssl_ciphers`, `elb_ssl_listeners`, `elb_insecure_ssl_ciphers`, `opensearch_service_domains_https_communications_enforced`, `rds_instance_transport_encrypted`, `redshift_cluster_in_transit_encryption_enabled`, `elasticache_redis_cluster_in_transit_encryption_enabled`, `dynamodb_accelerator_cluster_in_transit_encryption_enabled`, `dms_endpoint_ssl_enabled`, `kafka_cluster_in_transit_encryption_enabled`, `transfer_server_in_transit_encryption_enabled`, `glue_database_connections_ssl_enabled`, `sns_subscription_not_using_http_endpoints` | `storage_secure_transfer_required_is_enabled`, `storage_ensure_minimum_tls_version_12`, `postgresql_flexible_server_enforce_ssl_enabled`, `mysql_flexible_server_ssl_connection_enabled`, `mysql_flexible_server_minimum_tls_version_12`, `sqlserver_recommended_minimal_tls_version`, `app_minimum_tls_version_12`, `app_ensure_http_is_redirected_to_https`, `app_ftp_deployment_disabled` | `cloudsql_instance_ssl_connections` (almost only option) | +| **TLS 1.3 specifically** | Partial: `cloudfront_distributions_using_deprecated_ssl_protocols`, `elb*_insecure_ssl_ciphers`, `*_minimum_tls_version_12` | Partial: `*_minimum_tls_version_12` checks | None — accept as MANUAL | +| **SSH / port 22 hardening** | `ec2_instance_port_ssh_exposed_to_internet`, `ec2_securitygroup_allow_ingress_from_internet_to_tcp_port_22`, `ec2_networkacl_allow_ingress_tcp_port_22` | `network_ssh_internet_access_restricted`, `vm_linux_enforce_ssh_authentication` | `compute_firewall_ssh_access_from_the_internet_allowed`, `compute_instance_block_project_wide_ssh_keys_disabled`, `compute_project_os_login_enabled`, `compute_project_os_login_2fa_enabled` | +| **mTLS (mutual TLS)** | `kafka_cluster_mutual_tls_authentication_enabled`, `apigateway_restapi_client_certificate_enabled` | `app_client_certificates_on` | None — MANUAL | +| **Data at rest encrypted** | `s3_bucket_default_encryption`, `s3_bucket_kms_encryption`, `ec2_ebs_default_encryption`, `ec2_ebs_volume_encryption`, `rds_instance_storage_encrypted`, `rds_cluster_storage_encrypted`, `rds_snapshots_encrypted`, `dynamodb_tables_kms_cmk_encryption_enabled`, `redshift_cluster_encrypted_at_rest`, `neptune_cluster_storage_encrypted`, `documentdb_cluster_storage_encrypted`, `opensearch_service_domains_encryption_at_rest_enabled`, `kinesis_stream_encrypted_at_rest`, `firehose_stream_encrypted_at_rest`, `sns_topics_kms_encryption_at_rest_enabled`, `sqs_queues_server_side_encryption_enabled`, `efs_encryption_at_rest_enabled`, `athena_workgroup_encryption`, `glue_data_catalogs_metadata_encryption_enabled`, `backup_vaults_encrypted`, `backup_recovery_point_encrypted`, `cloudtrail_kms_encryption_enabled`, `cloudwatch_log_group_kms_encryption_enabled`, `eks_cluster_kms_cmk_encryption_in_secrets_enabled`, `sagemaker_notebook_instance_encryption_enabled`, `apigateway_restapi_cache_encrypted`, `kafka_cluster_encryption_at_rest_uses_cmk`, `dynamodb_accelerator_cluster_encryption_enabled`, `storagegateway_fileshare_encryption_enabled` | `storage_infrastructure_encryption_is_enabled`, `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encryption_enabled`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled`, `monitor_storage_account_with_activity_logs_cmk_encrypted` | `compute_instance_encryption_with_csek_enabled`, `dataproc_encrypted_with_cmks_disabled`, `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption` | +| **CMEK required (customer-managed keys)** | `kms_cmk_are_used` | `storage_ensure_encryption_with_customer_managed_keys`, `vm_ensure_attached_disks_encrypted_with_cmk`, `vm_ensure_unattached_disks_encrypted_with_cmk`, `sqlserver_tde_encrypted_with_cmk`, `databricks_workspace_cmk_encryption_enabled` | `bigquery_dataset_cmk_encryption`, `bigquery_table_cmk_encryption`, `dataproc_encrypted_with_cmks_disabled`, `compute_instance_encryption_with_csek_enabled` | +| **Key rotation enabled** | `kms_cmk_rotation_enabled` | `keyvault_key_rotation_enabled`, `storage_key_rotation_90_days` | `kms_key_rotation_enabled` | +| **MFA for UI access** | `iam_root_mfa_enabled`, `iam_root_hardware_mfa_enabled`, `iam_user_mfa_enabled_console_access`, `iam_user_hardware_mfa_enabled`, `iam_administrator_access_with_mfa`, `cognito_user_pool_mfa_enabled` | `entra_privileged_user_has_mfa`, `entra_non_privileged_user_has_mfa`, `entra_user_with_vm_access_has_mfa`, `entra_security_defaults_enabled` | `compute_project_os_login_2fa_enabled` | +| **API access / credentials** | `iam_no_root_access_key`, `iam_user_no_setup_initial_access_key`, `apigateway_restapi_authorizers_enabled`, `apigateway_restapi_public_with_authorizer`, `apigatewayv2_api_authorizers_enabled` | `entra_conditional_access_policy_require_mfa_for_management_api`, `app_function_access_keys_configured`, `app_function_identity_is_configured` | `apikeys_api_restrictions_configured`, `apikeys_key_exists`, `apikeys_key_rotated_in_90_days` | +| **Log all admin/config changes** | `cloudtrail_multi_region_enabled`, `cloudtrail_multi_region_enabled_logging_management_events`, `cloudtrail_cloudwatch_logging_enabled`, `cloudtrail_log_file_validation_enabled`, `cloudwatch_log_metric_filter_*`, `cloudwatch_changes_to_*_alarm_configured`, `config_recorder_all_regions_enabled` | `monitor_diagnostic_settings_exists`, `monitor_diagnostic_setting_with_appropriate_categories`, `monitor_alert_*` | `iam_audit_logs_enabled`, `logging_log_metric_filter_and_alert_for_*`, `logging_sink_created` | +| **Log integrity (digital signatures)** | `cloudtrail_log_file_validation_enabled` (exact) | None | None | +| **Public access denied** | `s3_bucket_public_access`, `s3_bucket_public_list_acl`, `s3_bucket_public_write_acl`, `s3_account_level_public_access_blocks`, `apigateway_restapi_public`, `awslambda_function_url_public`, `awslambda_function_not_publicly_accessible`, `rds_instance_no_public_access`, `rds_snapshots_public_access`, `ec2_securitygroup_allow_ingress_from_internet_to_all_ports`, `sns_topics_not_publicly_accessible`, `sqs_queues_not_publicly_accessible` | `storage_blob_public_access_level_is_disabled`, `storage_ensure_private_endpoints_in_storage_accounts`, `containerregistry_not_publicly_accessible`, `keyvault_private_endpoints`, `app_function_not_publicly_accessible`, `aks_clusters_public_access_disabled`, `network_http_internet_access_restricted` | `cloudstorage_bucket_public_access`, `compute_instance_public_ip`, `cloudsql_instance_public_ip`, `compute_firewall_*_access_from_the_internet_allowed` | +| **IAM least privilege** | `iam_*_no_administrative_privileges`, `iam_policy_allows_privilege_escalation`, `iam_inline_policy_allows_privilege_escalation`, `iam_role_administratoraccess_policy`, `iam_group_administrator_access_policy`, `iam_user_administrator_access_policy`, `iam_policy_attached_only_to_group_or_roles`, `iam_role_cross_service_confused_deputy_prevention` | `iam_role_user_access_admin_restricted`, `iam_subscription_roles_owner_custom_not_created`, `iam_custom_role_has_permissions_to_administer_resource_locks` | `iam_sa_no_administrative_privileges`, `iam_no_service_roles_at_project_level`, `iam_role_kms_enforce_separation_of_duties`, `iam_role_sa_enforce_separation_of_duties` | +| **Password policy** | `iam_password_policy_minimum_length_14`, `iam_password_policy_uppercase`, `iam_password_policy_lowercase`, `iam_password_policy_symbol`, `iam_password_policy_number`, `iam_password_policy_expires_passwords_within_90_days_or_less`, `iam_password_policy_reuse_24` | None | None | +| **Credential rotation / unused** | `iam_rotate_access_key_90_days`, `iam_user_accesskey_unused`, `iam_user_console_access_unused` | None | `iam_sa_user_managed_key_rotate_90_days`, `iam_sa_user_managed_key_unused`, `iam_service_account_unused` | +| **VPC / flow logs** | `vpc_flow_logs_enabled` | `network_flow_log_captured_sent`, `network_watcher_enabled`, `network_flow_log_more_than_90_days` | `compute_subnet_flow_logs_enabled` | +| **Backup / DR / Multi-AZ** | `backup_vaults_exist`, `backup_plans_exist`, `backup_reportplans_exist`, `rds_instance_backup_enabled`, `rds_*_protected_by_backup_plan`, `rds_cluster_multi_az`, `neptune_cluster_backup_enabled`, `documentdb_cluster_backup_enabled`, `efs_have_backup_enabled`, `s3_bucket_cross_region_replication`, `dynamodb_table_protected_by_backup_plan` | `vm_backup_enabled`, `vm_sufficient_daily_backup_retention_period`, `storage_geo_redundant_enabled` | `cloudsql_instance_automated_backups`, `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_sufficient_retention_period` | +| **Access analysis / discovery** | `accessanalyzer_enabled`, `accessanalyzer_enabled_without_findings` | None specific | `iam_account_access_approval_enabled`, `iam_cloud_asset_inventory_enabled` | +| **Object lock / retention** | `s3_bucket_object_lock`, `s3_bucket_object_versioning`, `s3_bucket_lifecycle_enabled`, `cloudtrail_bucket_requires_mfa_delete`, `s3_bucket_no_mfa_delete` | `storage_ensure_soft_delete_is_enabled`, `storage_blob_versioning_is_enabled`, `storage_ensure_file_shares_soft_delete_is_enabled` | `cloudstorage_bucket_log_retention_policy_lock`, `cloudstorage_bucket_soft_delete_enabled`, `cloudstorage_bucket_versioning_enabled`, `cloudstorage_bucket_sufficient_retention_period` | +| **Uniform bucket-level access** | `s3_bucket_acl_prohibited` | `storage_account_key_access_disabled`, `storage_default_to_entra_authorization_enabled` | `cloudstorage_bucket_uniform_bucket_level_access` | +| **Container vulnerability scanning** | `ecr_registry_scan_images_on_push_enabled`, `ecr_repositories_scan_vulnerabilities_in_latest_image` | `defender_container_images_scan_enabled`, `defender_container_images_resolved_vulnerabilities` | `artifacts_container_analysis_enabled`, `gcr_container_scanning_enabled` | +| **WAF / rate limiting** | `wafv2_webacl_with_rules`, `waf_*_webacl_with_rules`, `wafv2_webacl_logging_enabled`, `waf_global_webacl_logging_enabled` | None | None | +| **Deployment region restriction** | `organizations_scp_check_deny_regions` | None | None | +| **Secrets automatic rotation** | `secretsmanager_automatic_rotation_enabled`, `secretsmanager_secret_rotated_periodically` | `keyvault_rbac_secret_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | +| **Certificate management** | `acm_certificates_expiration_check`, `acm_certificates_with_secure_key_algorithms`, `acm_certificates_transparency_logs_enabled` | `keyvault_key_expiration_set_in_non_rbac`, `keyvault_rbac_key_expiration_set`, `keyvault_non_rbac_secret_expiration_set` | None | +| **GenAI guardrails / input/output filtering** | `bedrock_guardrail_prompt_attack_filter_enabled`, `bedrock_guardrail_sensitive_information_filter_enabled`, `bedrock_agent_guardrail_enabled`, `bedrock_model_invocation_logging_enabled`, `bedrock_api_key_no_administrative_privileges`, `bedrock_api_key_no_long_term_credentials` | None | None | +| **ML dev environment security** | `sagemaker_notebook_instance_root_access_disabled`, `sagemaker_notebook_instance_without_direct_internet_access_configured`, `sagemaker_notebook_instance_vpc_settings_configured`, `sagemaker_models_vpc_settings_configured`, `sagemaker_training_jobs_vpc_settings_configured`, `sagemaker_training_jobs_network_isolation_enabled`, `sagemaker_training_jobs_volume_and_output_encryption_enabled` | None | None | +| **Threat detection / anomalous behavior** | `cloudtrail_threat_detection_enumeration`, `cloudtrail_threat_detection_privilege_escalation`, `cloudtrail_threat_detection_llm_jacking`, `guardduty_is_enabled`, `guardduty_no_high_severity_findings` | None | None | +| **Serverless private access** | `awslambda_function_inside_vpc`, `awslambda_function_not_publicly_accessible`, `awslambda_function_url_public` | `app_function_not_publicly_accessible` | None | + +## What Prowler Does NOT Cover (accept MANUAL honestly) + +Don't pad mappings for these — mark the requirement's checks empty and move on: + +- **TLS 1.3 version specifically** — Prowler verifies TLS is enforced, not always the exact version +- **IANA port-protocol consistency** — no check for "protocol running on its assigned port" +- **mTLS on most Azure/GCP services** — limited to App Service client certs on Azure, nothing on GCP +- **Rate limiting** on monitoring endpoints, load balancers, serverless invocations, vector ingestion +- **Session cookie expiry** (LB stickiness) +- **HTTP header scrubbing** (Server, X-Powered-By) +- **Certificate transparency verification for imports** +- **Model version pinning, red teaming, AI quality review** +- **Vector embedding validation, dimensional constraints, ANN vs exact search** +- **Secret region replication** (cross-region residency) +- **Lifecycle cleanup policies on container registries** +- **Row-level / column-level security in data warehouses** +- **Deployment region restriction on Azure/GCP** (AWS has `organizations_scp_check_deny_regions`, others don't) +- **Cross-tenant alert silencing permissions** +- **Field-level masking in logs** +- **Managed view enforcement for database access** +- **Automatic MFA delete on all S3 buckets** (only CloudTrail bucket variant exists for some frameworks — AWS has the generic `s3_bucket_no_mfa_delete` though) + +## Provider coverage asymmetry + +AWS has dense coverage (in-transit encryption, IAM, database encryption, backup, +GenAI). Azure and GCP are thinner, especially for in-transit encryption, mTLS, +and ML/AI. Accept the asymmetry in mappings — don't force GCP parity where +Prowler genuinely can't verify. Newer providers (alibabacloud, oraclecloud, +googleworkspace, okta, cloudflare, linode...) have far smaller inventories: +always rebuild the inventory with `assets/build_inventory.py` before assuming +a mapping exists. diff --git a/skills/prowler-compliance/references/compliance-docs.md b/skills/prowler-compliance/references/compliance-docs.md index a8d11484a9..272aa10ef1 100644 --- a/skills/prowler-compliance/references/compliance-docs.md +++ b/skills/prowler-compliance/references/compliance-docs.md @@ -1,137 +1,154 @@ -# Compliance Framework Documentation +# Compliance Framework Quick Reference ## Code References -Key files for understanding and modifying compliance frameworks: - | File | Purpose | |------|---------| -| `prowler/lib/check/compliance_models.py` | Pydantic models defining attribute structures for each framework type | -| `prowler/lib/check/compliance.py` | Core compliance processing logic | -| `prowler/lib/check/utils.py` | Utility functions including `list_compliance_modules()` | -| `prowler/lib/outputs/compliance/` | Framework-specific output generators | -| `prowler/compliance/{provider}/` | JSON compliance framework definitions | +| `prowler/lib/check/compliance_models.py` | Legacy + universal Pydantic (v1) model trees, config-constraint model, loaders, legacy→universal adapter | +| `prowler/lib/check/compliance.py` | `update_checks_metadata_with_compliance()` (only) | +| `prowler/lib/check/compliance_config_eval.py` | Shared `ConfigRequirements` guardrail evaluation (SDK outputs + API) | +| `prowler/lib/outputs/compliance/compliance_check.py` | `get_check_compliance()` — per-finding `{Framework}-{Version}` → requirement ids | +| `prowler/lib/check/utils.py` | `list_compliance_modules()` | +| `prowler/lib/outputs/compliance/` | Output formatters (legacy per-framework + `universal/`) | +| `prowler/compliance/*.json` | Universal multi-provider framework definitions | +| `prowler/compliance/{provider}/` | Legacy per-provider framework definitions | -## Attribute Model Classes +## Attribute Model Classes (legacy schema) -Each framework type has a specific Pydantic model in `compliance_models.py`: +Registered in the `Compliance_Requirement.Attributes` Union, in this order +(order is load-bearing; Generic must stay last): -| Framework | Model Class | +| Framework family | Model Class | |-----------|-------------| +| ASD Essential Eight | `ASDEssentialEight_Requirement_Attribute` | | CIS | `CIS_Requirement_Attribute` | -| ISO 27001 | `ISO27001_2013_Requirement_Attribute` | | ENS | `ENS_Requirement_Attribute` | -| MITRE ATT&CK | `Mitre_Requirement` (uses different structure) | +| ISO 27001 | `ISO27001_2013_Requirement_Attribute` | | AWS Well-Architected | `AWS_Well_Architected_Requirement_Attribute` | | KISA ISMS-P | `KISA_ISMSP_Requirement_Attribute` | | Prowler ThreatScore | `Prowler_ThreatScore_Requirement_Attribute` | | CCC | `CCC_Requirement_Attribute` | | C5 Germany | `C5Germany_Requirement_Attribute` | -| Generic/Fallback | `Generic_Compliance_Requirement_Attribute` | +| CSA CCM (legacy shape) | `CSA_CCM_Requirement_Attribute` | +| DISA STIG (Okta IDaaS) | `STIG_Requirement_Attribute` | +| Generic/Fallback (NIST, PCI, GDPR, HIPAA, SOC2, FedRAMP, ...) | `Generic_Compliance_Requirement_Attribute` | -## How Compliance Frameworks are Loaded +MITRE ATT&CK uses the separate `Mitre_Requirement` model with per-provider +`Mitre_Requirement_Attribute_{AWS,Azure,GCP}` attribute classes. -1. `Compliance.get_bulk(provider)` is called at startup -2. Scans `prowler/compliance/{provider}/` for `.json` files -3. Each file is parsed using `load_compliance_framework()` -4. Pydantic validates against `Compliance` model -5. Framework is stored in dictionary with filename (without `.json`) as key +`Compliance_Requirement_ConfigConstraint` models each `ConfigRequirements` / +`config_requirements` entry (`Check`, `ConfigKey`, `Operator`, `Value`, +optional `Provider`) with load-time operator/value type validation. + +## Universal Schema Models + +| Model | Purpose | +|-------|---------| +| `ComplianceFramework` | Top-level container (`framework`, `name`, `version`, `requirements`, `attributes_metadata`, `outputs`); validates attributes against metadata at load | +| `UniversalComplianceRequirement` | Flat `attributes: dict`, `checks: dict[provider, list]`, `config_requirements`, MITRE extras | +| `AttributeMetadata` | Per-attribute schema descriptor (key/label/type/enum/required/`enum_display`/`enum_order`/`output_formats`) | +| `OutputsConfig` → `TableConfig` | CLI table rendering (`group_by`, `split_by`, `scoring`, `labels`) — consumed by `universal_table.py` | +| `OutputsConfig` → `PDFConfig` (+ `ChartConfig`, `ScoringFormula`, `I18nLabels`, ...) | Declarative PDF config — modeled but **not yet consumed** by the API PDF pipeline (it uses its own `FRAMEWORK_REGISTRY`) | + +## How Frameworks Are Loaded + +Two entry points — they see different files: + +1. **Legacy**: `Compliance.get_bulk(provider)` scans only + `prowler/compliance/{provider}/` (exact provider-segment match) plus + external JSONs from the `prowler.compliance` entry-point group. Invalid + built-in file → `logger.critical` + `sys.exit(1)` + (`load_compliance_framework`, `fatal=True`). +2. **Universal**: `get_bulk_compliance_frameworks_universal(provider)` scans + the top-level `prowler/compliance/` **and** every provider subdirectory, + plus the `prowler.compliance.universal` entry-point group (built-ins win + collisions). Legacy files are adapted via `adapt_legacy_to_universal()` + (flattens `Attributes[0]` into a dict, wraps `Checks` as + `{provider: [...]}`, infers `attributes_metadata` from the matched Pydantic + class). Invalid file → logged and **skipped** + (`load_compliance_framework_universal` returns `None`). + +The framework key in both bulk dicts is the JSON basename without `.json` — +that's also the `--compliance` CLI key. ## How Checks Map to Compliance -1. After loading, `update_checks_metadata_with_compliance()` is called -2. For each check, it finds all compliance requirements that reference it -3. Compliance info is attached to `CheckMetadata.Compliance` list -4. During output, `get_check_compliance()` retrieves mappings per finding +1. `update_checks_metadata_with_compliance()` attaches, per check, every + framework requirement that references it (`CheckMetadata.Compliance`). +2. During output, `get_check_compliance()` + (`prowler/lib/outputs/compliance/compliance_check.py`) returns the + per-finding dict `{"{Framework}-{Version}": [requirement_ids]}` — the + `-{Version}` suffix only exists when `Version` is non-empty. +3. `ConfigRequirements` guardrails are evaluated by + `evaluate_config_constraints()` (`compliance_config_eval.py`); a violated + constraint forces FAIL and prepends + `Configuration not valid for this requirement.` to `status_extended` in + every output format. -## File Naming Convention +## File Naming Conventions ```text -{framework}_{version}_{provider}.json +prowler/compliance/{framework}_{version}.json # universal +prowler/compliance/{provider}/{framework}_{version}_{provider}.json # legacy ``` -Examples: -- `cis_5.0_aws.json` -- `iso27001_2022_azure.json` -- `mitre_attack_gcp.json` -- `ens_rd2022_aws.json` -- `nist_800_53_revision_5_aws.json` +Examples: `dora_2022_2554.json`, `cis_controls_8.1.json`, `cis_7.0_aws.json`, +`iso27001_2022_azure.json`, `okta_idaas_stig_v1r2_okta.json`, +`cisa_scuba_0.6_googleworkspace.json`, `ccc_aws.json` (unversioned only when +the framework has no versioning). For legacy files the version substring in +the filename must equal `Version`. -## Validation +## Validation Summary -Prowler validates compliance JSON at startup. Invalid files cause: -- `ValidationError` logged with details -- Application exit with error code +- **Load time (universal)**: `attributes_metadata` root validator — required + keys, unknown-key drift guard, enums, int/float/bool types. Omit the + metadata and nothing is validated. +- **Load time (legacy)**: Pydantic attribute-class matching; a shape matching + no specific class silently falls through to Generic. +- **Never validated at load**: check-id existence. Cross-check manually + (see SKILL.md → Validation). +- **Test suite**: `tests/lib/check/universal_compliance_models_test.py::test_loads_as_universal` + is parametrized over every shipped JSON (top-level + per-provider). +- **CI**: `.github/workflows/pr-check-compliance-mapping.yml` flags new checks + not mapped in any framework (`needs-compliance-review` label; opt out with + `no-compliance-check`). +- **Pre-commit**: `check-json` + `pretty-format-json` only (syntax/format, no + semantics). +- **Manual**: `skills/prowler-compliance-review/assets/validate_compliance.py` + (legacy schema only). -Common validation errors: -- Missing required fields (`Id`, `Description`, `Checks`, `Attributes`) -- Invalid enum values (e.g., `Profile` must be "Level 1" or "Level 2" for CIS) -- Type mismatches (e.g., `Checks` must be array of strings) +## Repo Tooling (`util/compliance/`) -## Adding a New Framework - -1. Create JSON file in `prowler/compliance/{provider}/` -2. Use appropriate attribute model (see table above) -3. Map existing checks to requirements via `Checks` array -4. Use empty `Checks: []` for manual-only requirements -5. Test with `prowler {provider} --list-compliance` to verify loading -6. Run `prowler {provider} --compliance {framework_name}` to test execution - -## Templates - -See `assets/` directory for example templates: -- `cis_framework.json` - CIS Benchmark template -- `iso27001_framework.json` - ISO 27001 template -- `ens_framework.json` - ENS (Spain) template -- `mitre_attack_framework.json` - MITRE ATT&CK template -- `prowler_threatscore_framework.json` - Prowler ThreatScore template -- `generic_framework.json` - Generic/custom framework template +| Tool | Purpose | +|------|---------| +| `util/compliance/generate_json_from_csv/*.py` | CSV→JSON generators (CIS 1.5, CIS 2.0 GCP, CIS 1.0 GitHub, CIS 4.0 M365, ENS, ThreatScore) | +| `util/compliance/ccc/from_yaml_to_json.py` | FINOS CCC YAML→JSON converter | +| `util/compliance/compliance_mapper/` | Compliance mapper (see its README) | +| `util/compliance/threatscore/get_prowler_threatscore_from_generic_output.py` | Derive ThreatScore from generic output | ## Prowler ThreatScore Details -Prowler ThreatScore is a custom security scoring framework that calculates an overall security posture score based on: +Custom Prowler scoring framework. Pillars / ID prefixes: `1.x.x` IAM, `2.x.x` +Attack Surface, `3.x.x` Logging and Monitoring, `4.x.x` Encryption. -### Four Pillars -1. **IAM (Identity and Access Management)** - - SubSections: Authentication, Authorization, Credentials Management - -2. **Attack Surface** - - SubSections: Network Exposure, Storage Exposure, Service Exposure - -3. **Logging and Monitoring** - - SubSections: Audit Logging, Threat Detection, Alerting - -4. **Encryption** - - SubSections: Data at Rest, Data in Transit - -### Scoring Algorithm -The ThreatScore uses `LevelOfRisk` and `Weight` to calculate severity: - -| LevelOfRisk | Weight | Example Controls | -|-------------|--------|------------------| -| 5 (Critical) | 1000 | Root MFA, No root access keys, Public S3 buckets | -| 4 (High) | 100 | User MFA, Public EC2, GuardDuty enabled | -| 3 (Medium) | 10 | Password policies, EBS encryption, CloudTrail | -| 2 (Low) | 1-10 | Best practice recommendations | -| 1 (Info) | 1 | Informational controls | - -### ID Numbering Convention -- `1.x.x` - IAM controls -- `2.x.x` - Attack Surface controls -- `3.x.x` - Logging and Monitoring controls -- `4.x.x` - Encryption controls +Scoring: `LevelOfRisk` 1–5 (5=critical) × `Weight` (values in the shipped +catalogs: 1000 critical / 100 high / 8–10 standard / 1 low). Available for +aws, azure, gcp, kubernetes, m365, alibabacloud. ## External Resources -### Official Framework Documentation - [CIS Benchmarks](https://www.cisecurity.org/cis-benchmarks) -- [ISO 27001:2022](https://www.iso.org/standard/27001) +- [CIS Critical Security Controls](https://www.cisecurity.org/controls) +- [ISO 27001](https://www.iso.org/standard/27001) - [NIST 800-53](https://csrc.nist.gov/publications/detail/sp/800-53/rev-5/final) - [NIST CSF](https://www.nist.gov/cyberframework) - [PCI DSS](https://www.pcisecuritystandards.org/) - [MITRE ATT&CK](https://attack.mitre.org/) - [ENS (Spain)](https://www.ccn-cert.cni.es/es/ens.html) - -### Prowler Documentation -- [Prowler Docs - Compliance](https://docs.prowler.com/projects/prowler-open-source/en/latest/) -- [Prowler GitHub](https://github.com/prowler-cloud/prowler) +- [FINOS CCC](https://github.com/finos/common-cloud-controls) +- [CSA CCM](https://cloudsecurityalliance.org/research/cloud-controls-matrix) +- [DORA (EU 2022/2554)](https://eur-lex.europa.eu/eli/reg/2022/2554/oj) +- [ASD Essential Eight](https://www.cyber.gov.au/resources-business-and-government/essential-cybersecurity/essential-eight) +- [CISA SCuBA](https://www.cisa.gov/resources-tools/services/secure-cloud-business-applications-scuba-project) +- [DISA STIGs](https://public.cyber.mil/stigs/) +- [Prowler Docs — Compliance developer guide](https://docs.prowler.com/developer-guide/security-compliance-framework) diff --git a/skills/prowler-mcp/SKILL.md b/skills/prowler-mcp/SKILL.md index af3c597771..704acf8553 100644 --- a/skills/prowler-mcp/SKILL.md +++ b/skills/prowler-mcp/SKILL.md @@ -19,7 +19,7 @@ The Prowler MCP Server uses three sub-servers with prefixed namespacing: | Sub-Server | Prefix | Auth | Purpose | |------------|--------|------|---------| -| Prowler App | `prowler_app_*` | Required | Cloud management tools | +| Prowler | `prowler_*` | Required | Prowler Cloud, Private Cloud & Local Server management tools | | Prowler Hub | `prowler_hub_*` | No | Security checks catalog | | Prowler Docs | `prowler_docs_*` | No | Documentation search | @@ -27,7 +27,7 @@ For complete architecture, patterns, and examples, see [docs/developer-guide/mcp --- -## Critical Rules (Prowler App Only) +## Critical Rules (Prowler Tools Only) ### Tool Implementation @@ -56,7 +56,7 @@ Use `@mcp.tool()` decorator directly—no BaseTool or models required. --- -## Quick Reference: New Prowler App Tool +## Quick Reference: New Prowler Tool 1. Create tool class in `prowler_app/tools/` extending `BaseTool` 2. Create models in `prowler_app/models/` using `MinimalSerializerMixin` @@ -64,7 +64,7 @@ Use `@mcp.tool()` decorator directly—no BaseTool or models required. --- -## QA Checklist (Prowler App) +## QA Checklist (Prowler Tools) - [ ] Tool docstrings describe LLM-relevant behavior - [ ] Models use `MinimalSerializerMixin` diff --git a/tests/config/schema/exclusions_test.py b/tests/config/schema/exclusions_test.py new file mode 100644 index 0000000000..a13ed2f786 --- /dev/null +++ b/tests/config/schema/exclusions_test.py @@ -0,0 +1,113 @@ +"""Coverage for the ``excluded_checks`` / ``excluded_services`` fields +added to :class:`prowler.config.schema.base.ProviderConfigBase`. + +Because the fields live on the base class, every registered provider +schema exposes them and every provider must therefore share the same +whitespace / uniqueness / non-empty guarantees. These tests lock in that +contract at the base level and at the JSON-Schema level (which the UI +editor consumes via ``ajv``). +""" + +import pytest +from pydantic import ValidationError + +from prowler.config.scan_config_schema import SCAN_CONFIG_SCHEMA +from prowler.config.schema.aws import AWSProviderConfig +from prowler.config.schema.registry import SCHEMAS +from prowler.config.schema.validator import validate_provider_config + +EXCLUSION_FIELDS = ("excluded_checks", "excluded_services") + + +class Test_JSON_Schema_Exposes_Exclusion_Fields: + @pytest.mark.parametrize("provider", sorted(SCHEMAS)) + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_field_shape(self, provider, field): + field_schema = SCAN_CONFIG_SCHEMA["properties"][provider]["properties"][field] + assert field_schema["type"] == "array" + assert field_schema["items"] == {"type": "string", "minLength": 1} + assert field_schema["uniqueItems"] is True + assert field_schema["default"] == [] + + +class Test_Exclusion_Field_Validation: + def _model(self, **kwargs): + return AWSProviderConfig.model_validate(kwargs) + + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_empty_string_is_rejected(self, field): + with pytest.raises(ValidationError): + self._model(**{field: [""]}) + + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_whitespace_only_string_is_rejected(self, field): + with pytest.raises(ValidationError): + self._model(**{field: [" "]}) + + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_raw_duplicates_are_rejected(self, field): + with pytest.raises(ValidationError): + self._model(**{field: ["s3", "s3"]}) + + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_normalized_duplicates_are_rejected(self, field): + # After whitespace normalization ``" s3 "`` collapses to ``"s3"`` + # and must be caught by the duplicate check. + with pytest.raises(ValidationError): + self._model(**{field: ["s3", " s3 "]}) + + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_whitespace_is_stripped(self, field): + model = self._model(**{field: [" identifier "]}) + assert getattr(model, field) == ["identifier"] + + @pytest.mark.parametrize("field", EXCLUSION_FIELDS) + def test_non_string_item_is_rejected(self, field): + with pytest.raises(ValidationError): + self._model(**{field: [123]}) + + +class Test_Exclusion_Defaults_Are_Not_Injected: + """The strict normalization path uses ``model_dump(exclude_unset=True)`` + so pre-existing configs that never set an exclusion field must round-trip + without the default empty list being materialized.""" + + def test_absent_fields_are_not_injected_by_validator(self): + # The lenient SDK runtime path also uses ``exclude_unset=True`` under + # the hood; asserting on the validator output guards the promise + # against future refactors. + assert validate_provider_config("aws", {}, SCHEMAS["aws"]) == {} + + def test_absent_fields_are_not_injected_when_other_keys_are_present(self): + assert validate_provider_config( + "aws", + {"max_ec2_instance_age_in_days": 180}, + SCHEMAS["aws"], + ) == {"max_ec2_instance_age_in_days": 180} + + def test_explicit_empty_list_round_trips(self): + # Explicitly setting ``excluded_checks: []`` is different from + # omitting it — the empty list is user-provided and must be + # preserved by the strict-normalization contract. + assert validate_provider_config( + "aws", + {"excluded_checks": []}, + SCHEMAS["aws"], + ) == {"excluded_checks": []} + + +class Test_Extra_Fields_Are_Preserved: + """``extra="allow"`` must keep plugin-provided keys around so the + ecosystem contract in ``validator_test.py`` still holds after adding + the exclusion fields.""" + + def test_unknown_keys_are_preserved_alongside_exclusions(self): + out = validate_provider_config( + "aws", + {"excluded_checks": ["s3_bucket_public_access"], "plugin_option": "kept"}, + SCHEMAS["aws"], + ) + assert out == { + "excluded_checks": ["s3_bucket_public_access"], + "plugin_option": "kept", + } diff --git a/tests/config/schema/scan_config_schema_test.py b/tests/config/schema/scan_config_schema_test.py new file mode 100644 index 0000000000..037fcd9501 --- /dev/null +++ b/tests/config/schema/scan_config_schema_test.py @@ -0,0 +1,393 @@ +"""Coverage for the strict scan-config validation and normalization +contract exposed to the Prowler App backend. + +Split from :mod:`tests.config.schema.validator_test` because the strict +API (``validate_and_normalize_scan_config``) has different guarantees: +it never silently drops keys, and it returns a JSON-serializable payload +the backend can persist verbatim in a Django ``JSONField``. +""" + +import json +from unittest.mock import call, patch + +import pytest + +from prowler.config.scan_config_schema import ( + _get_provider_check_ids, + _get_provider_services, + validate_and_normalize_scan_config, + validate_scan_config, +) + + +@pytest.fixture(autouse=True) +def clear_provider_catalog_caches(): + """Keep provider catalog cache state isolated between tests.""" + _get_provider_check_ids.cache_clear() + _get_provider_services.cache_clear() + yield + _get_provider_check_ids.cache_clear() + _get_provider_services.cache_clear() + + +class Test_Non_Dict_Root: + @pytest.mark.parametrize("payload", [None, "string", 42, [], (1, 2)]) + def test_non_mapping_root_is_rejected(self, payload): + normalized, errors = validate_and_normalize_scan_config(payload) + assert normalized == {} + assert len(errors) == 1 + assert errors[0]["path"] == "" + + +class Test_Registered_Provider_Section_Must_Be_Mapping: + @pytest.mark.parametrize("section", ["a-string", 42, ["s3"], None]) + def test_non_mapping_section_reports_provider_path(self, section): + normalized, errors = validate_and_normalize_scan_config({"aws": section}) + assert normalized == {} + assert errors == [{"path": "aws", "message": "section must be a mapping."}] + + +class Test_Success_Path: + def test_whitespace_is_normalized_in_exclusions(self): + normalized, errors = validate_and_normalize_scan_config( + { + "aws": { + "excluded_checks": [" s3_bucket_default_encryption "], + "excluded_services": [" s3 "], + } + } + ) + assert errors == [] + assert normalized == { + "aws": { + "excluded_checks": ["s3_bucket_default_encryption"], + "excluded_services": ["s3"], + } + } + + def test_plugin_options_are_preserved(self): + # Third-party plugins inject arbitrary keys inside a provider + # section; ``extra="allow"`` on the schema keeps them alive + # through the dump/normalize round-trip. + normalized, errors = validate_and_normalize_scan_config( + {"aws": {"plugin_option": "preserved", "another": 42}} + ) + assert errors == [] + assert normalized == {"aws": {"plugin_option": "preserved", "another": 42}} + + def test_plugin_catalog_identifiers_are_accepted_and_catalogs_are_cached(self): + payload = { + "aws": { + "excluded_checks": ["plugin_check"], + "excluded_services": ["plugin_service"], + } + } + with ( + patch( + "prowler.config.scan_config_schema.CheckMetadata.get_bulk", + return_value={"plugin_check": object()}, + ) as check_catalog, + patch( + "prowler.config.scan_config_schema.list_services", + return_value=["plugin_service"], + ) as service_catalog, + ): + first_result = validate_and_normalize_scan_config(payload) + second_result = validate_and_normalize_scan_config(payload) + + normalized, errors = first_result + assert errors == [] + assert normalized == { + "aws": { + "excluded_checks": ["plugin_check"], + "excluded_services": ["plugin_service"], + } + } + assert second_result == first_result + check_catalog.assert_called_once_with("aws") + service_catalog.assert_called_once_with("aws") + + def test_catalog_caches_are_keyed_by_provider(self): + with ( + patch( + "prowler.config.scan_config_schema.CheckMetadata.get_bulk", + side_effect=lambda provider: {f"{provider}_plugin_check": object()}, + ) as check_catalog, + patch( + "prowler.config.scan_config_schema.list_services", + side_effect=lambda provider: [f"{provider}_plugin_service"], + ) as service_catalog, + ): + payload = { + "aws": { + "excluded_checks": ["aws_plugin_check"], + "excluded_services": ["aws_plugin_service"], + }, + "azure": { + "excluded_checks": ["azure_plugin_check"], + "excluded_services": ["azure_plugin_service"], + }, + } + first_result = validate_and_normalize_scan_config(payload) + second_result = validate_and_normalize_scan_config(payload) + + assert first_result[1] == [] + assert second_result == first_result + assert check_catalog.call_args_list == [call("aws"), call("azure")] + assert service_catalog.call_args_list == [call("aws"), call("azure")] + + def test_omitted_defaults_are_not_injected(self): + normalized, errors = validate_and_normalize_scan_config( + {"aws": {"max_ec2_instance_age_in_days": 90}} + ) + assert errors == [] + assert normalized == {"aws": {"max_ec2_instance_age_in_days": 90}} + assert "excluded_checks" not in normalized["aws"] + assert "excluded_services" not in normalized["aws"] + + def test_unknown_provider_sections_are_preserved_verbatim(self): + payload = {"future_provider": {"custom_option": True, "nested": {"k": 1}}} + normalized, errors = validate_and_normalize_scan_config(payload) + assert errors == [] + assert normalized == payload + + def test_normalized_payload_is_json_serializable(self): + normalized, _ = validate_and_normalize_scan_config( + { + "aws": { + "excluded_checks": ["s3_bucket_public_access"], + "excluded_services": ["s3"], + } + } + ) + # If ``model_dump(mode="json", ...)`` is ever dropped this + # ``json.dumps`` call is what will notice. + json.dumps(normalized) + + def test_input_payload_is_not_mutated(self): + payload = { + "aws": { + "excluded_checks": [" s3_bucket_public_access "], + "excluded_services": [" s3 "], + } + } + snapshot = json.loads(json.dumps(payload)) + validate_and_normalize_scan_config(payload) + assert payload == snapshot + + +class Test_Error_Path: + def test_unknown_excluded_check_is_rejected(self): + normalized, errors = validate_and_normalize_scan_config( + {"aws": {"excluded_checks": ["aws_check_that_does_not_exist"]}} + ) + assert normalized == {} + assert errors == [ + { + "path": "aws.excluded_checks[0]", + "message": ( + "Unknown check 'aws_check_that_does_not_exist' for provider " + "'aws'." + ), + } + ] + + def test_unknown_excluded_service_is_rejected(self): + normalized, errors = validate_and_normalize_scan_config( + {"aws": {"excluded_services": ["not_a_real_aws_service"]}} + ) + assert normalized == {} + assert errors == [ + { + "path": "aws.excluded_services[0]", + "message": ( + "Unknown service 'not_a_real_aws_service' for provider 'aws'." + ), + } + ] + + def test_multiple_unknown_exclusions_return_deterministic_errors(self): + normalized, errors = validate_and_normalize_scan_config( + { + "aws": { + "excluded_checks": [ + "unknown_check_one", + "s3_bucket_default_encryption", + "unknown_check_two", + ], + "excluded_services": [ + "unknown_service_one", + "s3", + "unknown_service_two", + ], + } + } + ) + assert normalized == {} + assert errors == [ + { + "path": "aws.excluded_checks[0]", + "message": "Unknown check 'unknown_check_one' for provider 'aws'.", + }, + { + "path": "aws.excluded_checks[2]", + "message": "Unknown check 'unknown_check_two' for provider 'aws'.", + }, + { + "path": "aws.excluded_services[0]", + "message": ( + "Unknown service 'unknown_service_one' for provider 'aws'." + ), + }, + { + "path": "aws.excluded_services[2]", + "message": ( + "Unknown service 'unknown_service_two' for provider 'aws'." + ), + }, + ] + + def test_check_from_another_provider_is_rejected(self): + azure_check = "postgresql_flexible_server_allow_access_services_disabled" + normalized, errors = validate_and_normalize_scan_config( + {"aws": {"excluded_checks": [azure_check]}} + ) + assert normalized == {} + assert errors == [ + { + "path": "aws.excluded_checks[0]", + "message": f"Unknown check '{azure_check}' for provider 'aws'.", + } + ] + + def test_invalid_input_returns_empty_normalized_and_errors(self): + normalized, errors = validate_and_normalize_scan_config( + {"aws": {"excluded_services": ["s3", " s3 "]}} + ) + assert normalized == {} + assert errors + assert any(err["path"].startswith("aws.excluded_services") for err in errors) + + def test_partial_error_zeros_the_normalized_payload(self): + # One valid provider + one invalid provider must not leak the + # valid section into a partially normalized result. + normalized, errors = validate_and_normalize_scan_config( + { + "aws": {"excluded_services": ["s3", "s3"]}, + "azure": {"vm_backup_min_daily_retention_days": 7}, + } + ) + assert normalized == {} + assert errors + assert any(err["path"].startswith("aws.") for err in errors) + + def test_value_error_prefix_is_stripped_from_user_facing_messages(self): + # Pydantic prefixes messages emitted from ``field_validator`` + # ValueError with ``"Value error, "``. If this test starts to fail + # because the prefix reappears, either pydantic changed the format + # or the strip in ``validate_and_normalize_scan_config`` was + # dropped — either way the UI would render the noisy prefix, so + # we lock the cleaned message in explicitly. + _, errors = validate_and_normalize_scan_config( + {"aws": {"excluded_services": ["s3", "s3"]}} + ) + assert errors + message = errors[0]["message"] + assert not message.startswith("Value error, ") + assert "duplicate values are not allowed" in message + + def test_all_errors_are_reported_not_only_the_first(self): + normalized, errors = validate_and_normalize_scan_config( + { + "aws": { + "excluded_checks": ["", ""], + "excluded_services": ["", ""], + } + } + ) + assert normalized == {} + # ``excluded_checks`` yields per-item empty-string errors AND a + # duplicate error; ``excluded_services`` yields the same set. + paths = {err["path"] for err in errors} + assert any(p.startswith("aws.excluded_checks") for p in paths) + assert any(p.startswith("aws.excluded_services") for p in paths) + + +class Test_Non_String_Provider_Keys: + """The normalized payload is later persisted in a Django JSONField + keyed by provider. Two entries whose ``str()`` collide (e.g. ``123`` + and ``"123"``) would silently overwrite each other, so non-string + keys are rejected up front instead of silently coerced.""" + + def test_non_string_key_is_rejected(self): + normalized, errors = validate_and_normalize_scan_config({123: {}}) + assert normalized == {} + assert errors == [{"path": "123", "message": "provider keys must be strings."}] + + def test_string_and_int_collision_does_not_silently_overwrite(self): + # If only ``str()`` coercion happened both keys would collapse to + # ``"aws"`` in the output — this test guards against that regression. + normalized, errors = validate_and_normalize_scan_config( + {"aws": {}, 123: {"a": 1}} + ) + assert normalized == {} + assert any(err["path"] == "123" for err in errors) + + +class Test_Unknown_Sections_Must_Be_JSON_Serializable: + """``normalized`` is persisted by the API in a Django JSONField, so + unknown provider sections must fail fast here instead of blowing up + at persist time. Registered sections cannot hit this path — they go + through ``model_dump(mode="json", ...)`` which already coerces.""" + + def test_set_inside_unknown_section_is_rejected(self): + # ``set`` is a common trap: ``yaml.safe_load`` never produces it, + # but a hand-built dict might. + normalized, errors = validate_and_normalize_scan_config( + {"future_provider": {"values": {1, 2, 3}}} + ) + assert normalized == {} + assert errors + assert errors[0]["path"] == "future_provider" + assert "JSON-serializable" in errors[0]["message"] + + def test_json_safe_unknown_section_is_still_preserved(self): + payload = {"future_provider": {"nested": {"k": [1, 2, 3]}}} + normalized, errors = validate_and_normalize_scan_config(payload) + assert errors == [] + assert normalized == payload + + +class Test_Backward_Compatible_Wrapper: + def test_valid_payload_yields_no_errors(self): + assert ( + validate_scan_config( + {"aws": {"excluded_checks": ["s3_bucket_public_access"]}} + ) + == [] + ) + + def test_invalid_payload_yields_only_the_errors(self): + errors = validate_scan_config({"aws": {"excluded_checks": ["", ""]}}) + assert errors + assert all(set(err) == {"path", "message"} for err in errors) + + def test_unknown_exclusion_yields_the_semantic_error(self): + assert validate_scan_config( + {"aws": {"excluded_services": ["not_a_real_aws_service"]}} + ) == [ + { + "path": "aws.excluded_services[0]", + "message": ( + "Unknown service 'not_a_real_aws_service' for provider 'aws'." + ), + } + ] + + def test_non_mapping_root_matches_new_contract(self): + assert validate_scan_config(None) == [ + { + "path": "", + "message": "Scan config must be a mapping with provider sections.", + } + ] diff --git a/tests/lib/outputs/jira/jira_test.py b/tests/lib/outputs/jira/jira_test.py index de4c901fc3..f38c1ba376 100644 --- a/tests/lib/outputs/jira/jira_test.py +++ b/tests/lib/outputs/jira/jira_test.py @@ -98,6 +98,41 @@ class TestJiraIntegration: return found return None + @staticmethod + def _find_link_mark_by_href(nodes: List[dict], href: str) -> Optional[dict]: + for node in nodes: + if node.get("type") == "text": + for mark in node.get("marks", []): + if ( + mark.get("type") == "link" + and mark.get("attrs", {}).get("href") == href + ): + return mark + found = TestJiraIntegration._find_link_mark_by_href( + node.get("content", []), href + ) + if found: + return found + return None + + @staticmethod + def _collect_link_texts_by_href(nodes: List[dict], href: str) -> List[str]: + link_texts: List[str] = [] + + for node in nodes: + if node.get("type") == "text" and any( + mark.get("type") == "link" and mark.get("attrs", {}).get("href") == href + for mark in node.get("marks", []) + ): + link_texts.append(node.get("text", "")) + link_texts.extend( + TestJiraIntegration._collect_link_texts_by_href( + node.get("content", []), href + ) + ) + + return link_texts + @staticmethod def _find_table_row(rows: List[dict], header: str) -> dict: for row in rows: @@ -918,6 +953,12 @@ class TestJiraIntegration: intro_text = intro_paragraph["content"][0] assert intro_text["type"] == "text" assert intro_text["text"] == "Prowler has discovered the following finding:" + assert all( + self._collect_text_from_cell({"content": node.get("content", [])}) + != "Summary" + for node in description_content + if node.get("type") == "heading" + ) table = description_content[1] assert table["type"] == "table" @@ -1201,6 +1242,218 @@ class TestJiraIntegration: value_cell = row["content"][1] assert self._collect_text_from_cell(value_cell) == "-" + def test_get_grouped_adf_description_uses_capped_finding_group_link_copy(self): + finding_group_url = ( + "https://security.example.com/findings?" + "filter%5Bcheck_id%5D=admincenter_users_admins_reduced_license_footprint&" + "expandedCheckId=admincenter_users_admins_reduced_license_footprint" + ) + finding_group_link_text = "View the remaining grouped findings." + recommendation_url = ( + "https://hub.prowler.com/check/" + "admincenter_users_admins_reduced_license_footprint" + ) + adf_description = self.jira_integration.get_grouped_adf_description( + check_id="admincenter_users_admins_reduced_license_footprint", + check_title="Administrative user has no license or an allowed license", + check_description="Administrative users are assigned productivity licenses.", + severity="HIGH", + status="FAIL", + provider="m365", + service="exchange", + affected_failing_resources=123, + last_seen="Jul 09, 2026 11:38AM UTC", + failing_for="< 1 day", + grouped_resources=[ + { + "resource_name": "rich@prowler.com", + "resource_uid": "3f9a216b-b66b-4d5d-a812-2ad538732cfb", + "provider": "m365", + "service": "exchange", + "provider_account": "ProwlerPro.onmicrosoft.com", + "status": "FAIL", + "severity": "high", + "region": "global", + "last_seen": "Jul 09, 2026 11:38AM UTC", + "failing_for": "< 1 day", + "triage": "Open", + } + ], + resources_total=123, + resources_shown=100, + finding_group_url=finding_group_url, + finding_group_link_text=finding_group_link_text, + risk="Productivity licenses on privileged identities create risk.", + recommendation_text="Maintain dedicated admin accounts.", + recommendation_url=recommendation_url, + ) + + assert adf_description["type"] == "doc" + assert self._find_empty_text_nodes(adf_description) == [] + + main_table = adf_description["content"][1] + main_rows = {} + for row in main_table["content"]: + key_cell, value_cell = row["content"] + main_rows[self._collect_text_from_cell(key_cell)] = ( + self._collect_text_from_cell(value_cell) + ) + + assert ( + main_rows["Check Id"] + == "admincenter_users_admins_reduced_license_footprint" + ) + assert main_rows["Service"] == "exchange" + assert main_rows["Affected Failing Resources"] == "123" + assert ( + main_rows["Risk"] + == "Productivity licenses on privileged identities create risk." + ) + assert main_rows["Recommendation"] == ( + "Maintain dedicated admin accounts. " + recommendation_url + ) + assert "Finding Group Link" not in main_rows + assert "Region" not in main_rows + + top_level_headings = [ + self._collect_text_from_cell({"content": node.get("content", [])}) + for node in adf_description["content"] + if node.get("type") == "heading" + ] + assert "Risk" not in top_level_headings + assert "Recommendation" not in top_level_headings + assert "Summary" not in top_level_headings + + def text_marks(cell: dict) -> list[dict]: + return cell["content"][0]["content"][0]["marks"] + + severity_marks = text_marks( + self._find_table_row(main_table["content"], "Severity")["content"][1] + ) + status_marks = text_marks( + self._find_table_row(main_table["content"], "Status")["content"][1] + ) + assert { + "type": "backgroundColor", + "attrs": {"color": "#FFA500"}, + } in severity_marks + assert {"type": "textColor", "attrs": {"color": "#FF0000"}} in status_marks + + resource_table = next( + node + for node in adf_description["content"] + if node.get("type") == "table" + and self._collect_text_from_cell(node["content"][0]["content"][0]) + == "Resource" + ) + resource_cells = resource_table["content"][1]["content"] + assert {"type": "textColor", "attrs": {"color": "#FF0000"}} in text_marks( + resource_cells[5] + ) + assert { + "type": "backgroundColor", + "attrs": {"color": "#FFA500"}, + } in text_marks(resource_cells[6]) + + document_text = self._collect_text_from_cell( + {"content": adf_description["content"]} + ) + assert ( + "Administrative users are assigned productivity licenses." + not in document_text + ) + assert "Affected failing resources" in document_text + capped_link_copy = ( + f"Showing 100 of 123 Findings in this Jira issue. {finding_group_link_text}" + ) + assert document_text.count(capped_link_copy) == 1 + assert "Finding Group Link" not in document_text + assert recommendation_url in document_text + recommendation_link_mark = self._find_link_mark_by_href( + adf_description["content"], recommendation_url + ) + assert recommendation_link_mark is not None + link_mark = self._find_link_mark_by_href( + adf_description["content"], finding_group_url + ) + assert link_mark is not None + assert link_mark["attrs"]["href"] == finding_group_url + assert self._collect_link_texts_by_href( + adf_description["content"], finding_group_url + ) == [finding_group_link_text] + assert ( + len( + self._collect_link_texts_by_href( + adf_description["content"], finding_group_url + ) + ) + == 1 + ) + assert "filter%5Bcheck_id%5D=" in link_mark["attrs"]["href"] + assert "expandedCheckId=" in link_mark["attrs"]["href"] + + def test_get_grouped_adf_description_includes_link_when_not_capped(self): + finding_group_url = ( + "https://security.example.com/findings?" + "filter%5Bcheck_id%5D=s3_bucket_public_access&" + "expandedCheckId=s3_bucket_public_access" + ) + finding_group_link_text = "View this grouped finding." + adf_description = self.jira_integration.get_grouped_adf_description( + check_id="s3_bucket_public_access", + check_title="S3 bucket public access", + severity="HIGH", + status="FAIL", + provider="aws", + service="s3", + affected_failing_resources=1, + grouped_resources=[ + { + "resource_name": "bucket-a", + "resource_uid": "arn:aws:s3:::bucket-a", + "provider": "aws", + "service": "s3", + "provider_account": "production (123456789012)", + "status": "FAIL", + "severity": "high", + "region": "us-east-1", + "last_seen": "Jul 09, 2026 11:38AM UTC", + "failing_for": "< 1 day", + "triage": "Open", + } + ], + resources_total=1, + resources_shown=1, + finding_group_url=finding_group_url, + finding_group_link_text=finding_group_link_text, + ) + + document_text = self._collect_text_from_cell( + {"content": adf_description["content"]} + ) + assert "Showing 1 of 1 Findings." not in document_text + assert "remaining Findings" not in document_text + assert document_text.count(finding_group_link_text) == 1 + assert "Finding Group Link" not in document_text + main_table = adf_description["content"][1] + main_row_headers = [ + self._collect_text_from_cell(row["content"][0]) + for row in main_table["content"] + ] + assert "Finding Group Link" not in main_row_headers + link_mark = self._find_link_mark_by_href( + adf_description["content"], finding_group_url + ) + assert link_mark is not None + assert link_mark["attrs"]["href"] == finding_group_url + assert self._collect_link_texts_by_href( + adf_description["content"], finding_group_url + ) == [finding_group_link_text] + assert ( + "filter%5Bcheck_id%5D=s3_bucket_public_access" in link_mark["attrs"]["href"] + ) + assert "expandedCheckId=s3_bucket_public_access" in link_mark["attrs"]["href"] + @patch.object(Jira, "get_access_token", return_value="valid_access_token") @patch.object( Jira, "get_available_issue_types", return_value=["Bug", "Task", "Story"] @@ -1709,6 +1962,54 @@ class TestJiraIntegration: assert result is True mock_post.assert_called_once() + @patch.object(Jira, "get_access_token", return_value="valid_access_token") + @patch.object( + Jira, "cloud_id", new_callable=PropertyMock, return_value="test_cloud_id" + ) + @patch.object(Jira, "get_projects", return_value={"TEST": {"name": "Test Project"}}) + @patch.object(Jira, "get_available_issue_types", return_value=["Bug"]) + @patch("prowler.lib.outputs.jira.jira.requests.post") + def test_send_finding_sanitizes_summary_control_characters( + self, + mock_post, + mock_get_issue_types, + mock_get_projects, + mock_cloud_id, + mock_get_access_token, + ): + """Test that Jira summary is sent as one line.""" + # To disable vulture + mock_cloud_id = mock_cloud_id + mock_get_access_token = mock_get_access_token + mock_get_projects = mock_get_projects + mock_get_issue_types = mock_get_issue_types + + mock_response = MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = {"id": "ISSUE-123", "key": "TEST-123"} + mock_post.return_value = mock_response + long_check_id = "check\nwith\rcontrol\tcharacters " + "x" * 260 + + result = self.jira_integration.send_finding( + check_id=long_check_id, + check_title="Test Finding", + severity="High\n", + status="FAIL", + project_key="TEST", + issue_type="Bug", + affected_failing_resources=2, + grouped_resources=[], + ) + + assert result is True + payload = mock_post.call_args.kwargs["json"] + expected_summary = ( + f"[Prowler] HIGH - {' '.join(long_check_id.split())} - " + "2 affected failing resources" + )[:255] + assert payload["fields"]["summary"] == expected_summary + assert len(payload["fields"]["summary"]) == 255 + @patch.object(Jira, "get_access_token", return_value="valid_access_token") @patch.object( Jira, "cloud_id", new_callable=PropertyMock, return_value="test_cloud_id" diff --git a/tests/lib/scan/scan_exclusions_test.py b/tests/lib/scan/scan_exclusions_test.py new file mode 100644 index 0000000000..2c8f260c20 --- /dev/null +++ b/tests/lib/scan/scan_exclusions_test.py @@ -0,0 +1,178 @@ +"""Coverage for ``Scan`` constructor exclusion semantics. + +The Scan class is the single execution entry point used by both the CLI +and the API worker. Its exclusion validation must: + +- Reject duplicates and unknown identifiers with actionable errors. +- Validate excluded checks against the FULL provider catalog so a global + configuration can exclude a valid check that is not part of a scoped + run (see the SDK acceptance criteria for scan-configuration exclusions). +- Refuse a configuration that would leave nothing to execute. +- Produce a deterministic, sorted final scope. + +The catalog dependencies (``CheckMetadata.get_bulk``, ``Compliance.get_bulk``, +``list_services``, ``load_checks_to_execute``) are patched so tests stay +focused on the exclusion logic and avoid walking the provider package tree. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from prowler.lib.scan.exceptions.exceptions import ( + ScanInvalidCheckError, + ScanInvalidServiceError, +) +from prowler.lib.scan.scan import Scan +from tests.providers.aws.utils import set_mocked_aws_provider + +# The provider catalog for these tests: three checks spread across two +# services (``accessanalyzer`` and ``s3``). Keeps assertions readable. +PROVIDER_CATALOG = { + "accessanalyzer_enabled", + "s3_bucket_encryption_enabled", + "s3_bucket_public_access", +} +PROVIDER_SERVICES = ["accessanalyzer", "s3"] + + +@pytest.fixture +def scan_provider(): + provider = set_mocked_aws_provider() + metadata = MagicMock() + metadata.Categories = [] + bulk = {check: metadata for check in PROVIDER_CATALOG} + + with ( + patch( + "prowler.lib.scan.scan.CheckMetadata.get_bulk", + return_value=bulk, + ), + patch("prowler.lib.scan.scan.Compliance.get_bulk", return_value={}), + patch( + "prowler.lib.scan.scan.update_checks_metadata_with_compliance", + side_effect=lambda _compliance, checks: checks, + ), + patch( + "prowler.lib.scan.scan.load_checks_to_execute", + side_effect=lambda **kwargs: set(kwargs["check_list"] or PROVIDER_CATALOG), + ), + patch( + "prowler.lib.scan.scan.list_services", + return_value=PROVIDER_SERVICES, + ), + ): + yield provider + + +class Test_Exclusion_No_Ops: + def test_none_lists_are_no_ops(self, scan_provider): + scan = Scan(scan_provider, excluded_checks=None, excluded_services=None) + assert scan.checks_to_execute == sorted(PROVIDER_CATALOG) + + def test_empty_lists_are_no_ops(self, scan_provider): + scan = Scan(scan_provider, excluded_checks=[], excluded_services=[]) + assert scan.checks_to_execute == sorted(PROVIDER_CATALOG) + + +class Test_Excluded_Checks: + def test_valid_check_is_removed_from_the_scope(self, scan_provider): + scan = Scan( + scan_provider, + excluded_checks=["s3_bucket_public_access"], + ) + assert scan.checks_to_execute == sorted( + PROVIDER_CATALOG - {"s3_bucket_public_access"} + ) + + def test_excluded_check_may_be_outside_the_selected_scope(self, scan_provider): + # ``s3_bucket_public_access`` is not in the explicitly selected + # ``checks`` list but is still a valid provider check, so the + # global exclusion must be accepted and be a no-op for this run. + scan = Scan( + scan_provider, + checks=["accessanalyzer_enabled"], + excluded_checks=["s3_bucket_public_access"], + ) + assert scan.checks_to_execute == ["accessanalyzer_enabled"] + + def test_unknown_check_is_rejected(self, scan_provider): + with pytest.raises(ScanInvalidCheckError): + Scan(scan_provider, excluded_checks=["not_a_real_check"]) + + def test_duplicate_checks_are_rejected(self, scan_provider): + with pytest.raises(ScanInvalidCheckError): + Scan( + scan_provider, + excluded_checks=[ + "s3_bucket_public_access", + "s3_bucket_public_access", + ], + ) + + +class Test_Excluded_Services: + def test_service_exclusion_removes_every_check_in_the_service(self, scan_provider): + scan = Scan(scan_provider, excluded_services=["s3"]) + assert scan.checks_to_execute == ["accessanalyzer_enabled"] + + def test_unknown_service_is_rejected(self, scan_provider): + with pytest.raises(ScanInvalidServiceError): + Scan(scan_provider, excluded_services=["not_a_real_service"]) + + def test_duplicate_services_are_rejected(self, scan_provider): + with pytest.raises(ScanInvalidServiceError): + Scan(scan_provider, excluded_services=["s3", "s3"]) + + +class Test_Combined_Exclusions: + def test_selected_checks_plus_excluded_checks_and_services(self, scan_provider): + scan = Scan( + scan_provider, + checks=["accessanalyzer_enabled", "s3_bucket_encryption_enabled"], + excluded_checks=["s3_bucket_public_access"], + excluded_services=["s3"], + ) + # The explicit ``checks`` selection is narrowed by both the + # excluded_checks (drops nothing extra here) and excluded_services + # (drops every s3 check), leaving accessanalyzer alone. + assert scan.checks_to_execute == ["accessanalyzer_enabled"] + + def test_result_is_sorted_and_deterministic(self, scan_provider): + scan = Scan( + scan_provider, + excluded_checks=["s3_bucket_public_access"], + ) + assert scan.checks_to_execute == sorted(scan.checks_to_execute) + + +class Test_Empty_Final_Scope_Is_Rejected: + def test_excluding_every_service_is_rejected(self, scan_provider): + with pytest.raises(ScanInvalidCheckError): + Scan(scan_provider, excluded_services=PROVIDER_SERVICES) + + def test_excluding_every_check_is_rejected(self, scan_provider): + with pytest.raises(ScanInvalidCheckError): + Scan(scan_provider, excluded_checks=sorted(PROVIDER_CATALOG)) + + +class Test_Already_Empty_Scope_Does_Not_Blame_Exclusions: + """When a positive filter (severity, categories, checks that resolve + to nothing) leaves the scope empty *before* exclusions run, the + exclusion pass must not falsely claim to be the cause. Otherwise the + real reason (empty selection) is masked by a misleading error.""" + + def test_empty_initial_scope_with_valid_exclusions_does_not_raise( + self, scan_provider + ): + # Force ``load_checks_to_execute`` to return an empty scope while + # keeping the exclusion inputs valid against the provider catalog. + with patch( + "prowler.lib.scan.scan.load_checks_to_execute", + return_value=set(), + ): + scan = Scan( + scan_provider, + excluded_checks=["s3_bucket_public_access"], + ) + assert scan.checks_to_execute == [] diff --git a/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet_test.py b/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet_test.py index 17709a94a7..adbd82d11a 100644 --- a/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet_test.py +++ b/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_rdp_internet/ecs_securitygroup_restrict_rdp_internet_test.py @@ -37,7 +37,7 @@ class TestEcsSecurityGroupRestrictRdpInternet: "ip_protocol": "tcp", "source_cidr_ip": "0.0.0.0/0", "port_range": "3389/3389", - "policy": "accept", + "policy": "Accept", } ], ) @@ -80,7 +80,7 @@ class TestEcsSecurityGroupRestrictRdpInternet: "ip_protocol": "tcp", "source_cidr_ip": "10.0.0.0/24", "port_range": "3389/3389", - "policy": "accept", + "policy": "Accept", } ], ) diff --git a/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet_test.py b/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet_test.py index 3278ce9a80..718baedce0 100644 --- a/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet_test.py +++ b/tests/providers/alibabacloud/services/ecs/ecs_securitygroup_restrict_ssh_internet/ecs_securitygroup_restrict_ssh_internet_test.py @@ -37,7 +37,7 @@ class TestEcsSecurityGroupRestrictSSHInternet: "ip_protocol": "tcp", "source_cidr_ip": "0.0.0.0/0", "port_range": "22/22", - "policy": "accept", + "policy": "Accept", } ], ) @@ -81,7 +81,7 @@ class TestEcsSecurityGroupRestrictSSHInternet: "ip_protocol": "tcp", "source_cidr_ip": "10.0.0.0/24", "port_range": "22/22", - "policy": "accept", + "policy": "Accept", } ], ) diff --git a/tests/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets_test.py b/tests/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets_test.py new file mode 100644 index 0000000000..106a3f2b1b --- /dev/null +++ b/tests/providers/aws/services/sagemaker/sagemaker_notebook_instance_no_secrets/sagemaker_notebook_instance_no_secrets_test.py @@ -0,0 +1,269 @@ +from unittest import mock + +from prowler.lib.utils.utils import SecretsScanError +from prowler.providers.aws.services.sagemaker.sagemaker_service import ( + NotebookInstance, +) +from tests.providers.aws.utils import ( + AWS_ACCOUNT_NUMBER, + AWS_REGION_EU_WEST_1, + set_mocked_aws_provider, +) + +test_notebook_instance = "test-notebook-instance" +notebook_instance_arn = ( + f"arn:aws:sagemaker:{AWS_REGION_EU_WEST_1}:" + f"{AWS_ACCOUNT_NUMBER}:notebook-instance/{test_notebook_instance}" +) + +other_notebook_instance = "other-notebook-instance" +other_notebook_instance_arn = ( + f"arn:aws:sagemaker:{AWS_REGION_EU_WEST_1}:" + f"{AWS_ACCOUNT_NUMBER}:notebook-instance/{other_notebook_instance}" +) + +CHECK_MODULE = "prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets" + + +class Test_sagemaker_notebook_instance_no_secrets: + def test_no_instances(self): + sagemaker_client = mock.MagicMock + sagemaker_client.sagemaker_notebook_instances = [] + sagemaker_client.audit_config = {} + + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch(f"{CHECK_MODULE}.sagemaker_client", sagemaker_client), + ): + from prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets import ( + sagemaker_notebook_instance_no_secrets, + ) + + check = sagemaker_notebook_instance_no_secrets() + result = check.execute() + + assert len(result) == 0 + + def test_pass_no_lifecycle_config(self): + sagemaker_client = mock.MagicMock + sagemaker_client.audit_config = {} + sagemaker_client.sagemaker_notebook_instances = [ + NotebookInstance( + name=test_notebook_instance, + arn=notebook_instance_arn, + region=AWS_REGION_EU_WEST_1, + lifecycle_config_name=None, + ) + ] + + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch(f"{CHECK_MODULE}.sagemaker_client", sagemaker_client), + mock.patch( + f"{CHECK_MODULE}.detect_secrets_scan_batch", + return_value={}, + ), + ): + from prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets import ( + sagemaker_notebook_instance_no_secrets, + ) + + check = sagemaker_notebook_instance_no_secrets() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "PASS" + assert ( + "does not have a lifecycle configuration" in result[0].status_extended + ) + assert result[0].resource_id == test_notebook_instance + assert result[0].resource_arn == notebook_instance_arn + + def test_pass_lifecycle_config_scanned_clean(self): + sagemaker_client = mock.MagicMock + sagemaker_client.audit_config = {} + sagemaker_client.sagemaker_notebook_instances = [ + NotebookInstance( + name=test_notebook_instance, + arn=notebook_instance_arn, + region=AWS_REGION_EU_WEST_1, + lifecycle_config_name="test-lifecycle-config", + lifecycle_scripts={"OnCreate[0]": "echo hello"}, + ) + ] + + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch(f"{CHECK_MODULE}.sagemaker_client", sagemaker_client), + mock.patch( + f"{CHECK_MODULE}.detect_secrets_scan_batch", + return_value={}, + ), + ): + from prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets import ( + sagemaker_notebook_instance_no_secrets, + ) + + check = sagemaker_notebook_instance_no_secrets() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "PASS" + assert "No secrets found" in result[0].status_extended + assert result[0].resource_id == test_notebook_instance + assert result[0].resource_arn == notebook_instance_arn + + def test_fail_secret_found(self): + notebook_instance = NotebookInstance( + name=test_notebook_instance, + arn=notebook_instance_arn, + region=AWS_REGION_EU_WEST_1, + lifecycle_config_name="test-lifecycle-config", + lifecycle_scripts={"OnCreate[0]": "echo API_KEY=12345"}, + ) + + sagemaker_client = mock.MagicMock + sagemaker_client.audit_config = {} + sagemaker_client.sagemaker_notebook_instances = [notebook_instance] + + fake_secret = {"type": "Secret Keyword", "line_number": 1} + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch(f"{CHECK_MODULE}.sagemaker_client", sagemaker_client), + mock.patch( + f"{CHECK_MODULE}.detect_secrets_scan_batch", + return_value={(notebook_instance_arn, "OnCreate[0]"): [fake_secret]}, + ), + mock.patch( + f"{CHECK_MODULE}.annotate_verified_secrets", + lambda *_: None, + ), + ): + from prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets import ( + sagemaker_notebook_instance_no_secrets, + ) + + check = sagemaker_notebook_instance_no_secrets() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "FAIL" + assert "Secret Keyword" in result[0].status_extended + assert "OnCreate[0]" in result[0].status_extended + assert result[0].resource_id == test_notebook_instance + assert result[0].resource_arn == notebook_instance_arn + + def test_manual_lifecycle_describe_failed(self): + # Service could not fully describe/decode the lifecycle config. + notebook_instance = NotebookInstance( + name=test_notebook_instance, + arn=notebook_instance_arn, + region=AWS_REGION_EU_WEST_1, + lifecycle_config_name="test-lifecycle-config", + lifecycle_scripts={}, + lifecycle_scan_failed=True, + ) + + sagemaker_client = mock.MagicMock + sagemaker_client.audit_config = {} + sagemaker_client.sagemaker_notebook_instances = [notebook_instance] + + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch(f"{CHECK_MODULE}.sagemaker_client", sagemaker_client), + mock.patch( + f"{CHECK_MODULE}.detect_secrets_scan_batch", + return_value={}, + ), + ): + from prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets import ( + sagemaker_notebook_instance_no_secrets, + ) + + check = sagemaker_notebook_instance_no_secrets() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "MANUAL" + assert result[0].resource_id == test_notebook_instance + assert result[0].resource_arn == notebook_instance_arn + + def test_manual_scan_error_only_scanned_instances(self): + # Batch scan fails. The instance with scripts must be MANUAL; the + # instance without a lifecycle config (nothing to scan) must PASS. + scanned_instance = NotebookInstance( + name=test_notebook_instance, + arn=notebook_instance_arn, + region=AWS_REGION_EU_WEST_1, + lifecycle_config_name="test-lifecycle-config", + lifecycle_scripts={"OnStart[0]": "echo hello"}, + ) + unscanned_instance = NotebookInstance( + name=other_notebook_instance, + arn=other_notebook_instance_arn, + region=AWS_REGION_EU_WEST_1, + lifecycle_config_name=None, + lifecycle_scripts={}, + ) + + sagemaker_client = mock.MagicMock + sagemaker_client.audit_config = {} + sagemaker_client.sagemaker_notebook_instances = [ + scanned_instance, + unscanned_instance, + ] + + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch(f"{CHECK_MODULE}.sagemaker_client", sagemaker_client), + mock.patch( + f"{CHECK_MODULE}.detect_secrets_scan_batch", + side_effect=SecretsScanError("scan failed"), + ), + ): + from prowler.providers.aws.services.sagemaker.sagemaker_notebook_instance_no_secrets.sagemaker_notebook_instance_no_secrets import ( + sagemaker_notebook_instance_no_secrets, + ) + + check = sagemaker_notebook_instance_no_secrets() + result = check.execute() + + assert len(result) == 2 + results_by_id = {report.resource_id: report for report in result} + + assert results_by_id[test_notebook_instance].status == "MANUAL" + assert results_by_id[other_notebook_instance].status == "PASS" + assert ( + "does not have a lifecycle configuration" + in results_by_id[other_notebook_instance].status_extended + ) diff --git a/tests/providers/aws/services/sagemaker/sagemaker_service_test.py b/tests/providers/aws/services/sagemaker/sagemaker_service_test.py index 50431c2e13..bfadd59efe 100644 --- a/tests/providers/aws/services/sagemaker/sagemaker_service_test.py +++ b/tests/providers/aws/services/sagemaker/sagemaker_service_test.py @@ -28,6 +28,10 @@ test_training_job = "test-training-job" test_arn_training_job = f"arn:aws:sagemaker:{AWS_REGION_EU_WEST_1}:{AWS_ACCOUNT_NUMBER}:training-job/{test_model}" subnet_id = "subnet-" + str(uuid4()) kms_key_id = str(uuid4()) +lifecycle_config_name = "test-lifecycle-config" +# base64 of "echo OnCreate" / "echo OnStart" +lifecycle_on_create_b64 = "ZWNobyBPbkNyZWF0ZQ==" +lifecycle_on_start_b64 = "ZWNobyBPblN0YXJ0" endpoint_config_name = "endpoint-config-test" endpoint_config_arn = f"arn:aws:sagemaker:{AWS_REGION_EU_WEST_1}:{AWS_ACCOUNT_NUMBER}:endpoint-config/{endpoint_config_name}" prod_variant_name = "Variant1" @@ -76,6 +80,12 @@ def mock_make_api_call(self, operation_name, kwarg): "KmsKeyId": kms_key_id, "DirectInternetAccess": "Enabled", "RootAccess": "Enabled", + "NotebookInstanceLifecycleConfigName": lifecycle_config_name, + } + if operation_name == "DescribeNotebookInstanceLifecycleConfig": + return { + "OnCreate": [{"Content": lifecycle_on_create_b64}], + "OnStart": [{"Content": lifecycle_on_start_b64}], } if operation_name == "DescribeModel": return { @@ -247,6 +257,21 @@ class Test_SageMaker_Service: assert sagemaker.sagemaker_notebook_instances[0].subnet_id == subnet_id assert sagemaker.sagemaker_notebook_instances[0].direct_internet_access assert sagemaker.sagemaker_notebook_instances[0].kms_key_id == kms_key_id + assert ( + sagemaker.sagemaker_notebook_instances[0].lifecycle_config_name + == lifecycle_config_name + ) + + # Test SageMaker describe notebook instance lifecycle config + def test_describe_notebook_instance_lifecycle_config(self): + aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1]) + sagemaker = SageMaker(aws_provider) + notebook_instance = sagemaker.sagemaker_notebook_instances[0] + assert notebook_instance.lifecycle_scan_failed is False + assert notebook_instance.lifecycle_scripts == { + "OnCreate[0]": "echo OnCreate", + "OnStart[0]": "echo OnStart", + } # Test SageMaker describe model def test_describe_model(self): diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md index 26385b3adc..40cb61691f 100644 --- a/ui/CHANGELOG.md +++ b/ui/CHANGELOG.md @@ -4,6 +4,26 @@ All notable changes to the **Prowler UI** are documented in this file. +## [1.35.0] (Prowler v5.35.0) + +### 🔄 Changed + +- AWS Organizations onboarding now deploys the management account role and the member-account StackSet from a single CloudFormation stack, replacing the manual StackSet console step [(#11927)](https://github.com/prowler-cloud/prowler/pull/11927) +- Dynamic providers can now be renamed and deleted from the Providers table [(#11957)](https://github.com/prowler-cloud/prowler/pull/11957) +- Sidebar navigation with grouped sections, clearer active states, and a responsive mobile overlay [(#11994)](https://github.com/prowler-cloud/prowler/pull/11994) +- Core Prowler tools in Lighthouse use the `prowler_*` namespace while preserving legacy `prowler_app_*` compatibility [(#12017)](https://github.com/prowler-cloud/prowler/pull/12017) + +### 🐞 Fixed + +- The AWS S3 integration CloudFormation quick-create link now sets the bucket owner account ID, preventing a stack validation error when S3 integration is enabled [(#11927)](https://github.com/prowler-cloud/prowler/pull/11927) +- `Scan ID` filter on the Findings page now shows the active scan when opening findings from a scan's `View Findings` action [(#11997)](https://github.com/prowler-cloud/prowler/pull/11997) + +### 🔐 Security + +- `js-yaml` to 4.3.0, `@sentry/nextjs` to 10.65.0 with `import-in-the-middle` 3.3.1, and transitive `hono`, `dompurify`, `ws`, `vite`, `@babel/core` and `@opentelemetry/core` to patched versions, resolving 13 npm audit advisories (3 high, 9 moderate, 1 low) plus `hono` CVE-2026-59896, published on NVD but not yet in the npm audit feed [(#12029)](https://github.com/prowler-cloud/prowler/pull/12029) + +--- + ## [1.34.0] (Prowler v5.34.0) ### 🚀 Added diff --git a/ui/actions/organizations/organizations.adapter.test.ts b/ui/actions/organizations/organizations.adapter.test.ts index 07cff60c0c..41e0b8d89f 100644 --- a/ui/actions/organizations/organizations.adapter.test.ts +++ b/ui/actions/organizations/organizations.adapter.test.ts @@ -11,6 +11,7 @@ import { buildOrgTreeData, getOuIdsForSelectedAccounts, getSelectableAccountIds, + getSelectableAccountIdsForTarget, } from "./organizations.adapter"; const discoveryFixture: DiscoveryResult = { @@ -164,6 +165,68 @@ describe("buildAccountLookup", () => { }); }); +describe("getSelectableAccountIdsForTarget", () => { + it("scopes selection to accounts under a target OU, including nested OUs", () => { + // ou-parent contains ou-child (holds 111...) and the blocked 222... + const scoped = getSelectableAccountIdsForTarget( + discoveryFixture, + "ou-parent", + ); + + // Only the selectable descendant is returned; blocked 222... is excluded, + // and 333... (under the root, outside the OU) is not included. + expect(scoped).toEqual(["111111111111"]); + }); + + it("scopes selection to a leaf OU", () => { + const scoped = getSelectableAccountIdsForTarget( + discoveryFixture, + "ou-child", + ); + + expect(scoped).toEqual(["111111111111"]); + }); + + it("includes the deployment account even when it lives outside the target OU", () => { + // Deployment (management) account 333... sits under the root, but gets the + // role via DeployLocalRole, so it must be pre-selected alongside the OU. + const scoped = getSelectableAccountIdsForTarget( + discoveryFixture, + "ou-child", + "333333333333", + ); + + expect(scoped).toEqual(["111111111111", "333333333333"]); + }); + + it("does not include a deployment account that is not selectable", () => { + // 222... is blocked, so even as the deployment account it stays unselected. + const scoped = getSelectableAccountIdsForTarget( + discoveryFixture, + "ou-child", + "222222222222", + ); + + expect(scoped).toEqual(["111111111111"]); + }); + + it("returns every selectable account for a root target (whole organization)", () => { + const scoped = getSelectableAccountIdsForTarget(discoveryFixture, "r-root"); + + expect(scoped).toEqual(["111111111111", "333333333333"]); + }); + + it("falls back to all selectable accounts for an empty or unknown target", () => { + expect(getSelectableAccountIdsForTarget(discoveryFixture, "")).toEqual([ + "111111111111", + "333333333333", + ]); + expect( + getSelectableAccountIdsForTarget(discoveryFixture, "ou-does-not-exist"), + ).toEqual(["111111111111", "333333333333"]); + }); +}); + describe("getOuIdsForSelectedAccounts", () => { it("collects all ancestor OUs for selected accounts without duplicates", () => { const ouIds = getOuIdsForSelectedAccounts(discoveryFixture, [ diff --git a/ui/actions/organizations/organizations.adapter.ts b/ui/actions/organizations/organizations.adapter.ts index e120a1a3dc..2b0c7b5a0b 100644 --- a/ui/actions/organizations/organizations.adapter.ts +++ b/ui/actions/organizations/organizations.adapter.ts @@ -109,6 +109,71 @@ export function buildAccountLookup( return map; } +/** + * Returns the selectable account IDs that fall under a deployment target + * (an OU or root ID), optionally including the deployment account itself. + * + * The StackSet only rolls the role out to member accounts beneath the chosen + * target, and the deployment (management or delegated administrator) account + * gets the role via DeployLocalRole even though it usually lives outside that + * target. Pre-selecting exactly those accounts keeps the confirmation step in + * sync with what was actually deployed. + * + * Falls back to every selectable account when the target is empty or is not + * part of this discovery (e.g. a root ID), preserving the whole-organization + * default. + */ +export function getSelectableAccountIdsForTarget( + result: DiscoveryResult, + targetId: string, + deploymentAccountId?: string, +): string[] { + const selectableAccountIds = getSelectableAccountIds(result); + const normalizedTarget = targetId.trim(); + + if (!normalizedTarget) { + return selectableAccountIds; + } + + const isKnownOu = result.organizational_units.some( + (ou) => ou.id === normalizedTarget, + ); + + // Only a specific OU narrows the selection. A root ID (whole org) or an + // unknown target keeps the whole-organization default. + if (!isKnownOu) { + return selectableAccountIds; + } + + // Collect the target OU plus all of its nested descendant OUs. + const scopeIds = new Set([normalizedTarget]); + let addedNewOu = true; + while (addedNewOu) { + addedNewOu = false; + for (const ou of result.organizational_units) { + if (!scopeIds.has(ou.id) && scopeIds.has(ou.parent_id)) { + scopeIds.add(ou.id); + addedNewOu = true; + } + } + } + + const selectableSet = new Set(selectableAccountIds); + const scopedIds = new Set(); + + for (const account of result.accounts) { + if (scopeIds.has(account.parent_id) && selectableSet.has(account.id)) { + scopedIds.add(account.id); + } + } + + if (deploymentAccountId && selectableSet.has(deploymentAccountId)) { + scopedIds.add(deploymentAccountId); + } + + return selectableAccountIds.filter((id) => scopedIds.has(id)); +} + /** * Given selected account IDs, returns OU IDs that are ancestors of selected accounts. */ diff --git a/ui/actions/overview/resources-inventory/resources-inventory.adapter.ts b/ui/actions/overview/resources-inventory/resources-inventory.adapter.ts index 5e6f5fd1f2..508d9a46ad 100644 --- a/ui/actions/overview/resources-inventory/resources-inventory.adapter.ts +++ b/ui/actions/overview/resources-inventory/resources-inventory.adapter.ts @@ -1,5 +1,5 @@ -import { LucideIcon } from "lucide-react"; import { + LucideIcon, Activity, BarChart3, Bot, diff --git a/ui/app/(auth)/(guest-only)/sign-up/page.tsx b/ui/app/(auth)/(guest-only)/sign-up/page.tsx index 4854de012b..c625415ce0 100644 --- a/ui/app/(auth)/(guest-only)/sign-up/page.tsx +++ b/ui/app/(auth)/(guest-only)/sign-up/page.tsx @@ -1,6 +1,9 @@ import { AuthForm } from "@/components/auth/oss"; -import { getAuthUrl, isGithubOAuthEnabled } from "@/lib/helper"; -import { isGoogleOAuthEnabled } from "@/lib/helper"; +import { + getAuthUrl, + isGithubOAuthEnabled, + isGoogleOAuthEnabled, +} from "@/lib/helper"; import { SearchParamsProps } from "@/types"; const SignUp = async ({ diff --git a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx index cec9c8fc55..c412eaf48c 100644 --- a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx +++ b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx @@ -10,6 +10,7 @@ import { getInvitationErrorDisplay, INVITATION_ERROR_FLOW, } from "@/app/(auth)/invitation/_lib/invitation-errors"; +import { AuthBrand } from "@/components/auth/oss/auth-brand"; import { Button } from "@/components/shadcn"; type AcceptState = @@ -69,6 +70,7 @@ export function AcceptInvitationClient({ return (
+ {/* No token */} {state.kind === "no-token" && (
diff --git a/ui/app/(auth)/layout.tsx b/ui/app/(auth)/layout.tsx index b0a9bc2061..27fff93fe5 100644 --- a/ui/app/(auth)/layout.tsx +++ b/ui/app/(auth)/layout.tsx @@ -5,7 +5,6 @@ import { Metadata, Viewport } from "next"; import { connection } from "next/server"; import { ReactNode, Suspense } from "react"; -import { PublicAuthShell } from "@/components/auth/oss/public-auth-shell"; import { RuntimePublicConfig } from "@/components/runtime-config/runtime-public-config"; import { NavigationProgress, Toaster } from "@/components/shadcn"; import { fontMono, fontSans } from "@/config/fonts"; @@ -66,7 +65,7 @@ export default async function AuthLayout({ - {children} + {children} {gtmId && } diff --git a/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.test.tsx b/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.test.tsx index 62bb053f3a..b407396ce8 100644 --- a/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.test.tsx +++ b/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.test.tsx @@ -1,39 +1,53 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it } from "vitest"; +import { LIGHTHOUSE_OVERVIEW_BANNER_HREF } from "../_lib/lighthouse-banner"; + import { LighthouseOverviewBanner } from "./lighthouse-overview-banner"; -const REMEDIATION_PROMPT = - "Find and guide me to remediate which actually matters. What do I have to do today to be secure?"; -const REMEDIATION_HREF = `/lighthouse?prompt=${encodeURIComponent( - REMEDIATION_PROMPT, -)}` as const; - describe("LighthouseOverviewBanner", () => { - it("renders Toni copy and links to Lighthouse when connected", () => { + it("renders Toni copy and opens a prompted chat when connected", () => { // Given / When - render(); + render( + , + ); // Then const link = screen.getByRole("link", { - name: /Find and remediate which actually matters\./, + name: /Find and remediate what actually matters\./, }); - expect(link).toHaveAttribute("href", REMEDIATION_HREF); + expect(link).toHaveAttribute("href", LIGHTHOUSE_OVERVIEW_BANNER_HREF.CHAT); expect(link).toHaveTextContent("Lighthouse AI"); - expect(link).toHaveTextContent( - "Find and remediate which actually matters.", - ); + expect(link).toHaveTextContent("Find and remediate what actually matters."); }); it("links to Lighthouse settings when no connected configuration exists", () => { // Given / When - render(); + render( + , + ); // Then expect( screen.getByRole("link", { - name: /Find and remediate which actually matters\./, + name: /Find and remediate what actually matters\./, }), ).toHaveAttribute("href", "/lighthouse/settings"); }); + + it("isolates its stacking so content never paints over the sticky navbar", () => { + // Given / When: the banner's inner z-10 must stay scoped to the card — + // without isolation it ties the sticky header's z-10 and wins by DOM order + render( + , + ); + + // Then + const card = screen + .getByRole("link", { name: /Find and remediate what actually matters\./ }) + .querySelector("[data-slot='card']"); + expect(card).toHaveClass("isolate"); + }); }); diff --git a/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.tsx b/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.tsx index eb2e0539fb..8bd8dea414 100644 --- a/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.tsx +++ b/ui/app/(prowler)/_overview/_components/lighthouse-overview-banner.tsx @@ -66,7 +66,9 @@ export function LighthouseOverviewBanner({ @@ -116,7 +118,7 @@ export function LighthouseOverviewBanner({ Lighthouse AI

- Find and remediate which actually matters. + Find and remediate what actually matters.

diff --git a/ui/app/(prowler)/_overview/_lib/lighthouse-banner.test.ts b/ui/app/(prowler)/_overview/_lib/lighthouse-banner.test.ts index 7668cfafac..baf5e8206f 100644 --- a/ui/app/(prowler)/_overview/_lib/lighthouse-banner.test.ts +++ b/ui/app/(prowler)/_overview/_lib/lighthouse-banner.test.ts @@ -16,7 +16,7 @@ const LIGHTHOUSE_OVERVIEW_CHAT_HREF = `/lighthouse?prompt=${encodeURIComponent( )}`; describe("resolveLighthouseOverviewBannerHref", () => { - it("routes to Lighthouse chat when any v2 configuration is connected", () => { + it("opens Lighthouse with the remediation prompt when any v2 configuration is connected", () => { // Given / When const href = resolveLighthouseOverviewBannerHref([ configuration("openai", false), diff --git a/ui/app/(prowler)/_overview/_lib/lighthouse-banner.ts b/ui/app/(prowler)/_overview/_lib/lighthouse-banner.ts index a80522bf6c..b0b0d1a08c 100644 --- a/ui/app/(prowler)/_overview/_lib/lighthouse-banner.ts +++ b/ui/app/(prowler)/_overview/_lib/lighthouse-banner.ts @@ -2,8 +2,9 @@ import type { LighthouseV2Configuration } from "@/app/(prowler)/lighthouse/_type import { LIGHTHOUSE_ROUTE } from "@/lib/lighthouse-routes"; import type { ServerActionResult } from "@/types/server-actions"; +// Prefilled in the composer when the overview banner opens Lighthouse. export const LIGHTHOUSE_OVERVIEW_PROMPT = - "Find and guide me to remediate which actually matters. What do I have to do today to be secure?"; + "Find and guide me to remediate what actually matters. What do I have to do today to be secure?"; export const LIGHTHOUSE_OVERVIEW_BANNER_HREF = { CHAT: `${LIGHTHOUSE_ROUTE.CHAT}?prompt=${encodeURIComponent(LIGHTHOUSE_OVERVIEW_PROMPT)}`, diff --git a/ui/app/(prowler)/_overview/attack-surface/attack-surface.ssr.tsx b/ui/app/(prowler)/_overview/attack-surface/attack-surface.ssr.tsx index 7f5b0b0f5f..03b89db714 100644 --- a/ui/app/(prowler)/_overview/attack-surface/attack-surface.ssr.tsx +++ b/ui/app/(prowler)/_overview/attack-surface/attack-surface.ssr.tsx @@ -5,6 +5,7 @@ import { import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { AttackSurface } from "./_components/attack-surface"; export const AttackSurfaceSSR = async ({ searchParams }: SSRComponentProps) => { diff --git a/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot.ssr.tsx b/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot.ssr.tsx index 1f4d3625d4..940362dfe6 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot.ssr.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot.ssr.tsx @@ -13,6 +13,7 @@ import { filterProvidersByScope, parseFilterIds, } from "../../_lib/provider-scope"; + import { RiskPlotClient } from "./risk-plot-client"; export async function RiskPlotSSR({ diff --git a/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view.ssr.tsx b/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view.ssr.tsx index 932251e08a..355c03c397 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view.ssr.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view.ssr.tsx @@ -7,6 +7,7 @@ import { import { SearchParamsProps } from "@/types"; import { pickFilterParams } from "../../_lib/filter-params"; + import { RiskRadarViewClient } from "./risk-radar-view-client"; export async function RiskRadarViewSSR({ diff --git a/ui/app/(prowler)/_overview/resources-inventory/resources-inventory.ssr.tsx b/ui/app/(prowler)/_overview/resources-inventory/resources-inventory.ssr.tsx index a95f65f9d8..17a7df9f52 100644 --- a/ui/app/(prowler)/_overview/resources-inventory/resources-inventory.ssr.tsx +++ b/ui/app/(prowler)/_overview/resources-inventory/resources-inventory.ssr.tsx @@ -5,6 +5,7 @@ import { import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { ResourcesInventory } from "./_components/resources-inventory"; export const ResourcesInventorySSR = async ({ diff --git a/ui/app/(prowler)/_overview/risk-severity/risk-severity-chart.ssr.tsx b/ui/app/(prowler)/_overview/risk-severity/risk-severity-chart.ssr.tsx index c825748169..d59b961c92 100644 --- a/ui/app/(prowler)/_overview/risk-severity/risk-severity-chart.ssr.tsx +++ b/ui/app/(prowler)/_overview/risk-severity/risk-severity-chart.ssr.tsx @@ -2,6 +2,7 @@ import { getFindingsBySeverity } from "@/actions/overview"; import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { RiskSeverityChart } from "./_components/risk-severity-chart"; export const RiskSeverityChartSSR = async ({ diff --git a/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx b/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx index b65b6a21f0..31cbc42cc4 100644 --- a/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx +++ b/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx @@ -15,6 +15,7 @@ import { } from "@/types/severities"; import { DEFAULT_TIME_RANGE } from "../_constants/time-range.constants"; + import { type TimeRange, TimeRangeSelector } from "./time-range-selector"; interface FindingSeverityOverTimeProps { diff --git a/ui/app/(prowler)/_overview/severity-over-time/finding-severity-over-time.ssr.tsx b/ui/app/(prowler)/_overview/severity-over-time/finding-severity-over-time.ssr.tsx index a2d7a36d5c..e1f9c6635e 100644 --- a/ui/app/(prowler)/_overview/severity-over-time/finding-severity-over-time.ssr.tsx +++ b/ui/app/(prowler)/_overview/severity-over-time/finding-severity-over-time.ssr.tsx @@ -3,6 +3,7 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/shadcn"; import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { FindingSeverityOverTime } from "./_components/finding-severity-over-time"; import { FindingSeverityOverTimeSkeleton } from "./_components/finding-severity-over-time.skeleton"; import { DEFAULT_TIME_RANGE } from "./_constants/time-range.constants"; diff --git a/ui/app/(prowler)/_overview/threat-score/threat-score.ssr.tsx b/ui/app/(prowler)/_overview/threat-score/threat-score.ssr.tsx index 7f5364d147..244c50527e 100644 --- a/ui/app/(prowler)/_overview/threat-score/threat-score.ssr.tsx +++ b/ui/app/(prowler)/_overview/threat-score/threat-score.ssr.tsx @@ -2,6 +2,7 @@ import { getThreatScore } from "@/actions/overview"; import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { ThreatScore } from "./_components/threat-score"; export const ThreatScoreSSR = async ({ searchParams }: SSRComponentProps) => { diff --git a/ui/app/(prowler)/_overview/watchlist/compliance-watchlist.ssr.tsx b/ui/app/(prowler)/_overview/watchlist/compliance-watchlist.ssr.tsx index d0aae42dad..e897a58ea6 100644 --- a/ui/app/(prowler)/_overview/watchlist/compliance-watchlist.ssr.tsx +++ b/ui/app/(prowler)/_overview/watchlist/compliance-watchlist.ssr.tsx @@ -5,6 +5,7 @@ import { import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { ComplianceWatchlist } from "./_components/compliance-watchlist"; export const ComplianceWatchlistSSR = async ({ diff --git a/ui/app/(prowler)/_overview/watchlist/service-watchlist.ssr.tsx b/ui/app/(prowler)/_overview/watchlist/service-watchlist.ssr.tsx index 9ec08a6cb1..a84d9f1c56 100644 --- a/ui/app/(prowler)/_overview/watchlist/service-watchlist.ssr.tsx +++ b/ui/app/(prowler)/_overview/watchlist/service-watchlist.ssr.tsx @@ -2,6 +2,7 @@ import { getServicesOverview, ServiceOverview } from "@/actions/overview"; import { pickFilterParams } from "../_lib/filter-params"; import { SSRComponentProps } from "../_types"; + import { ServiceWatchlist } from "./_components/service-watchlist"; export const ServiceWatchlistSSR = async ({ diff --git a/ui/app/(prowler)/alerts/_actions/alerts.test.ts b/ui/app/(prowler)/alerts/_actions/alerts.test.ts index 5f19b33733..4dfe3dd563 100644 --- a/ui/app/(prowler)/alerts/_actions/alerts.test.ts +++ b/ui/app/(prowler)/alerts/_actions/alerts.test.ts @@ -25,6 +25,7 @@ vi.mock("@/lib/server-actions-helper", () => ({ })); import { ALERT_AGGREGATE_OPS, ALERT_TRIGGER_KINDS } from "../_types"; + import { createAlert, deleteAlert, diff --git a/ui/app/(prowler)/alerts/_components/alerts-manager.tsx b/ui/app/(prowler)/alerts/_components/alerts-manager.tsx index 92f11631ad..c9b499b251 100644 --- a/ui/app/(prowler)/alerts/_components/alerts-manager.tsx +++ b/ui/app/(prowler)/alerts/_components/alerts-manager.tsx @@ -15,12 +15,10 @@ import { ALERT_TRIGGER_KINDS, type AlertRule, } from "@/app/(prowler)/alerts/_types"; -import { Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Button, useToast } from "@/components/shadcn"; import { Modal } from "@/components/shadcn/modal"; import { DOCS_URLS } from "@/lib/external-urls"; -import type { MetaDataProps } from "@/types"; -import type { ScanEntity } from "@/types"; +import type { MetaDataProps, ScanEntity } from "@/types"; import type { ProviderProps } from "@/types/providers"; import { toAlertPayload } from "../_lib/alert-adapter"; @@ -29,6 +27,7 @@ import type { AlertFormSubmitResult, AlertFormValues, } from "../_types/alert-form"; + import { AlertFormModal } from "./alert-form-modal"; import { AlertsEmptyState } from "./alerts-empty-state"; import { AlertsTable } from "./alerts-table"; diff --git a/ui/app/(prowler)/alerts/_components/seed-from-findings-button.tsx b/ui/app/(prowler)/alerts/_components/seed-from-findings-button.tsx index 6b94549a9e..988f7e70f8 100644 --- a/ui/app/(prowler)/alerts/_components/seed-from-findings-button.tsx +++ b/ui/app/(prowler)/alerts/_components/seed-from-findings-button.tsx @@ -28,8 +28,9 @@ import { Tooltip, TooltipContent, TooltipTrigger, + ToastAction, + useToast, } from "@/components/shadcn"; -import { ToastAction, useToast } from "@/components/shadcn"; import { useCloudUpgradeStore } from "@/store"; import type { ScanEntity } from "@/types"; import { CLOUD_UPGRADE_FEATURE } from "@/types/cloud-upgrade"; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/attack-paths-status-panel.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/attack-paths-status-panel.test.tsx index 20210a3126..f95aa99870 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/attack-paths-status-panel.test.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/attack-paths-status-panel.test.tsx @@ -2,6 +2,7 @@ import { fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import { ATTACK_PATHS_VIEW_STATES } from "../_lib/get-attack-paths-view-state"; + import { AttackPathsStatusPanel } from "./attack-paths-status-panel"; describe("AttackPathsStatusPanel", () => { diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/attack-path-graph.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/attack-path-graph.tsx index 6e9c608f64..a987e3a7ac 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/attack-path-graph.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/attack-path-graph.tsx @@ -34,6 +34,7 @@ import { resolveHiddenFindingIds, } from "../../_lib"; import { isFindingNode, layoutWithDagre } from "../../_lib/layout"; + import { FindingNode } from "./nodes/finding-node"; import { InternetNode } from "./nodes/internet-node"; import { ResourceNode } from "./nodes/resource-node"; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/finding-node.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/finding-node.tsx index 789a379551..78583c628a 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/finding-node.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/finding-node.tsx @@ -12,6 +12,7 @@ import type { GraphNode } from "@/types/attack-paths"; import { resolveNodeColors, resolveNodeVisual } from "../../../_lib"; import { FINDING_NODE_DIMENSIONS } from "../../../_lib/node-dimensions"; import { getNodeLabelDisplay } from "../../../_lib/node-label-lines"; + import { HiddenHandles } from "./hidden-handles"; interface FindingNodeData { diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/internet-node.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/internet-node.tsx index e2009f71c9..097e2903c5 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/internet-node.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/internet-node.tsx @@ -5,6 +5,7 @@ import { type NodeProps } from "@xyflow/react"; import type { GraphNode } from "@/types/attack-paths"; import { resolveNodeColors } from "../../../_lib"; + import { HiddenHandles } from "./hidden-handles"; interface InternetNodeData { diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/resource-node.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/resource-node.tsx index 9860dbea27..3120129091 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/resource-node.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/nodes/resource-node.tsx @@ -12,6 +12,7 @@ import type { GraphNode } from "@/types/attack-paths"; import { resolveNodeColors, resolveNodeVisual } from "../../../_lib"; import { RESOURCE_NODE_DIMENSIONS } from "../../../_lib/node-dimensions"; import { getNodeLabelDisplay } from "../../../_lib/node-label-lines"; + import { HiddenHandles } from "./hidden-handles"; interface ResourceNodeData { diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx index cc87406aa8..3fc2e59f94 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/attack-paths-page.tsx @@ -28,13 +28,13 @@ import { import { StatusAlert } from "@/components/shared/status-alert"; import { useMountEffect } from "@/hooks/use-mount-effect"; import { isCloud } from "@/lib/shared/env"; +import { attackPathsEmptyTour } from "@/lib/tours/attack-paths-empty.tour"; import { attackPathsTour, type AttackPathsTourTarget, pickDemoQuery, pickDemoScan, } from "@/lib/tours/attack-paths.tour"; -import { attackPathsEmptyTour } from "@/lib/tours/attack-paths-empty.tour"; import { advanceActiveTour, useDriverTour } from "@/lib/tours/use-driver-tour"; import type { AttackPathQuery, diff --git a/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx index 617d943030..531cfbe79d 100644 --- a/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx +++ b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx @@ -6,6 +6,7 @@ import { useCloudUpgradeStore } from "@/store"; import { CLOUD_UPGRADE_FEATURE } from "@/types/cloud-upgrade"; import { COMPLIANCE_TAB } from "../_types"; + import { CompliancePageTabs } from "./compliance-page-tabs"; import { getComplianceTab } from "./compliance-page-tabs.shared"; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx index 7972379b1f..4901294ad4 100644 --- a/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx +++ b/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx @@ -29,6 +29,7 @@ import { parseCrossProviderFilters, } from "../_lib/cross-provider-frameworks"; import { CROSS_PROVIDER_OVERVIEW_RESULT_STATUS } from "../_types"; + import { CrossProviderErrorAlert } from "./cross-provider-error-alert"; import type { CrossProviderAccountOption, diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx index b2fb3b66f4..1e5a71deba 100644 --- a/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx +++ b/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx @@ -11,6 +11,7 @@ import { CROSS_PROVIDER_OVERVIEW_RESULT_STATUS, CROSS_PROVIDER_OVERVIEW_TYPE, } from "../_types"; + import { CrossProviderOverview } from "./cross-provider-overview"; vi.mock("../_actions/cross-provider", () => ({ diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx index 17af2b38bf..d115ed4e0e 100644 --- a/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx +++ b/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx @@ -15,6 +15,7 @@ import { } from "../_lib/cross-provider-frameworks"; import type { CrossProviderFrameworkSummary } from "../_types"; import { CROSS_PROVIDER_OVERVIEW_RESULT_STATUS } from "../_types"; + import { CrossProviderErrorAlert } from "./cross-provider-error-alert"; import type { CrossProviderAccountOption, diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx index 98a9f11aab..5ec45475ce 100644 --- a/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx +++ b/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import type { CheckProviderTypesMap, Requirement } from "@/types/compliance"; import type { CrossProviderRequirementExtras } from "../_types"; + import { CrossProviderRequirementContent } from "./cross-provider-requirement-content"; const { clientAccordionContentMock } = vi.hoisted(() => ({ diff --git a/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx b/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx index d1f73d264a..ac30a01f53 100644 --- a/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx +++ b/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx @@ -2,6 +2,7 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import type { ProviderBreakdownEntry } from "../_types"; + import { ProviderCoverageCard } from "./provider-coverage-card"; vi.mock("@/components/icons/providers-badge/provider-type-icon", () => ({ diff --git a/ui/app/(prowler)/invitations/(send-invite)/new/page.tsx b/ui/app/(prowler)/invitations/(send-invite)/new/page.tsx index 520760092b..d0372c4f4b 100644 --- a/ui/app/(prowler)/invitations/(send-invite)/new/page.tsx +++ b/ui/app/(prowler)/invitations/(send-invite)/new/page.tsx @@ -1,4 +1,3 @@ -import React from "react"; import { Suspense } from "react"; import { getRoles } from "@/actions/roles"; diff --git a/ui/app/(prowler)/layout.tsx b/ui/app/(prowler)/layout.tsx index 801b1ea28f..f657f32199 100644 --- a/ui/app/(prowler)/layout.tsx +++ b/ui/app/(prowler)/layout.tsx @@ -16,6 +16,7 @@ import { RuntimePublicConfig } from "@/components/runtime-config/runtime-public- import { NavigationProgress } from "@/components/shadcn/navigation-progress"; import { Toaster } from "@/components/shadcn/toast"; import { TaskPollingWatcher } from "@/components/shared/task-polling-watcher"; +import { GlobalSidePanel } from "@/components/side-panel"; import { fontMono, fontSans } from "@/config/fonts"; import { siteConfig } from "@/config/site"; import { isCloud } from "@/lib/shared/env"; @@ -107,6 +108,9 @@ export default async function RootLayout({ )} {children} + {/* Always mounted: it hosts the detail (finding/resource) views in + every deployment; the AI tab inside is cloud-gated on its own. */} + {/* Resumes persisted background-task polling (e.g. cross-provider PDF generation) so completion toasts survive hard reloads. */} diff --git a/ui/app/(prowler)/lighthouse/_components/chat/composer.tsx b/ui/app/(prowler)/lighthouse/_components/chat/composer.tsx index 62d39cfb05..b9f854dbdb 100644 --- a/ui/app/(prowler)/lighthouse/_components/chat/composer.tsx +++ b/ui/app/(prowler)/lighthouse/_components/chat/composer.tsx @@ -118,7 +118,9 @@ function ChatComposer({ aria-label="Message" value={input} onChange={(event) => onInputChange(event.target.value)} - disabled={!canSend} + // Typing stays available while a response streams (sending is what is + // gated, via canSend); only a disconnected provider blocks the input. + disabled={!selectedConfigurationConnected} placeholder={ selectedConfigurationConnected ? "Ask a question" diff --git a/ui/app/(prowler)/lighthouse/_components/chat/empty-state.tsx b/ui/app/(prowler)/lighthouse/_components/chat/empty-state.tsx index 021be182e2..b84c71b3b7 100644 --- a/ui/app/(prowler)/lighthouse/_components/chat/empty-state.tsx +++ b/ui/app/(prowler)/lighthouse/_components/chat/empty-state.tsx @@ -5,6 +5,7 @@ import { type ReactNode, type SubmitEvent } from "react"; import { LighthouseIconWithAura } from "@/components/icons"; import { Button } from "@/components/shadcn/button/button"; +import { cn } from "@/lib/utils"; import { ChatComposerPanel } from "./composer"; import { DecryptedText } from "./decrypted-text"; @@ -45,34 +46,58 @@ interface ChatEmptyStateProps { onInputChange: (value: string) => void; onSubmit: (event: SubmitEvent) => void; onSubmitText: (text: string) => Promise; + footer?: ReactNode; + // Side-panel variant: smaller logo and static (non-animated) copy — the + // decrypt animation reflows multi-line text in narrow widths. + compact?: boolean; } export function ChatEmptyState({ onInputChange, + footer, + compact = false, ...composerPanelProps }: ChatEmptyStateProps) { return (
- +
-

- +

+ {compact ? ( + "Find and remediate what actually matters." + ) : ( + + )}

-

- +

+ {compact ? ( + "What do you want to know today?" + ) : ( + + )}

@@ -101,6 +126,7 @@ export function ChatEmptyState({ ); })}
+ {footer ?
{footer}
: null}
); diff --git a/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-chat-store-provider.tsx b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-chat-store-provider.tsx new file mode 100644 index 0000000000..55866304bb --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-chat-store-provider.tsx @@ -0,0 +1,41 @@ +"use client"; + +import { createContext, type ReactNode, useContext } from "react"; +import { useStore } from "zustand"; + +import type { + LighthouseChatState, + LighthouseChatStore, +} from "@/app/(prowler)/lighthouse/_lib/chat-store"; + +const LighthouseChatStoreContext = createContext( + null, +); + +interface LighthouseChatStoreProviderProps { + store: LighthouseChatStore; + children: ReactNode; +} + +export function LighthouseChatStoreProvider({ + store, + children, +}: LighthouseChatStoreProviderProps) { + return ( + + {children} + + ); +} + +export function useLighthouseChatStore( + selector: (state: LighthouseChatState) => T, +): T { + const store = useContext(LighthouseChatStoreContext); + if (!store) { + throw new Error( + "useLighthouseChatStore must be used within LighthouseChatStoreProvider", + ); + } + return useStore(store, selector); +} diff --git a/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.test.tsx b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.test.tsx index 5642762901..b6c5890db4 100644 --- a/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.test.tsx +++ b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.test.tsx @@ -3,10 +3,18 @@ import userEvent from "@testing-library/user-event"; import { type ReactNode } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getOrCreatePanelChatStore, + resetPanelChatStoreForTests, +} from "@/app/(prowler)/lighthouse/_lib/panel-chat-store"; import { LIGHTHOUSE_V2_SESSIONS_CHANGED_EVENT, notifyLighthouseV2SessionArchived, } from "@/app/(prowler)/lighthouse/_lib/session-events"; +import { + type MockEventSource, + stubEventSource, +} from "@/app/(prowler)/lighthouse/_lib/testing/event-source-mock"; import type { LighthouseV2Configuration, LighthouseV2Message, @@ -16,51 +24,8 @@ import type { import { LighthouseV2ChatPage } from "./lighthouse-v2-chat-page"; -// Controllable EventSource mock: records each instance so tests can drive -// named SSE events and connection failures, while still being a vi.fn so -// `expect(EventSource).toHaveBeenCalledWith(...)` keeps working. -interface MockEventSource { - url: string; - readyState: number; - onerror: ((event: Event) => void) | null; - listeners: Map>; - addEventListener: (type: string, cb: EventListener) => void; - close: ReturnType; - emit: (type: string, data: unknown) => void; - fail: (readyState: number) => void; -} - let eventSources: MockEventSource[] = []; -function stubEventSource() { - eventSources = []; - const EventSourceMock = vi.fn(function (this: MockEventSource, url: string) { - this.url = url; - this.readyState = 0; - this.onerror = null; - this.listeners = new Map(); - this.addEventListener = (type: string, cb: EventListener) => { - const set = this.listeners.get(type) ?? new Set(); - set.add(cb); - this.listeners.set(type, set); - }; - this.close = vi.fn(() => { - this.readyState = 2; - }); - this.emit = (type: string, data: unknown) => { - const event = new MessageEvent(type, { data: JSON.stringify(data) }); - this.listeners.get(type)?.forEach((cb) => cb(event)); - }; - this.fail = (readyState: number) => { - this.readyState = readyState; - this.onerror?.(new Event("error")); - }; - eventSources.push(this); - }); - Object.assign(EventSourceMock, { CONNECTING: 0, OPEN: 1, CLOSED: 2 }); - vi.stubGlobal("EventSource", EventSourceMock); -} - const { createSessionMock, getMessagesMock, @@ -142,11 +107,8 @@ describe("LighthouseV2ChatPage", () => { getMessagesMock.mockReset(); sendMessageMock.mockReset(); updateConfigurationMock.mockReset(); - // The mock never fires "open": the client must POST the message without - // waiting for it (the backend sends no bytes until the worker emits, which - // only happens after the POST). This is the regression guard for the - // open-gate deadlock. - stubEventSource(); + resetPanelChatStoreForTests(); + eventSources = stubEventSource(); createSessionMock.mockResolvedValue({ data: { @@ -185,6 +147,62 @@ describe("LighthouseV2ChatPage", () => { ).toHaveAttribute("href", "/lighthouse/settings"); }); + it("renders the empty-state headline with correct wording", () => { + // Given / When + renderPage(); + + // Then + expect( + screen.getByText("Find and remediate what actually matters."), + ).toBeInTheDocument(); + }); + + it("continues using the panel chat store on the full-page surface", () => { + // Given: the panel owns an in-progress new chat with a draft + const panelStore = getOrCreatePanelChatStore({ + configurations, + modelsByProvider, + supportedProviders, + }); + panelStore.getState().setInput("Draft from the side panel"); + + // When + renderPage(); + + // Then: the page owns the same live store, not a stale server snapshot + const input = screen.getByRole("textbox", { name: "Message" }); + expect(input).toHaveValue("Draft from the side panel"); + act(() => panelStore.getState().setInput("Updated after navigation")); + expect(input).toHaveValue("Updated after navigation"); + }); + + it("enables session URL sync after claiming a new panel chat", async () => { + // Given: the panel owns a new chat before full-page navigation + const user = userEvent.setup(); + getOrCreatePanelChatStore({ + configurations, + modelsByProvider, + supportedProviders, + }); + const replaceStateSpy = vi.spyOn(window.history, "replaceState"); + renderPage(); + + // When: the first page message creates its session + await user.type( + screen.getByRole("textbox", { name: "Message" }), + ["Summarize findings", "{Enter}"].join(""), + ); + + // Then: the claimed panel store now follows the full-page URL contract + await waitFor(() => + expect(replaceStateSpy).toHaveBeenCalledWith( + window.history.state, + "", + "/lighthouse?session=session-1", + ), + ); + }); + it("shows the current OpenAI model without a selector when OpenAI is the only connected provider", () => { // Given / When renderPage({ @@ -208,7 +226,7 @@ describe("LighthouseV2ChatPage", () => { expect(within(currentModel).getByText("GPT-5.1")).toBeInTheDocument(); }); - it("defaults to gpt-5.5 when OpenAI has no remembered model", () => { + it("defaults to gpt-5.6-terra when OpenAI has no remembered model", () => { // Given / When renderPage({ configurations: [ @@ -216,15 +234,20 @@ describe("LighthouseV2ChatPage", () => { { ...configurations[1], connected: false }, ], modelsByProvider: { - openai: [model("gpt-4.1", "GPT-4.1"), model("gpt-5.5", "GPT-5.5")], + openai: [ + model("gpt-4.1", "GPT-4.1"), + model("gpt-5.6-terra", "GPT-5.6 Terra"), + ], bedrock: [model("anthropic.claude-4")], "openai-compatible": [model("llama-3.3")], }, }); // Then - const currentModel = screen.getByLabelText("Current model: OpenAI GPT-5.5"); - expect(within(currentModel).getByText("GPT-5.5")).toBeInTheDocument(); + const currentModel = screen.getByLabelText( + "Current model: OpenAI GPT-5.6 Terra", + ); + expect(within(currentModel).getByText("GPT-5.6 Terra")).toBeInTheDocument(); }); it("uses the AWS onboarding quick prompt instead of the docs prompt", async () => { @@ -251,7 +274,7 @@ describe("LighthouseV2ChatPage", () => { it("prefills the overview remediation prompt without starting a conversation", () => { // Given const initialPrompt = - "Find and guide me to remediate which actually matters. What do I have to do today to be secure?"; + "Find and guide me to remediate what actually matters. What do I have to do today to be secure?"; // When renderPage({ initialPrompt }); @@ -688,6 +711,28 @@ describe("LighthouseV2ChatPage", () => { expect(screen.getByText("Existing answer")).toBeInTheDocument(); }); + it("lets the user draft the next message while a response is streaming, without sending it", async () => { + // Given: a message is in flight (spinner replaces the send button) + const user = userEvent.setup(); + renderPage(); + await user.type( + screen.getByRole("textbox", { name: "Message" }), + ["Summarize findings", "{Enter}"].join(""), + ); + await waitFor(() => expect(sendMessageMock).toHaveBeenCalledTimes(1)); + expect( + screen.getByRole("status", { name: "Generating response" }), + ).toBeInTheDocument(); + + // When: the user types a follow-up and presses Enter mid-stream + const input = screen.getByRole("textbox", { name: "Message" }); + await user.type(input, ["Next question", "{Enter}"].join("")); + + // Then: the draft is kept in the input and no second message is sent + expect(input).toHaveValue("Next question"); + expect(sendMessageMock).toHaveBeenCalledTimes(1); + }); + it("surfaces a connection error when the stream closes without retrying", async () => { // Given const user = userEvent.setup(); diff --git a/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.tsx b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.tsx index 3062874649..14b1e18b9d 100644 --- a/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.tsx +++ b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.tsx @@ -1,61 +1,33 @@ "use client"; -import { type SubmitEvent, useRef, useState } from "react"; +import { useState } from "react"; import { - createLighthouseV2Session, - getLighthouseV2Messages, - sendLighthouseV2Message, - updateLighthouseV2Configuration, -} from "@/app/(prowler)/lighthouse/_actions"; + createLighthouseChatStore, + type LighthouseChatStore, +} from "@/app/(prowler)/lighthouse/_lib/chat-store"; import { - Conversation, - ConversationContent, - ConversationScrollButton, -} from "@/app/(prowler)/lighthouse/_components/ai-elements/conversation"; + getPanelChatStoreForSession, + isPanelChatStore, +} from "@/app/(prowler)/lighthouse/_lib/panel-chat-store"; import { - createInitialLighthouseV2StreamState, - type LighthouseV2StreamState, - reduceLighthouseV2Event, -} from "@/app/(prowler)/lighthouse/_lib/event-reducer"; -import { - buildOptimisticMessage, - buildSessionTitle, -} from "@/app/(prowler)/lighthouse/_lib/messages"; -import { - buildLighthouseV2ModelSelectionValue, - type LighthouseV2ModelSelection, - parseLighthouseV2ModelSelectionValue, -} from "@/app/(prowler)/lighthouse/_lib/model-selection"; -import { - LIGHTHOUSE_V2_NEW_CHAT_EVENT, - LIGHTHOUSE_V2_SESSION_ARCHIVED_EVENT, - notifyLighthouseV2SessionsChanged, + onLighthouseV2NewChat, + onLighthouseV2SessionArchived, } from "@/app/(prowler)/lighthouse/_lib/session-events"; -import { parseStreamEvent } from "@/app/(prowler)/lighthouse/_lib/stream-event-parser"; -import { buildLighthouseV2StreamUrl } from "@/app/(prowler)/lighthouse/_lib/stream-url"; -import { - LIGHTHOUSE_V2_PROVIDER_TYPE, - LIGHTHOUSE_V2_SSE_EVENT, - type LighthouseV2Configuration, - type LighthouseV2Message, - type LighthouseV2ProviderType, - type LighthouseV2SSEEvent, - type LighthouseV2SupportedModel, - type LighthouseV2SupportedProvider, +import type { + LighthouseV2Configuration, + LighthouseV2Message, + LighthouseV2ProviderType, + LighthouseV2SupportedModel, + LighthouseV2SupportedProvider, } from "@/app/(prowler)/lighthouse/_types"; -import { Card } from "@/components/shadcn"; -import { - Combobox, - type ComboboxGroup, -} from "@/components/shadcn/combobox/combobox"; import { useMountEffect } from "@/hooks/use-mount-effect"; -import { ProviderIcon } from "../config/provider-icon"; -import { ChatComposerPanel } from "./composer"; -import { ChatEmptyState } from "./empty-state"; -import { MessageBubble } from "./message-bubble"; -import { StreamingAssistantMessage } from "./streaming-message"; +import { LighthouseChatStoreProvider } from "./lighthouse-chat-store-provider"; +import { + LIGHTHOUSE_CHAT_SURFACE, + LighthouseV2ChatView, +} from "./lighthouse-v2-chat-view"; interface LighthouseV2ChatPageProps { configurations: LighthouseV2Configuration[]; @@ -79,572 +51,61 @@ export function LighthouseV2ChatPage({ initialPrompt, initialError, }: LighthouseV2ChatPageProps) { - const eventSourceRef = useRef(null); - const connectedConfigurations = configurations.filter( - (configuration) => configuration.connected === true, - ); - const initialModelSelection = resolveInitialModelSelection( - connectedConfigurations, - modelsByProvider, - ); - const [selectedModelSelection, setSelectedModelSelection] = - useState(initialModelSelection); - const [modelPreferenceSaving, setModelPreferenceSaving] = useState(false); - const [activeSessionId, setActiveSessionId] = useState( - initialSessionId ?? null, - ); - // Mirror for window listeners registered on mount, whose closures would - // otherwise keep the first render's activeSessionId. - const activeSessionIdRef = useRef(initialSessionId ?? null); - const [messages, setMessages] = useState(initialMessages); - const [input, setInput] = useState(initialPrompt ?? ""); - const [feedback, setFeedback] = useState(initialError ?? null); - const [blockedByConflict, setBlockedByConflict] = useState(false); - const [isSubmitting, setIsSubmitting] = useState(false); - const [lastSubmittedText, setLastSubmittedText] = useState( - null, - ); - const [streamState, setStreamState] = useState(() => - createInitialLighthouseV2StreamState(), - ); - const selectedConfiguration = selectedModelSelection - ? connectedConfigurations.find( - (configuration) => - configuration.providerType === selectedModelSelection.providerType, - ) - : undefined; - const modelSelectorGroups = buildModelSelectorGroups( - connectedConfigurations, - modelsByProvider, - supportedProviders, - ); - const showStaticOpenAIModel = - isOnlyConnectedProvider( - connectedConfigurations, - LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI, - ) && - selectedModelSelection?.providerType === LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI; - const selectedModelLabel = selectedModelSelection - ? getModelSelectionLabel(selectedModelSelection, modelsByProvider) - : "No model selected"; - const selectedProviderName = selectedModelSelection - ? getProviderDisplayName( - selectedModelSelection.providerType, - supportedProviders, - ) - : "OpenAI"; - const selectedModelValue = selectedModelSelection - ? buildLighthouseV2ModelSelectionValue( - selectedModelSelection.providerType, - selectedModelSelection.modelId, - ) - : ""; - - const canSend = - selectedConfiguration?.connected === true && - Boolean(selectedModelSelection?.modelId) && - !streamState.activeTaskId && - !blockedByConflict && - !isSubmitting; - - const refreshMessages = async (sessionId: string): Promise => { - const result = await getLighthouseV2Messages(sessionId); - // The fetch is async, so a reset (new chat, or archiving this session) can - // land while it is in flight. Drop the stale result instead of repopulating - // a chat that no longer points at this session. - if (sessionId !== activeSessionIdRef.current) return false; - if ("data" in result) { - setMessages(result.data); - return true; - } - return false; - }; - - const closeStream = () => { - eventSourceRef.current?.close(); - eventSourceRef.current = null; - }; - - const handleTerminalEvent = async ( - sessionId: string, - event: LighthouseV2SSEEvent, - ) => { - if ( - event.type === LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_END || - event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR - ) { - closeStream(); - setBlockedByConflict(false); - if (event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR) { - setFeedback(event.detail || "Agent run failed."); - } - const refreshed = await refreshMessages(sessionId); - if (refreshed) { - setStreamState(createInitialLighthouseV2StreamState()); - } - notifyLighthouseV2SessionsChanged(); - } - }; - - const startStream = (streamUrl: string, sessionId: string) => { - closeStream(); - const source = new EventSource(streamUrl); - eventSourceRef.current = source; - - const applyEvent = (event: LighthouseV2SSEEvent) => { - setStreamState((current) => reduceLighthouseV2Event(current, event)); - void handleTerminalEvent(sessionId, event); - }; - - source.addEventListener("message.delta", (event) => - applyEvent( - parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_DELTA), - ), + // Navigation from the side panel transfers its live store so drafts, + // streamed output and the open EventSource continue without a snapshot gap. + // Direct/session-mismatched navigation builds the normal page-owned store. + const [store] = useState(() => { + const panelStore = + initialPrompt === undefined + ? getPanelChatStoreForSession(initialSessionId) + : null; + return ( + panelStore ?? + createLighthouseChatStore({ + config: { configurations, modelsByProvider, supportedProviders }, + syncUrlToSession: true, + initialSessionId, + initialMessages, + initialInput: initialPrompt, + initialError, + }) ); - source.addEventListener("tool_call.start", (event) => - applyEvent( - parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.TOOL_CALL_START), - ), - ); - source.addEventListener("tool_call.end", (event) => - applyEvent( - parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.TOOL_CALL_END), - ), - ); - source.addEventListener("message.end", (event) => - applyEvent(parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_END)), - ); - source.addEventListener("error", (event) => { - if (event instanceof MessageEvent) { - applyEvent(parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.ERROR)); - } - }); - // The browser fires `onerror` both on a transient drop (it auto-reconnects) - // and on a non-retryable failure such as a 401/404 on the SSE GET. Only the - // latter leaves the source CLOSED, so surface a connection error there and - // treat everything else as a reconnect. - source.onerror = () => { - if (eventSourceRef.current !== source) return; - if (source.readyState === EventSource.CLOSED) { - closeStream(); - setFeedback("Unable to connect to the response stream."); - } - setStreamState((current) => - reduceLighthouseV2Event(current, { type: "disconnect" }), - ); - }; - }; + }); + const reusesPanelStore = isPanelChatStore(store); - const ensureSession = async (text: string) => { - if (activeSessionId) { - return activeSessionId; + // A reused panel store returns to panel URL semantics when the page leaves; + // a page-owned store closes its EventSource as before. + useMountEffect(() => { + if (reusesPanelStore) { + store.getState().setSessionUrlSyncEnabled(true); } - - const title = buildSessionTitle(text); - const result = await createLighthouseV2Session(title); - if ("error" in result) { - setFeedback(result.error); - return null; - } - - // Update the URL in place (not router.push) so the force-dynamic server - // component is NOT re-run mid-submit. A re-run would change `key` in - // page.tsx and remount this component, tearing down the open EventSource. - replaceLighthouseV2SessionUrl(result.data.id); - setActiveSessionId(result.data.id); - activeSessionIdRef.current = result.data.id; - notifyLighthouseV2SessionsChanged(); - return result.data.id; - }; - - const submitMessage = async (text: string) => { - const trimmedText = text.trim(); - if (!trimmedText) return; - if (!selectedModelSelection) { - setFeedback("Select a model before sending a message."); - return; - } - if (!canSend) return; - - setIsSubmitting(true); - try { - const sessionId = await ensureSession(trimmedText); - if (!sessionId) return; - - const provisionalTaskId = `pending-${Date.now()}`; - setFeedback(null); - setBlockedByConflict(false); - setLastSubmittedText(trimmedText); - setInput(""); - setMessages((current) => [ - ...current, - buildOptimisticMessage("user", trimmedText), - ]); - setStreamState(createInitialLighthouseV2StreamState(provisionalTaskId)); - - // Subscribe to the same-origin SSE proxy BEFORE sending the message: the - // backend has no replay buffer, so the listener must be attached before - // the worker starts emitting. - startStream(buildLighthouseV2StreamUrl(sessionId), sessionId); - - const result = await sendLighthouseV2Message({ - sessionId, - text: trimmedText, - provider: selectedModelSelection.providerType, - model: selectedModelSelection.modelId, - }); - - if ("error" in result) { - closeStream(); - setStreamState(createInitialLighthouseV2StreamState()); - setFeedback(result.error); - if (result.status === 409) { - setBlockedByConflict(true); - await refreshMessages(sessionId); - } + return () => { + if (reusesPanelStore) { + store.getState().setSessionUrlSyncEnabled(false); return; } - - setStreamState((current) => - current.activeTaskId === provisionalTaskId - ? { ...current, activeTaskId: result.data.task.id } - : current, - ); - notifyLighthouseV2SessionsChanged(); - } finally { - setIsSubmitting(false); - } - }; - - const handleModelValueChange = (value: string) => { - const selection = parseLighthouseV2ModelSelectionValue(value); - if (!selection) return; - void handleModelSelectionChange(selection); - }; - - const handleModelSelectionChange = async ( - selection: LighthouseV2ModelSelection, - ) => { - // The selection drives the model used for the next message, so it stays - // applied even if persisting it as the provider's default model fails — - // reverting it would make a connected provider unusable when the save 4xxs. - setSelectedModelSelection(selection); - setFeedback(null); - - const configId = connectedConfigurations.find( - (configuration) => configuration.providerType === selection.providerType, - )?.id; - if (!configId) return; - - setModelPreferenceSaving(true); - - const result = await updateLighthouseV2Configuration(configId, { - defaultModel: selection.modelId, - }); - - setModelPreferenceSaving(false); - - if ("error" in result) { - setFeedback(result.error); - } - }; - - const handleSubmit = (event: SubmitEvent) => { - event.preventDefault(); - void submitMessage(input); - }; - - // Close any open EventSource when the chat unmounts (e.g. route/session change). - useMountEffect(() => { - return () => closeStream(); + store.getState().destroy(); + }; }); - const resetToNewChat = () => { - closeStream(); - setActiveSessionId(null); - activeSessionIdRef.current = null; - setMessages([]); - setInput(""); - setFeedback(null); - setBlockedByConflict(false); - setIsSubmitting(false); - setLastSubmittedText(null); - setStreamState(createInitialLighthouseV2StreamState()); - replaceLighthouseV2SessionUrl(null); - }; - // The sidebar "+" can't rely on routing to reset the latest conversation (its // URL was set via replaceState, invisible to Next's router), so reset in place. useMountEffect(() => { - window.addEventListener(LIGHTHOUSE_V2_NEW_CHAT_EVENT, resetToNewChat); - return () => - window.removeEventListener(LIGHTHOUSE_V2_NEW_CHAT_EVENT, resetToNewChat); + const unsubscribeNewChat = onLighthouseV2NewChat(() => + store.getState().resetToNewChat(), + ); + const unsubscribeSessionArchived = onLighthouseV2SessionArchived( + (sessionId) => store.getState().handleSessionArchived(sessionId), + ); + return () => { + unsubscribeNewChat(); + unsubscribeSessionArchived(); + }; }); - // Archiving deletes the session; when it's the open one, fall back to a new - // chat instead of leaving a dead conversation and its URL on screen. - useMountEffect(() => { - const handleSessionArchived = (event: Event) => { - const archivedId = (event as CustomEvent<{ sessionId: string }>).detail - ?.sessionId; - if (archivedId && archivedId === activeSessionIdRef.current) { - resetToNewChat(); - } - }; - - window.addEventListener( - LIGHTHOUSE_V2_SESSION_ARCHIVED_EVENT, - handleSessionArchived, - ); - return () => - window.removeEventListener( - LIGHTHOUSE_V2_SESSION_ARCHIVED_EVENT, - handleSessionArchived, - ); - }); - - const hasLiveAssistantActivity = - Boolean(streamState.activeTaskId) || - Boolean(streamState.assistantText) || - streamState.toolCalls.length > 0; - const hasConversation = messages.length > 0 || hasLiveAssistantActivity; - - const composerPanelProps = { - feedback, - canRetry: - streamState.status === "disconnected" && lastSubmittedText !== null, - onRetry: () => - lastSubmittedText ? void submitMessage(lastSubmittedText) : undefined, - onDismissFeedback: () => setFeedback(null), - canSend, - input, - isStreaming: Boolean(streamState.activeTaskId), - modelSelector: showStaticOpenAIModel ? ( - - ) : ( -
- -
- ), - selectedConfigurationConnected: selectedConfiguration?.connected === true, - onInputChange: setInput, - onSubmit: handleSubmit, - onSubmitText: submitMessage, - }; - return ( - - {hasConversation ? ( -
-
- - - {messages.map((message) => ( - - ))} - {hasLiveAssistantActivity && ( - - )} - - - -
-
-
-
- -
-
-
- ) : ( - - )} - - ); -} - -function replaceLighthouseV2SessionUrl(sessionId: string | null) { - const url = sessionId - ? `/lighthouse?session=${encodeURIComponent(sessionId)}` - : "/lighthouse"; - - window.history.replaceState(window.history.state, "", url); -} - -function CurrentModelDisplay({ - provider, - providerName, - modelName, -}: { - provider: LighthouseV2ProviderType; - providerName: string; - modelName: string; -}) { - return ( -
- - - {providerName} - - - {modelName} - -
- ); -} - -// Fixed precedence used to pick which connected provider opens the chat. Any -// provider outside this list keeps its relative order behind these. -const LIGHTHOUSE_V2_PROVIDER_PRIORITY = [ - LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI, - LIGHTHOUSE_V2_PROVIDER_TYPE.BEDROCK, - LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI_COMPATIBLE, -] as const; - -// Fallback model per provider when the configuration has no remembered model. -const LIGHTHOUSE_V2_PREFERRED_DEFAULT_MODEL: Partial< - Record -> = { - [LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI]: "gpt-5.5", -}; - -function resolveInitialModelSelection( - connectedConfigurations: LighthouseV2Configuration[], - modelsByProvider: Record< - LighthouseV2ProviderType, - LighthouseV2SupportedModel[] - >, -): LighthouseV2ModelSelection | null { - const priorityIndex = (providerType: LighthouseV2ProviderType) => { - const index = LIGHTHOUSE_V2_PROVIDER_PRIORITY.indexOf(providerType); - return index === -1 ? LIGHTHOUSE_V2_PROVIDER_PRIORITY.length : index; - }; - // Stable sort keeps providers outside the priority list in their original order. - const orderedConfigurations = [...connectedConfigurations].sort( - (a, b) => priorityIndex(a.providerType) - priorityIndex(b.providerType), - ); - - for (const configuration of orderedConfigurations) { - const providerModels = modelsByProvider[configuration.providerType] ?? []; - if (providerModels.length === 0) continue; - // Prefer the provider's remembered model when it is still supported, then - // the provider's preferred default, then the first supported model. - const rememberedModel = providerModels.find( - (model) => model.id === configuration.defaultModel, - ); - const preferredModel = providerModels.find( - (model) => - model.id === - LIGHTHOUSE_V2_PREFERRED_DEFAULT_MODEL[configuration.providerType], - ); - return { - providerType: configuration.providerType, - modelId: (rememberedModel ?? preferredModel ?? providerModels[0]).id, - }; - } - - return null; -} - -function buildModelSelectorGroups( - connectedConfigurations: LighthouseV2Configuration[], - modelsByProvider: Record< - LighthouseV2ProviderType, - LighthouseV2SupportedModel[] - >, - supportedProviders: LighthouseV2SupportedProvider[], -): ComboboxGroup[] { - const groups: ComboboxGroup[] = []; - - for (const provider of supportedProviders) { - const configuration = connectedConfigurations.find( - (item) => item.providerType === provider.id, - ); - if (!configuration) continue; - - const options = (modelsByProvider[configuration.providerType] ?? []).map( - (model) => ({ - value: buildLighthouseV2ModelSelectionValue( - configuration.providerType, - model.id, - ), - label: model.name, - }), - ); - - if (options.length === 0) continue; - - groups.push({ - heading: provider.name, - options, - }); - } - - return groups; -} - -function isOnlyConnectedProvider( - connectedConfigurations: LighthouseV2Configuration[], - providerType: LighthouseV2ProviderType, -) { - return ( - connectedConfigurations.length === 1 && - connectedConfigurations[0]?.providerType === providerType - ); -} - -function getModelSelectionLabel( - selection: LighthouseV2ModelSelection, - modelsByProvider: Record< - LighthouseV2ProviderType, - LighthouseV2SupportedModel[] - >, -) { - return ( - modelsByProvider[selection.providerType]?.find( - (model) => model.id === selection.modelId, - )?.name ?? selection.modelId - ); -} - -function getProviderDisplayName( - providerType: LighthouseV2ProviderType, - supportedProviders: LighthouseV2SupportedProvider[], -) { - return ( - supportedProviders.find((provider) => provider.id === providerType)?.name ?? - providerType + + + ); } diff --git a/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-view.tsx b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-view.tsx new file mode 100644 index 0000000000..9b687383bc --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-view.tsx @@ -0,0 +1,340 @@ +"use client"; + +import { type ReactNode, type SubmitEvent } from "react"; + +import { + Conversation, + ConversationContent, + ConversationScrollButton, +} from "@/app/(prowler)/lighthouse/_components/ai-elements/conversation"; +import { selectLighthouseChatCanSend } from "@/app/(prowler)/lighthouse/_lib/chat-store"; +import { LIGHTHOUSE_V2_STREAM_STATUS } from "@/app/(prowler)/lighthouse/_lib/event-reducer"; +import { + buildLighthouseV2ModelSelectionValue, + type LighthouseV2ModelSelection, + parseLighthouseV2ModelSelectionValue, +} from "@/app/(prowler)/lighthouse/_lib/model-selection"; +import { + LIGHTHOUSE_V2_PROVIDER_TYPE, + type LighthouseV2Configuration, + type LighthouseV2ProviderType, + type LighthouseV2SupportedModel, + type LighthouseV2SupportedProvider, +} from "@/app/(prowler)/lighthouse/_types"; +import { Card } from "@/components/shadcn"; +import { + Combobox, + type ComboboxGroup, +} from "@/components/shadcn/combobox/combobox"; +import { Skeleton } from "@/components/shadcn/skeleton/skeleton"; + +import { ProviderIcon } from "../config/provider-icon"; + +import { ChatComposerPanel } from "./composer"; +import { ChatEmptyState } from "./empty-state"; +import { useLighthouseChatStore } from "./lighthouse-chat-store-provider"; +import { MessageBubble } from "./message-bubble"; +import { StreamingAssistantMessage } from "./streaming-message"; + +export const LIGHTHOUSE_CHAT_SURFACE = { + PAGE: "page", + PANEL: "panel", +} as const; + +export type LighthouseChatSurface = + (typeof LIGHTHOUSE_CHAT_SURFACE)[keyof typeof LIGHTHOUSE_CHAT_SURFACE]; + +interface LighthouseV2ChatViewProps { + surface: LighthouseChatSurface; + emptyStateFooter?: ReactNode; +} + +export function LighthouseV2ChatView({ + surface, + emptyStateFooter, +}: LighthouseV2ChatViewProps) { + // Whole-store subscription is intentional: the view renders most of the state and selectLighthouseChatCanSend takes full state. + const state = useLighthouseChatStore((current) => current); + const { + config, + messages, + streamState, + input, + feedback, + isLoadingSession, + lastSubmittedText, + selectedModelSelection, + modelPreferenceSaving, + setInput, + dismissFeedback, + selectModel, + submitMessage, + } = state; + const { modelsByProvider, supportedProviders } = config; + const connectedConfigurations = config.configurations.filter( + (configuration) => configuration.connected === true, + ); + + const selectedConfiguration = selectedModelSelection + ? connectedConfigurations.find( + (configuration) => + configuration.providerType === selectedModelSelection.providerType, + ) + : undefined; + const modelSelectorGroups = buildModelSelectorGroups( + connectedConfigurations, + modelsByProvider, + supportedProviders, + ); + const showStaticOpenAIModel = + isOnlyConnectedProvider( + connectedConfigurations, + LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI, + ) && + selectedModelSelection?.providerType === LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI; + const selectedModelLabel = selectedModelSelection + ? getModelSelectionLabel(selectedModelSelection, modelsByProvider) + : "No model selected"; + const selectedProviderName = selectedModelSelection + ? getProviderDisplayName( + selectedModelSelection.providerType, + supportedProviders, + ) + : "OpenAI"; + const selectedModelValue = selectedModelSelection + ? buildLighthouseV2ModelSelectionValue( + selectedModelSelection.providerType, + selectedModelSelection.modelId, + ) + : ""; + + const canSend = selectLighthouseChatCanSend(state); + + const handleModelValueChange = (value: string) => { + const selection = parseLighthouseV2ModelSelectionValue(value); + if (!selection) return; + void selectModel(selection); + }; + + const handleSubmit = (event: SubmitEvent) => { + event.preventDefault(); + void submitMessage(input); + }; + + const hasLiveAssistantActivity = + Boolean(streamState.activeTaskId) || + Boolean(streamState.assistantText) || + streamState.toolCalls.length > 0; + const hasConversation = messages.length > 0 || hasLiveAssistantActivity; + + const composerPanelProps = { + feedback, + canRetry: + streamState.status === LIGHTHOUSE_V2_STREAM_STATUS.DISCONNECTED && + lastSubmittedText !== null, + onRetry: () => + lastSubmittedText ? void submitMessage(lastSubmittedText) : undefined, + onDismissFeedback: dismissFeedback, + canSend, + input, + isStreaming: Boolean(streamState.activeTaskId), + modelSelector: showStaticOpenAIModel ? ( + + ) : ( +
+ +
+ ), + selectedConfigurationConnected: selectedConfiguration?.connected === true, + onInputChange: setInput, + onSubmit: handleSubmit, + onSubmitText: submitMessage, + }; + + const chatBody = isLoadingSession ? ( + + ) : hasConversation ? ( +
+
+ + + {messages.map((message) => ( + + ))} + {hasLiveAssistantActivity && ( + + )} + + + +
+
+
+
+ +
+
+
+ ) : ( + + ); + + if (surface === LIGHTHOUSE_CHAT_SURFACE.PAGE) { + return ( + + {chatBody} + + ); + } + + return ( +
+ {chatBody} +
+ ); +} + +function SessionLoadingState() { + return ( +
+ + + + +
+ ); +} + +interface CurrentModelDisplayProps { + provider: LighthouseV2ProviderType; + providerName: string; + modelName: string; +} + +function CurrentModelDisplay({ + provider, + providerName, + modelName, +}: CurrentModelDisplayProps) { + return ( +
+ + + {providerName} + + + {modelName} + +
+ ); +} + +function buildModelSelectorGroups( + connectedConfigurations: LighthouseV2Configuration[], + modelsByProvider: Record< + LighthouseV2ProviderType, + LighthouseV2SupportedModel[] + >, + supportedProviders: LighthouseV2SupportedProvider[], +): ComboboxGroup[] { + const groups: ComboboxGroup[] = []; + + for (const provider of supportedProviders) { + const configuration = connectedConfigurations.find( + (item) => item.providerType === provider.id, + ); + if (!configuration) continue; + + const options = (modelsByProvider[configuration.providerType] ?? []).map( + (model) => ({ + value: buildLighthouseV2ModelSelectionValue( + configuration.providerType, + model.id, + ), + label: model.name, + }), + ); + + if (options.length === 0) continue; + + groups.push({ + heading: provider.name, + options, + }); + } + + return groups; +} + +function isOnlyConnectedProvider( + connectedConfigurations: LighthouseV2Configuration[], + providerType: LighthouseV2ProviderType, +) { + return ( + connectedConfigurations.length === 1 && + connectedConfigurations[0]?.providerType === providerType + ); +} + +function getModelSelectionLabel( + selection: LighthouseV2ModelSelection, + modelsByProvider: Record< + LighthouseV2ProviderType, + LighthouseV2SupportedModel[] + >, +) { + return ( + modelsByProvider[selection.providerType]?.find( + (model) => model.id === selection.modelId, + )?.name ?? selection.modelId + ); +} + +function getProviderDisplayName( + providerType: LighthouseV2ProviderType, + supportedProviders: LighthouseV2SupportedProvider[], +) { + return ( + supportedProviders.find((provider) => provider.id === providerType)?.name ?? + providerType + ); +} diff --git a/ui/app/(prowler)/lighthouse/_components/chat/message-bubble.test.tsx b/ui/app/(prowler)/lighthouse/_components/chat/message-bubble.test.tsx index 8b6855392f..ab11429774 100644 --- a/ui/app/(prowler)/lighthouse/_components/chat/message-bubble.test.tsx +++ b/ui/app/(prowler)/lighthouse/_components/chat/message-bubble.test.tsx @@ -51,7 +51,7 @@ describe("MessageBubble", () => { // Given const orderedMessage = buildAssistantMessage([ textPart("part-1", "Voy a buscar los findings por severidad"), - toolCallPart("part-2", "prowler_app_search_security_findings"), + toolCallPart("part-2", "prowler_search_security_findings"), textPart("part-3", "Ahora voy a buscar en los criticos"), ]); diff --git a/ui/app/(prowler)/lighthouse/_components/config/configuration-form.tsx b/ui/app/(prowler)/lighthouse/_components/config/configuration-form.tsx index 5bb6780f33..6ce5bab8b2 100644 --- a/ui/app/(prowler)/lighthouse/_components/config/configuration-form.tsx +++ b/ui/app/(prowler)/lighthouse/_components/config/configuration-form.tsx @@ -24,6 +24,7 @@ import { trimToNullable, } from "@/app/(prowler)/lighthouse/_lib/config"; import { formatLastChecked } from "@/app/(prowler)/lighthouse/_lib/format"; +import { notifyLighthouseV2ConfigurationsChanged } from "@/app/(prowler)/lighthouse/_lib/session-events"; import { type LighthouseV2Configuration, type LighthouseV2ConfigurationUpdateInput, @@ -138,6 +139,7 @@ export function LighthouseV2ConfigurationForm({ form.reset(getFormDefaults(result.data)); onConfigurationSaved(result.data); + notifyLighthouseV2ConfigurationsChanged(); if (shouldTestAfterSave) { await runConnectionTest(result.data.id); } @@ -178,6 +180,7 @@ export function LighthouseV2ConfigurationForm({ setDeleteOpen(false); form.reset(EMPTY_FORM_VALUES); onConfigurationDeleted(configuration.id); + notifyLighthouseV2ConfigurationsChanged(); } catch { onFeedback({ title: "Configuration not removed", diff --git a/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat-skeleton.tsx b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat-skeleton.tsx new file mode 100644 index 0000000000..4a75f6c1e9 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat-skeleton.tsx @@ -0,0 +1,71 @@ +import { Skeleton } from "@/components/shadcn/skeleton/skeleton"; +import { cn } from "@/lib/utils"; + +// 1:1 skeleton of the panel chat empty state (logo, headline, composer, +// suggestion chips, recent chats). Kept in its own file with light imports: +// the side-panel shell uses it as the Suspense fallback while the real +// (lazy) chat bundle downloads, so it must not pull that bundle in. +export function LighthousePanelChatSkeleton() { + return ( +
+ {/* Lighthouse logo */} + + + {/* Headline + subline */} +
+ + +
+ + {/* Composer: textarea, then model selector + send button row */} +
+ +
+ + +
+
+ + {/* "Try Lighthouse AI for..." suggestion chips */} +
+ +
+ + + + +
+
+ + {/* Recent chats: label, search + new-chat row, session rows */} +
+ +
+ + +
+
+ + + +
+
+
+ ); +} + +interface SessionRowSkeletonProps { + titleWidth: string; +} + +function SessionRowSkeleton({ titleWidth }: SessionRowSkeletonProps) { + return ( +
+ + +
+ ); +} diff --git a/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat.test.tsx b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat.test.tsx new file mode 100644 index 0000000000..f903bd87e2 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat.test.tsx @@ -0,0 +1,464 @@ +import { act, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { type ReactNode } from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { resetPanelChatStoreForTests } from "@/app/(prowler)/lighthouse/_lib/panel-chat-store"; +import { notifyLighthouseV2ConfigurationsChanged } from "@/app/(prowler)/lighthouse/_lib/session-events"; +import { stubEventSource } from "@/app/(prowler)/lighthouse/_lib/testing/event-source-mock"; +import type { + LighthouseV2Configuration, + LighthouseV2Session, + LighthouseV2SupportedModel, +} from "@/app/(prowler)/lighthouse/_types"; + +import { + LighthousePanelChat, + resetPanelChatConfigCacheForTests, +} from "./lighthouse-panel-chat"; +import { LighthousePanelHeaderActions } from "./lighthouse-panel-header-actions"; + +const { + getConfigurationsMock, + getSupportedProvidersMock, + getSupportedModelsMock, + getSessionsMock, + archiveSessionMock, + createSessionMock, + getMessagesMock, + sendMessageMock, + updateConfigurationMock, +} = vi.hoisted(() => ({ + getConfigurationsMock: vi.fn(), + getSupportedProvidersMock: vi.fn(), + getSupportedModelsMock: vi.fn(), + getSessionsMock: vi.fn(), + archiveSessionMock: vi.fn(), + createSessionMock: vi.fn(), + getMessagesMock: vi.fn(), + sendMessageMock: vi.fn(), + updateConfigurationMock: vi.fn(), +})); + +vi.mock("@/app/(prowler)/lighthouse/_actions", () => ({ + getLighthouseV2Configurations: getConfigurationsMock, + getLighthouseV2SupportedProviders: getSupportedProvidersMock, + getLighthouseV2SupportedModels: getSupportedModelsMock, + getLighthouseV2Sessions: getSessionsMock, + archiveLighthouseV2Session: archiveSessionMock, + createLighthouseV2Session: createSessionMock, + getLighthouseV2Messages: getMessagesMock, + sendLighthouseV2Message: sendMessageMock, + updateLighthouseV2Configuration: updateConfigurationMock, +})); + +// Streamdown pulls in shiki/wasm syntax highlighting that doesn't run under +// jsdom; render its text passthrough so message bodies are still assertable. +vi.mock("streamdown", () => ({ + Streamdown: ({ children }: { children: ReactNode }) => <>{children}, + defaultRehypePlugins: { katex: undefined, harden: undefined }, +})); + +const configurations: LighthouseV2Configuration[] = [ + { + id: "config-openai", + providerType: "openai", + baseUrl: null, + defaultModel: "gpt-5.1", + businessContext: "Production account", + connected: true, + connectionLastCheckedAt: "2026-06-24T10:00:00Z", + insertedAt: "2026-06-24T09:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }, +]; + +describe("LighthousePanelChat", () => { + beforeEach(() => { + vi.stubGlobal( + "ResizeObserver", + class ResizeObserver { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); + }, + ); + Object.defineProperty(Element.prototype, "scrollIntoView", { + configurable: true, + value: vi.fn(), + }); + getConfigurationsMock.mockReset(); + getSupportedProvidersMock.mockReset(); + getSupportedModelsMock.mockReset(); + getSessionsMock.mockReset(); + archiveSessionMock.mockReset(); + getMessagesMock.mockReset(); + stubEventSource(); + resetPanelChatStoreForTests(); + resetPanelChatConfigCacheForTests(); + + getConfigurationsMock.mockResolvedValue({ data: configurations }); + getSupportedProvidersMock.mockResolvedValue({ + data: [ + { id: "openai", name: "OpenAI" }, + { id: "bedrock", name: "AWS Bedrock" }, + { id: "openai-compatible", name: "OpenAI Compatible" }, + ], + }); + getSupportedModelsMock.mockResolvedValue({ data: [model("gpt-5.1")] }); + getSessionsMock.mockResolvedValue({ data: [] }); + getMessagesMock.mockResolvedValue({ data: [] }); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("shows a loading skeleton while the config loads", () => { + // Given: a config fetch that never resolves within the assertion window + getConfigurationsMock.mockReturnValue(new Promise(() => {})); + + // When + render(); + + // Then + expect(screen.getByLabelText("Loading Lighthouse AI")).toBeInTheDocument(); + }); + + it("shows the error state with a Retry that reloads the config", async () => { + // Given + getConfigurationsMock.mockResolvedValueOnce({ + error: "Something went wrong.", + status: 500, + }); + const user = userEvent.setup(); + render(); + expect(await screen.findByRole("alert")).toHaveTextContent( + "Something went wrong.", + ); + + // When: retrying after the backend recovers + await user.click(screen.getByRole("button", { name: "Retry" })); + + // Then + expect( + await screen.findByRole("textbox", { name: "Message" }), + ).toBeInTheDocument(); + }); + + it("shows an in-panel connect CTA instead of redirecting when no LLM is connected", async () => { + // Given + getConfigurationsMock.mockResolvedValue({ + data: [{ ...configurations[0], connected: false }], + }); + + // When + render(); + + // Then + expect( + await screen.findByRole("link", { name: "Connect an LLM provider" }), + ).toHaveAttribute("href", "/lighthouse/settings"); + }); + + it("renders the chat composer and recent chats once ready", async () => { + // Given + getSessionsMock.mockResolvedValue({ + data: [session("session-1", "Counting critical findings")], + }); + + // When + render(); + + // Then: composer is live and the empty state lists recent chats + expect( + await screen.findByRole("textbox", { name: "Message" }), + ).toBeInTheDocument(); + expect(screen.getByText("Recent chats")).toBeInTheDocument(); + expect( + await screen.findByText("Counting critical findings"), + ).toBeInTheDocument(); + }); + + it("opens a recent chat in place without navigating", async () => { + // Given + const user = userEvent.setup(); + const replaceStateSpy = vi.spyOn(window.history, "replaceState"); + getSessionsMock.mockResolvedValue({ + data: [session("session-1", "Counting critical findings")], + }); + getMessagesMock.mockResolvedValue({ + data: [ + { + id: "message-1", + role: "assistant", + model: null, + tokenUsage: null, + insertedAt: "2026-06-25T10:00:00Z", + parts: [ + { + id: "message-1-part", + type: "text", + content: "There are 3 critical findings.", + toolCallOutcome: null, + insertedAt: "2026-06-25T10:00:00Z", + updatedAt: "2026-06-25T10:00:00Z", + }, + ], + }, + ], + }); + render(); + + // When + await user.click( + await screen.findByRole("button", { + name: /^Counting critical findings/, + }), + ); + + // Then: the conversation loads in the panel and the URL never changes + expect( + await screen.findByText("There are 3 critical findings."), + ).toBeInTheDocument(); + expect(replaceStateSpy).not.toHaveBeenCalled(); + replaceStateSpy.mockRestore(); + }); + + it("explains why a new chat is unavailable before the first message", async () => { + // Given + const user = userEvent.setup(); + render(); + const newChatButton = screen.getByRole("button", { name: "New chat" }); + + // When + const disabledTrigger = newChatButton.parentElement; + + // Then + expect(newChatButton).toBeDisabled(); + expect(disabledTrigger).toHaveClass("cursor-not-allowed"); + await user.hover(disabledTrigger!); + expect(await screen.findByRole("tooltip")).toHaveTextContent( + "Send a message before starting a new chat", + ); + }); + + it("opens the active panel conversation on the full-page chat route", async () => { + // Given: the panel starts on a new chat and exposes the full-page action + const user = userEvent.setup(); + getSessionsMock.mockResolvedValue({ + data: [session("session-1", "Counting critical findings")], + }); + render( + <> + + + , + ); + const fullPageLink = await screen.findByRole("link", { + name: "Open Lighthouse AI full page", + }); + expect(fullPageLink).toHaveAttribute("href", "/lighthouse"); + + // When: an existing conversation becomes active in the panel + await user.click( + await screen.findByRole("button", { + name: /^Counting critical findings/, + }), + ); + + // Then: full-page navigation carries the active session in the URL + expect(fullPageLink).toHaveAttribute( + "href", + "/lighthouse?session=session-1", + ); + }); + + it("starts a new chat from the panel header", async () => { + // Given: an existing conversation is open in the panel + const user = userEvent.setup(); + getSessionsMock.mockResolvedValue({ + data: [session("session-1", "Counting critical findings")], + }); + getMessagesMock.mockResolvedValue({ + data: [ + { + id: "message-1", + role: "assistant", + model: null, + tokenUsage: null, + insertedAt: "2026-06-25T10:00:00Z", + parts: [ + { + id: "message-1-part", + type: "text", + content: "There are 3 critical findings.", + toolCallOutcome: null, + insertedAt: "2026-06-25T10:00:00Z", + updatedAt: "2026-06-25T10:00:00Z", + }, + ], + }, + ], + }); + render( + <> +
+ +
+ + , + ); + await screen.findByRole("textbox", { name: "Message" }); + const panelHeader = screen.getByLabelText("Panel header actions"); + const newChatButton = within(panelHeader).getByRole("button", { + name: "New chat", + }); + expect(newChatButton).toBeDisabled(); + + await user.click( + await screen.findByRole("button", { + name: /^Counting critical findings/, + }), + ); + expect( + await screen.findByText("There are 3 critical findings."), + ).toBeInTheDocument(); + expect(newChatButton).toBeEnabled(); + + // When + await user.click(newChatButton); + + // Then + expect( + screen.queryByText("There are 3 critical findings."), + ).not.toBeInTheDocument(); + expect(screen.getByRole("textbox", { name: "Message" })).toHaveValue(""); + expect(newChatButton).toBeDisabled(); + }); + + it("caches the loaded config so a remount skips the skeleton", async () => { + // Given: a first mount that loads successfully + const { unmount } = render(); + await screen.findByRole("textbox", { name: "Message" }); + unmount(); + getConfigurationsMock.mockClear(); + + // When + render(); + + // Then: ready immediately, no refetch + await waitFor(() => + expect( + screen.getByRole("textbox", { name: "Message" }), + ).toBeInTheDocument(), + ); + expect(getConfigurationsMock).not.toHaveBeenCalled(); + }); + + it("reloads models after a transient model-loading failure", async () => { + // Given: configuration loads, but the first model request fails + getSupportedModelsMock.mockResolvedValueOnce({ + error: "Models are temporarily unavailable.", + status: 500, + }); + const { unmount } = render(); + await screen.findByRole("textbox", { name: "Message" }); + unmount(); + getSupportedModelsMock.mockClear(); + + // When: the panel reopens after the model endpoint recovers + render(); + + // Then: the partial ready state is not reused as a successful cache entry + await waitFor(() => expect(getSupportedModelsMock).toHaveBeenCalled()); + }); + + it("removes an archived chat from the recent chats list", async () => { + // Given: one recent chat + const user = userEvent.setup(); + getSessionsMock.mockResolvedValue({ + data: [session("session-1", "Counting critical findings")], + }); + archiveSessionMock.mockResolvedValue({ data: { id: "session-1" } }); + render(); + await screen.findByText("Counting critical findings"); + + // When: archiving it from the panel (hover action + confirm dialog) + getSessionsMock.mockResolvedValue({ data: [] }); + await user.click( + screen.getByRole("button", { + name: "Archive Counting critical findings", + }), + ); + await user.click( + within(await screen.findByRole("dialog")).getByRole("button", { + name: "Archive", + }), + ); + + // Then: the archived chat leaves the list + await waitFor(() => + expect( + screen.queryByText("Counting critical findings"), + ).not.toBeInTheDocument(), + ); + }); + + it("swaps the connect CTA for the chat once a provider is connected", async () => { + // Given: no connected provider yet + getConfigurationsMock.mockResolvedValueOnce({ + data: [{ ...configurations[0], connected: false }], + }); + render(); + await screen.findByRole("link", { name: "Connect an LLM provider" }); + + // When: a provider gets connected on the settings page + act(() => notifyLighthouseV2ConfigurationsChanged()); + + // Then: the panel reloads into the live chat + expect( + await screen.findByRole("textbox", { name: "Message" }), + ).toBeInTheDocument(); + }); + + it("drops the cached config when configurations change while unmounted", async () => { + // Given: a cached config from a previous mount + const { unmount } = render(); + await screen.findByRole("textbox", { name: "Message" }); + unmount(); + getConfigurationsMock.mockClear(); + + // When: config CRUD happens with the panel closed, then it reopens + notifyLighthouseV2ConfigurationsChanged(); + render(); + + // Then: the stale cache is gone and the config reloads + expect( + await screen.findByRole("textbox", { name: "Message" }), + ).toBeInTheDocument(); + expect(getConfigurationsMock).toHaveBeenCalled(); + }); +}); + +function model(id: string, name = id): LighthouseV2SupportedModel { + return { + id, + name, + maxInputTokens: null, + maxOutputTokens: null, + supportsFunctionCalling: null, + supportsVision: null, + supportsReasoning: null, + }; +} + +function session(id: string, title: string): LighthouseV2Session { + return { + id, + title, + isArchived: false, + insertedAt: "2026-06-24T10:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }; +} diff --git a/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat.tsx b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat.tsx new file mode 100644 index 0000000000..371075a088 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-chat.tsx @@ -0,0 +1,341 @@ +"use client"; + +import Link from "next/link"; +import { useState } from "react"; + +import { + archiveLighthouseV2Session, + getLighthouseV2Sessions, +} from "@/app/(prowler)/lighthouse/_actions"; +import { LighthouseV2SessionHistory } from "@/app/(prowler)/lighthouse/_components/history"; +import type { LighthouseChatConfig } from "@/app/(prowler)/lighthouse/_lib/chat-store"; +import { + LIGHTHOUSE_CHAT_CONFIG_STATUS, + loadLighthouseChatConfig, +} from "@/app/(prowler)/lighthouse/_lib/load-chat-config"; +import { + resetPanelChatMessageState, + setPanelChatMessageState, +} from "@/app/(prowler)/lighthouse/_lib/panel-chat-message-state"; +import { + getOrCreatePanelChatStore, + resetPanelChatStore, +} from "@/app/(prowler)/lighthouse/_lib/panel-chat-store"; +import { + notifyLighthouseV2SessionArchived, + onLighthouseV2ConfigurationsChanged, + onLighthouseV2NewChat, + onLighthouseV2SessionArchived, + onLighthouseV2SessionsChanged, +} from "@/app/(prowler)/lighthouse/_lib/session-events"; +import type { LighthouseV2Session } from "@/app/(prowler)/lighthouse/_types"; +import { LighthouseIconWithAura } from "@/components/icons"; +import { Button } from "@/components/shadcn/button/button"; +import { useMountEffect } from "@/hooks/use-mount-effect"; +import { LIGHTHOUSE_ROUTE } from "@/lib/lighthouse-routes"; + +import { + LighthouseChatStoreProvider, + useLighthouseChatStore, +} from "../chat/lighthouse-chat-store-provider"; +import { + LIGHTHOUSE_CHAT_SURFACE, + LighthouseV2ChatView, +} from "../chat/lighthouse-v2-chat-view"; + +import { LighthousePanelChatSkeleton } from "./lighthouse-panel-chat-skeleton"; + +const PANEL_CHAT_STATUS = { + LOADING: "loading", + ERROR: "error", + NOT_CONFIGURED: "not-configured", + READY: "ready", +} as const; + +interface PanelChatLoadingState { + status: typeof PANEL_CHAT_STATUS.LOADING; +} + +interface PanelChatErrorState { + status: typeof PANEL_CHAT_STATUS.ERROR; + message: string; +} + +interface PanelChatNotConfiguredState { + status: typeof PANEL_CHAT_STATUS.NOT_CONFIGURED; +} + +interface PanelChatReadyState { + status: typeof PANEL_CHAT_STATUS.READY; + config: LighthouseChatConfig; + modelsError?: string; +} + +type PanelChatState = + | PanelChatLoadingState + | PanelChatErrorState + | PanelChatNotConfiguredState + | PanelChatReadyState; + +// Config cache: the panel loads its configs/models lazily on first open (never +// in the layout, so pages don't pay for a panel most sessions never open); the +// cache makes every later mount — reopen, drawer AI tab — instant. +let cachedReadyState: PanelChatReadyState | null = null; + +export function resetPanelChatConfigCacheForTests(): void { + cachedReadyState = null; + resetPanelChatMessageState(); +} + +// Config CRUD happens on the settings route while the global panel can remain +// mounted. Invalidate at module scope so the open panel reloads in place and a +// later open rebuilds cache and store against the new configuration. +if (typeof window !== "undefined") { + onLighthouseV2ConfigurationsChanged(() => { + cachedReadyState = null; + resetPanelChatMessageState(); + resetPanelChatStore(); + }); +} + +export function LighthousePanelChat() { + const [state, setState] = useState( + () => cachedReadyState ?? { status: PANEL_CHAT_STATUS.LOADING }, + ); + + const load = async () => { + setState({ status: PANEL_CHAT_STATUS.LOADING }); + const next = await loadPanelChatState(); + if ( + next.status === PANEL_CHAT_STATUS.READY && + next.modelsError === undefined + ) { + cachedReadyState = next; + } else { + cachedReadyState = null; + } + setState(next); + }; + + useMountEffect(() => { + if (state.status !== PANEL_CHAT_STATUS.READY) { + void load(); + } + // The module-scope listener above already invalidated cache and store + // (registration order); reload so an open panel refreshes in place. + return onLighthouseV2ConfigurationsChanged(() => void load()); + }); + + if (state.status === PANEL_CHAT_STATUS.LOADING) { + return ; + } + if (state.status === PANEL_CHAT_STATUS.ERROR) { + return ( + void load()} /> + ); + } + if (state.status === PANEL_CHAT_STATUS.NOT_CONFIGURED) { + return ; + } + return ( + + ); +} + +interface PanelChatReadyProps { + config: LighthouseChatConfig; + modelsError?: string; +} + +function PanelChatReady({ config, modelsError }: PanelChatReadyProps) { + const [store] = useState(() => + getOrCreatePanelChatStore(config, { initialError: modelsError }), + ); + const [sessions, setSessions] = useState([]); + + const refreshSessions = async () => { + try { + const result = await getLighthouseV2Sessions(); + if ("data" in result) { + setSessions(result.data); + } + } catch { + // Best-effort refresh: swallow transport-level failures so a rejected + // server action never escapes the mount effect as an unhandled error. + } + }; + + useMountEffect(() => { + void refreshSessions(); + const syncPanelChatState = () => { + const chatState = store.getState(); + setPanelChatMessageState({ + hasMessages: chatState.messages.length > 0, + activeSessionId: chatState.activeSessionId, + }); + }; + syncPanelChatState(); + const unsubscribeChatStore = store.subscribe(syncPanelChatState); + const unsubscribeSessionsChanged = onLighthouseV2SessionsChanged(() => { + void refreshSessions(); + }); + const unsubscribeNewChat = onLighthouseV2NewChat(() => + store.getState().resetToNewChat(), + ); + // Archiving from any surface (sidebar, popover) must reset the panel chat + // when its open session is the archived one, and drop the archived chat + // from the "Recent chats" list. + const unsubscribeSessionArchived = onLighthouseV2SessionArchived( + (sessionId) => { + store.getState().handleSessionArchived(sessionId); + void refreshSessions(); + }, + ); + return () => { + unsubscribeChatStore(); + unsubscribeSessionsChanged(); + unsubscribeNewChat(); + unsubscribeSessionArchived(); + // A partial config cannot be reused after the model endpoint recovers: + // this store captured the incomplete model list at creation time. + if (modelsError) resetPanelChatStore(); + }; + }); + + return ( + +
+
+ 0 ? ( +
+ + Recent chats + + +
+ ) : undefined + } + /> +
+
+ + ); +} + +interface PanelChatSessionsProps { + sessions: LighthouseV2Session[]; + onAfterSelect?: () => void; +} + +function PanelChatSessions({ + sessions, + onAfterSelect, +}: PanelChatSessionsProps) { + const [search, setSearch] = useState(""); + const activeSessionId = useLighthouseChatStore( + (state) => state.activeSessionId, + ); + const isOnNewChat = useLighthouseChatStore( + (state) => state.activeSessionId === null && state.messages.length === 0, + ); + const openSession = useLighthouseChatStore((state) => state.openSession); + const resetToNewChat = useLighthouseChatStore( + (state) => state.resetToNewChat, + ); + + const handleArchiveSession = async (sessionId: string) => { + try { + const result = await archiveLighthouseV2Session(sessionId); + if ("data" in result) { + // Resets this chat when its open session is archived, and prompts + // every session list (sidebar included) to refresh. + notifyLighthouseV2SessionArchived(sessionId); + } + } catch { + // Archiving is recoverable from the list; ignore transient failures. + } + }; + + return ( + { + resetToNewChat(); + onAfterSelect?.(); + }} + onOpenSession={(sessionId) => { + void openSession(sessionId); + onAfterSelect?.(); + }} + onArchiveSession={(sessionId) => void handleArchiveSession(sessionId)} + /> + ); +} + +interface PanelChatErrorProps { + message: string; + onRetry: () => void; +} + +function PanelChatError({ message, onRetry }: PanelChatErrorProps) { + return ( +
+

+ {message} +

+ +
+ ); +} + +function PanelChatConnectCta() { + return ( +
+ +
+

+ Lighthouse AI is not set up yet +

+

+ Connect an LLM provider to start asking questions about your cloud + security posture. +

+
+ +
+ ); +} + +async function loadPanelChatState(): Promise { + try { + const result = await loadLighthouseChatConfig(); + if (result.status === LIGHTHOUSE_CHAT_CONFIG_STATUS.ERROR) { + return { status: PANEL_CHAT_STATUS.ERROR, message: result.message }; + } + if (result.status === LIGHTHOUSE_CHAT_CONFIG_STATUS.NOT_CONFIGURED) { + return { status: PANEL_CHAT_STATUS.NOT_CONFIGURED }; + } + return { + status: PANEL_CHAT_STATUS.READY, + config: result.config, + modelsError: result.modelsError, + }; + } catch { + return { + status: PANEL_CHAT_STATUS.ERROR, + message: "Could not load Lighthouse AI. Try again shortly.", + }; + } +} diff --git a/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-header-actions.tsx b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-header-actions.tsx new file mode 100644 index 0000000000..9f71449626 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_components/panel/lighthouse-panel-header-actions.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { Maximize2, Plus } from "lucide-react"; +import Link from "next/link"; +import { useSyncExternalStore } from "react"; + +import { + getPanelChatActiveSessionId, + getPanelChatHasMessages, + subscribePanelChatHasMessages, +} from "@/app/(prowler)/lighthouse/_lib/panel-chat-message-state"; +import { notifyLighthouseV2NewChat } from "@/app/(prowler)/lighthouse/_lib/session-events"; +import { Button } from "@/components/shadcn/button/button"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; +import { LIGHTHOUSE_ROUTE } from "@/lib/lighthouse-routes"; +import { cn } from "@/lib/utils"; + +export function LighthousePanelHeaderActions() { + const hasMessages = useSyncExternalStore( + subscribePanelChatHasMessages, + getPanelChatHasMessages, + () => false, + ); + const activeSessionId = useSyncExternalStore( + subscribePanelChatHasMessages, + getPanelChatActiveSessionId, + () => null, + ); + const fullPageHref = activeSessionId + ? `${LIGHTHOUSE_ROUTE.CHAT}?session=${encodeURIComponent(activeSessionId)}` + : LIGHTHOUSE_ROUTE.CHAT; + + return ( + <> + + + + + + + + {hasMessages + ? "New chat" + : "Send a message before starting a new chat"} + + + + + + + Open full page + + + ); +} diff --git a/ui/app/(prowler)/lighthouse/_lib/chat-store.test.ts b/ui/app/(prowler)/lighthouse/_lib/chat-store.test.ts new file mode 100644 index 0000000000..16fc3c3e8e --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_lib/chat-store.test.ts @@ -0,0 +1,503 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + createLighthouseChatStore, + selectLighthouseChatCanSend, +} from "@/app/(prowler)/lighthouse/_lib/chat-store"; +import { + type MockEventSource, + stubEventSource, +} from "@/app/(prowler)/lighthouse/_lib/testing/event-source-mock"; +import type { + LighthouseV2Configuration, + LighthouseV2Message, + LighthouseV2SupportedModel, + LighthouseV2SupportedProvider, +} from "@/app/(prowler)/lighthouse/_types"; + +const { + createSessionMock, + getMessagesMock, + sendMessageMock, + updateConfigurationMock, +} = vi.hoisted(() => ({ + createSessionMock: vi.fn(), + getMessagesMock: vi.fn(), + sendMessageMock: vi.fn(), + updateConfigurationMock: vi.fn(), +})); + +vi.mock("@/app/(prowler)/lighthouse/_actions", () => ({ + createLighthouseV2Session: createSessionMock, + getLighthouseV2Messages: getMessagesMock, + sendLighthouseV2Message: sendMessageMock, + updateLighthouseV2Configuration: updateConfigurationMock, +})); + +const configurations: LighthouseV2Configuration[] = [ + { + id: "config-openai", + providerType: "openai", + baseUrl: null, + defaultModel: "gpt-5.1", + businessContext: "Production account", + connected: true, + connectionLastCheckedAt: "2026-06-24T10:00:00Z", + insertedAt: "2026-06-24T09:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }, +]; + +const modelsByProvider = { + openai: [model("gpt-5.1")], + bedrock: [], + "openai-compatible": [], +}; + +const supportedProviders: LighthouseV2SupportedProvider[] = [ + { id: "openai", name: "OpenAI" }, + { id: "bedrock", name: "AWS Bedrock" }, + { id: "openai-compatible", name: "OpenAI Compatible" }, +]; + +const config = { configurations, modelsByProvider, supportedProviders }; + +let eventSources: MockEventSource[] = []; + +describe("createLighthouseChatStore", () => { + beforeEach(() => { + createSessionMock.mockReset(); + getMessagesMock.mockReset(); + sendMessageMock.mockReset(); + updateConfigurationMock.mockReset(); + eventSources = stubEventSource(); + + createSessionMock.mockResolvedValue({ + data: { + id: "session-1", + title: "Summarize findings", + isArchived: false, + insertedAt: "2026-06-24T10:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }, + }); + getMessagesMock.mockResolvedValue({ data: [] }); + sendMessageMock.mockResolvedValue({ + data: { + task: { id: "task-1", name: "lighthouse-run", state: "executing" }, + }, + }); + window.history.replaceState(null, "", "/"); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("resolves the connected provider's remembered model on creation", () => { + // Given / When + const store = makeStore(); + + // Then + expect(store.getState().selectedModelSelection).toEqual({ + providerType: "openai", + modelId: "gpt-5.1", + }); + expect(selectLighthouseChatCanSend(store.getState())).toBe(true); + }); + + it("creates a session and subscribes to the stream before sending (no replay buffer)", async () => { + // Given + const store = makeStore(); + + // When + await store.getState().submitMessage("Summarize findings"); + + // Then + expect(createSessionMock).toHaveBeenCalledWith("Summarize findings"); + expect(EventSource).toHaveBeenCalledWith( + "/api/lighthouse/v2/sessions/session-1/event-stream", + ); + const eventSourceOrder = vi.mocked(EventSource).mock.invocationCallOrder[0]; + const sendOrder = sendMessageMock.mock.invocationCallOrder[0]; + expect(eventSourceOrder).toBeLessThan(sendOrder); + // The optimistic user message renders immediately and the task id is live. + expect(store.getState().messages.at(-1)?.parts[0]?.content).toEqual({ + text: "Summarize findings", + }); + expect(store.getState().streamState.activeTaskId).toBe("task-1"); + }); + + it("does not touch the URL when syncUrlToSession is off (panel surface)", async () => { + // Given + const store = makeStore({ syncUrlToSession: false }); + const replaceStateSpy = vi.spyOn(window.history, "replaceState"); + + // When + await store.getState().submitMessage("Summarize findings"); + + // Then + expect(store.getState().activeSessionId).toBe("session-1"); + expect(replaceStateSpy).not.toHaveBeenCalled(); + }); + + it("writes the session URL in place when syncUrlToSession is on (page surface)", async () => { + // Given + const store = makeStore({ syncUrlToSession: true }); + const replaceStateSpy = vi.spyOn(window.history, "replaceState"); + + // When + await store.getState().submitMessage("Summarize findings"); + + // Then + expect(replaceStateSpy).toHaveBeenCalledWith( + null, + "", + "/lighthouse?session=session-1", + ); + }); + + it("reloads persisted messages and closes the stream on message.end", async () => { + // Given + const store = makeStore(); + await store.getState().submitMessage("Summarize findings"); + getMessagesMock.mockResolvedValue({ + data: [message("message-1", "assistant", "Persisted answer")], + }); + + // When + eventSources[0].emit("message.end", { message_id: "message-1" }); + await vi.waitFor(() => + expect(getMessagesMock).toHaveBeenCalledWith("session-1"), + ); + + // Then + await vi.waitFor(() => + expect(store.getState().messages[0]?.parts[0]?.content).toBe( + "Persisted answer", + ), + ); + expect(eventSources[0].close).toHaveBeenCalled(); + expect(store.getState().streamState.activeTaskId).toBeNull(); + }); + + it("blocks sending and refreshes messages on a 409 conflict", async () => { + // Given + const store = makeStore(); + sendMessageMock.mockResolvedValue({ + error: "Another run is in progress.", + status: 409, + }); + + // When + await store.getState().submitMessage("Summarize findings"); + + // Then + expect(store.getState().blockedByConflict).toBe(true); + expect(store.getState().feedback).toBe("Another run is in progress."); + expect(getMessagesMock).toHaveBeenCalledWith("session-1"); + expect(eventSources[0].close).toHaveBeenCalled(); + expect(selectLighthouseChatCanSend(store.getState())).toBe(false); + }); + + it("reconciles the optimistic message when the send fails without a conflict", async () => { + // Given: the backend rejects the message with a plain failure + const store = makeStore(); + sendMessageMock.mockResolvedValue({ error: "Send failed.", status: 500 }); + + // When + await store.getState().submitMessage("Summarize findings"); + + // Then: feedback surfaces without blocking, and the optimistic user + // message is reconciled against the server (it was never persisted) + expect(store.getState().feedback).toBe("Send failed."); + expect(store.getState().blockedByConflict).toBe(false); + expect(getMessagesMock).toHaveBeenCalledWith("session-1"); + expect(store.getState().messages).toHaveLength(0); + }); + + it("drops a failed send once the chat points at another session", async () => { + // Given: a send still in flight + const store = makeStore(); + let resolveSend: (value: unknown) => void = () => {}; + sendMessageMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveSend = resolve; + }), + ); + const submitting = store.getState().submitMessage("Summarize findings"); + await vi.waitFor(() => expect(sendMessageMock).toHaveBeenCalled()); + + // When: the user opens another session before the send fails + await store.getState().openSession("session-9"); + resolveSend({ error: "Send failed.", status: 500 }); + await submitting; + + // Then: the dead submission's failure never surfaces in the new session + expect(store.getState().activeSessionId).toBe("session-9"); + expect(store.getState().feedback).toBeNull(); + }); + + it("keeps a fast follow-up intact when the terminal refresh resolves late", async () => { + // Given: a completed run whose terminal message refresh is still in flight + const store = makeStore(); + await store.getState().submitMessage("First question"); + let resolveRefresh: (value: unknown) => void = () => {}; + getMessagesMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveRefresh = resolve; + }), + ); + eventSources[0].emit("message.end", { message_id: "message-1" }); + await vi.waitFor(() => + expect(getMessagesMock).toHaveBeenCalledWith("session-1"), + ); + + // When: the user sends a follow-up before that refresh resolves + sendMessageMock.mockResolvedValue({ + data: { + task: { id: "task-2", name: "lighthouse-run", state: "executing" }, + }, + }); + await store.getState().submitMessage("Follow-up question"); + resolveRefresh({ data: [message("message-1", "assistant", "Answer")] }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Then: the stale snapshot erases neither the new optimistic message nor + // the follow-up's task id + expect(store.getState().streamState.activeTaskId).toBe("task-2"); + expect(store.getState().messages.at(-1)?.parts[0]?.content).toEqual({ + text: "Follow-up question", + }); + }); + + it("abandons an in-flight submit after destroy", async () => { + // Given: destroy fires while the session create is still in flight + const store = makeStore({ syncUrlToSession: true }); + const replaceStateSpy = vi.spyOn(window.history, "replaceState"); + let resolveCreate: (value: unknown) => void = () => {}; + createSessionMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveCreate = resolve; + }), + ); + const submitting = store.getState().submitMessage("Summarize findings"); + store.getState().destroy(); + + // When + resolveCreate({ + data: { + id: "session-1", + title: "Summarize findings", + isArchived: false, + insertedAt: "2026-06-24T10:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }, + }); + await submitting; + + // Then: no URL rewrite on whatever page is now open, no orphan stream + expect(replaceStateSpy).not.toHaveBeenCalled(); + expect(eventSources).toHaveLength(0); + }); + + it("does not replace a session opened while a new session is being created", async () => { + // Given: creating the first session is still in flight + const store = makeStore(); + let resolveCreate: (value: unknown) => void = () => {}; + createSessionMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveCreate = resolve; + }), + ); + const submitting = store.getState().submitMessage("Summarize findings"); + await vi.waitFor(() => expect(createSessionMock).toHaveBeenCalled()); + + // When: the user opens another conversation before creation resolves + await store.getState().openSession("session-9"); + resolveCreate({ + data: { + id: "session-1", + title: "Summarize findings", + isArchived: false, + insertedAt: "2026-06-24T10:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }, + }); + await submitting; + + // Then: the stale creation cannot replace or submit into the open chat + expect(store.getState().activeSessionId).toBe("session-9"); + expect(sendMessageMock).not.toHaveBeenCalled(); + }); + + it("does not revive a session creation cancelled by a new-chat reset", async () => { + // Given: creating the first session is still in flight + const store = makeStore(); + let resolveCreate: (value: unknown) => void = () => {}; + createSessionMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveCreate = resolve; + }), + ); + const submitting = store.getState().submitMessage("Summarize findings"); + await vi.waitFor(() => expect(createSessionMock).toHaveBeenCalled()); + + // When: the user resets to a new chat before creation resolves + store.getState().resetToNewChat(); + resolveCreate({ + data: { + id: "session-1", + title: "Summarize findings", + isArchived: false, + insertedAt: "2026-06-24T10:00:00Z", + updatedAt: "2026-06-24T10:00:00Z", + }, + }); + await submitting; + + // Then + expect(store.getState().activeSessionId).toBeNull(); + expect(sendMessageMock).not.toHaveBeenCalled(); + }); + + it("opens an existing session client-side without navigation", async () => { + // Given + const store = makeStore({ syncUrlToSession: false }); + const replaceStateSpy = vi.spyOn(window.history, "replaceState"); + getMessagesMock.mockResolvedValue({ + data: [message("message-1", "assistant", "Old answer")], + }); + + // When + await store.getState().openSession("session-9"); + + // Then + expect(store.getState().activeSessionId).toBe("session-9"); + expect(store.getState().messages[0]?.parts[0]?.content).toBe("Old answer"); + expect(replaceStateSpy).not.toHaveBeenCalled(); + }); + + it("drops a stale openSession result when the chat was reset meanwhile", async () => { + // Given: opening a session whose message fetch is still in flight + const store = makeStore(); + let resolveLoad: (value: unknown) => void = () => {}; + getMessagesMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveLoad = resolve; + }), + ); + const opening = store.getState().openSession("session-9"); + + // When: the user starts a new chat before the fetch resolves + store.getState().resetToNewChat(); + resolveLoad({ data: [message("message-1", "assistant", "Stale answer")] }); + await opening; + + // Then: the stale messages never repopulate the reset chat + expect(store.getState().activeSessionId).toBeNull(); + expect(store.getState().messages).toHaveLength(0); + }); + + it("resets to a new chat and closes any open stream", async () => { + // Given + const store = makeStore(); + await store.getState().submitMessage("Summarize findings"); + expect(store.getState().activeSessionId).toBe("session-1"); + + // When + store.getState().resetToNewChat(); + + // Then + expect(eventSources[0].close).toHaveBeenCalled(); + expect(store.getState().activeSessionId).toBeNull(); + expect(store.getState().messages).toHaveLength(0); + expect(store.getState().streamState.activeTaskId).toBeNull(); + }); + + it("resets only when the archived session is the active one", async () => { + // Given + const store = makeStore(); + await store.getState().submitMessage("Summarize findings"); + + // When / Then: an unrelated session leaves the conversation intact + store.getState().handleSessionArchived("session-other"); + expect(store.getState().activeSessionId).toBe("session-1"); + + // When / Then: archiving the active session resets in place + store.getState().handleSessionArchived("session-1"); + expect(store.getState().activeSessionId).toBeNull(); + }); + + it("closes the stream on destroy", async () => { + // Given + const store = makeStore(); + await store.getState().submitMessage("Summarize findings"); + + // When + store.getState().destroy(); + + // Then + expect(eventSources[0].close).toHaveBeenCalled(); + }); + + it("surfaces a connection error when the stream closes terminally", async () => { + // Given + const store = makeStore(); + await store.getState().submitMessage("Summarize findings"); + + // When: the EventSource fails terminally (e.g. 401/404 on the SSE GET) + eventSources[0].fail(2 /* EventSource.CLOSED */); + + // Then + expect(store.getState().feedback).toBe( + "Unable to connect to the response stream.", + ); + }); +}); + +function makeStore( + overrides?: Partial[0]>, +) { + return createLighthouseChatStore({ + config, + syncUrlToSession: false, + ...overrides, + }); +} + +function model(id: string, name = id): LighthouseV2SupportedModel { + return { + id, + name, + maxInputTokens: null, + maxOutputTokens: null, + supportsFunctionCalling: null, + supportsVision: null, + supportsReasoning: null, + }; +} + +function message( + id: string, + role: LighthouseV2Message["role"], + content: string, +): LighthouseV2Message { + return { + id, + role, + model: null, + tokenUsage: null, + insertedAt: "2026-06-25T10:00:00Z", + parts: [ + { + id: `${id}-part`, + type: "text", + content, + toolCallOutcome: null, + insertedAt: "2026-06-25T10:00:00Z", + updatedAt: "2026-06-25T10:00:00Z", + }, + ], + }; +} diff --git a/ui/app/(prowler)/lighthouse/_lib/chat-store.ts b/ui/app/(prowler)/lighthouse/_lib/chat-store.ts new file mode 100644 index 0000000000..0cdbde9478 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_lib/chat-store.ts @@ -0,0 +1,485 @@ +import { createStore, type StoreApi } from "zustand/vanilla"; + +import { + createLighthouseV2Session, + getLighthouseV2Messages, + sendLighthouseV2Message, + updateLighthouseV2Configuration, +} from "@/app/(prowler)/lighthouse/_actions"; +import { + createInitialLighthouseV2StreamState, + type LighthouseV2StreamState, + reduceLighthouseV2Event, +} from "@/app/(prowler)/lighthouse/_lib/event-reducer"; +import { + buildOptimisticMessage, + buildSessionTitle, +} from "@/app/(prowler)/lighthouse/_lib/messages"; +import type { LighthouseV2ModelSelection } from "@/app/(prowler)/lighthouse/_lib/model-selection"; +import { notifyLighthouseV2SessionsChanged } from "@/app/(prowler)/lighthouse/_lib/session-events"; +import { parseStreamEvent } from "@/app/(prowler)/lighthouse/_lib/stream-event-parser"; +import { buildLighthouseV2StreamUrl } from "@/app/(prowler)/lighthouse/_lib/stream-url"; +import { + LIGHTHOUSE_V2_PROVIDER_TYPE, + LIGHTHOUSE_V2_SSE_EVENT, + type LighthouseV2Configuration, + type LighthouseV2Message, + type LighthouseV2ProviderType, + type LighthouseV2SSEEvent, + type LighthouseV2SupportedModel, + type LighthouseV2SupportedProvider, +} from "@/app/(prowler)/lighthouse/_types"; + +export interface LighthouseChatConfig { + configurations: LighthouseV2Configuration[]; + modelsByProvider: Record< + LighthouseV2ProviderType, + LighthouseV2SupportedModel[] + >; + supportedProviders: LighthouseV2SupportedProvider[]; +} + +export interface CreateLighthouseChatStoreOptions { + config: LighthouseChatConfig; + // The /lighthouse page mirrors the active session into the URL via + // replaceState; other surfaces (side panel, drawers) must never touch it. + syncUrlToSession: boolean; + initialSessionId?: string; + initialMessages?: LighthouseV2Message[]; + initialInput?: string; + initialError?: string; +} + +export interface LighthouseChatState { + config: LighthouseChatConfig; + activeSessionId: string | null; + messages: LighthouseV2Message[]; + streamState: LighthouseV2StreamState; + input: string; + feedback: string | null; + blockedByConflict: boolean; + isSubmitting: boolean; + isLoadingSession: boolean; + lastSubmittedText: string | null; + selectedModelSelection: LighthouseV2ModelSelection | null; + modelPreferenceSaving: boolean; + setSessionUrlSyncEnabled: (enabled: boolean) => void; + setInput: (value: string) => void; + dismissFeedback: () => void; + selectModel: (selection: LighthouseV2ModelSelection) => Promise; + submitMessage: (text: string) => Promise; + openSession: (sessionId: string) => Promise; + resetToNewChat: () => void; + handleSessionArchived: (sessionId: string) => void; + destroy: () => void; +} + +export type LighthouseChatStore = StoreApi; + +export function selectLighthouseChatCanSend( + state: LighthouseChatState, +): boolean { + const selectedConfiguration = state.config.configurations.find( + (configuration) => + configuration.connected === true && + configuration.providerType === state.selectedModelSelection?.providerType, + ); + return ( + selectedConfiguration?.connected === true && + Boolean(state.selectedModelSelection?.modelId) && + !state.streamState.activeTaskId && + !state.blockedByConflict && + !state.isSubmitting + ); +} + +export function createLighthouseChatStore( + options: CreateLighthouseChatStoreOptions, +): LighthouseChatStore { + const { config } = options; + const connectedConfigurations = config.configurations.filter( + (configuration) => configuration.connected === true, + ); + // The EventSource lives in this closure (never in state): it isn't + // serializable, no render depends on it, and here it survives the consuming + // component unmounting — the reason this factory exists. + let eventSource: EventSource | null = null; + // Set by destroy(): async flows check it after each await so a torn-down + // store never rewrites the URL of another page or opens an orphan stream. + let destroyed = false; + // User-driven session changes invalidate async session creation. Comparing + // only activeSessionId is insufficient because both the initial chat and a + // later reset intentionally use null. + let sessionIntentVersion = 0; + let syncUrlToSession = options.syncUrlToSession; + + const syncSessionUrl = (sessionId: string | null) => { + if (!syncUrlToSession) return; + const url = sessionId + ? `/lighthouse?session=${encodeURIComponent(sessionId)}` + : "/lighthouse"; + window.history.replaceState(window.history.state, "", url); + }; + + return createStore()((set, get) => { + const closeStream = () => { + eventSource?.close(); + eventSource = null; + }; + + const refreshMessages = async ( + sessionId: string, + shouldApply: () => boolean = () => true, + ): Promise => { + const result = await getLighthouseV2Messages(sessionId); + // The fetch is async, so a reset (new chat, or archiving this session) + // can land while it is in flight. Drop the stale result instead of + // repopulating a chat that no longer points at this session. + if (sessionId !== get().activeSessionId || !shouldApply()) return false; + if ("data" in result) { + set({ messages: result.data }); + return true; + } + return false; + }; + + const handleTerminalEvent = async ( + sessionId: string, + event: LighthouseV2SSEEvent, + ) => { + if ( + event.type === LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_END || + event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR + ) { + closeStream(); + set({ blockedByConflict: false }); + if (event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR) { + set({ feedback: event.detail || "Agent run failed." }); + } + // A fast follow-up can start while this refresh is in flight; applying + // it would erase the new optimistic message and provisional task id. + const noNewerSubmission = () => + !get().isSubmitting && !get().streamState.activeTaskId; + const refreshed = await refreshMessages(sessionId, noNewerSubmission); + if (refreshed) { + set({ streamState: createInitialLighthouseV2StreamState() }); + } + notifyLighthouseV2SessionsChanged(); + } + }; + + const startStream = (streamUrl: string, sessionId: string) => { + closeStream(); + const source = new EventSource(streamUrl); + eventSource = source; + + const applyEvent = (event: LighthouseV2SSEEvent) => { + set((current) => ({ + streamState: reduceLighthouseV2Event(current.streamState, event), + })); + void handleTerminalEvent(sessionId, event); + }; + + source.addEventListener("message.delta", (event) => + applyEvent( + parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_DELTA), + ), + ); + source.addEventListener("tool_call.start", (event) => + applyEvent( + parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.TOOL_CALL_START), + ), + ); + source.addEventListener("tool_call.end", (event) => + applyEvent( + parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.TOOL_CALL_END), + ), + ); + source.addEventListener("message.end", (event) => + applyEvent( + parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_END), + ), + ); + source.addEventListener("error", (event) => { + if (event instanceof MessageEvent) { + applyEvent(parseStreamEvent(event, LIGHTHOUSE_V2_SSE_EVENT.ERROR)); + } + }); + // The browser fires `onerror` both on a transient drop (it auto-reconnects) + // and on a non-retryable failure such as a 401/404 on the SSE GET. Only the + // latter leaves the source CLOSED, so surface a connection error there and + // treat everything else as a reconnect. + source.onerror = () => { + if (eventSource !== source) return; + if (source.readyState === EventSource.CLOSED) { + closeStream(); + set({ feedback: "Unable to connect to the response stream." }); + } + set((current) => ({ + streamState: reduceLighthouseV2Event(current.streamState, { + type: "disconnect", + }), + })); + }; + }; + + const ensureSession = async (text: string) => { + const existingSessionId = get().activeSessionId; + if (existingSessionId) { + return existingSessionId; + } + + const intentVersion = sessionIntentVersion; + const title = buildSessionTitle(text); + const result = await createLighthouseV2Session(title); + if (destroyed || intentVersion !== sessionIntentVersion) return null; + if ("error" in result) { + set({ feedback: result.error }); + return null; + } + + // Update the URL in place (not router.push) so the force-dynamic server + // component is NOT re-run mid-submit. A re-run would change `key` in + // page.tsx and remount the chat, tearing down the open EventSource. + syncSessionUrl(result.data.id); + set({ activeSessionId: result.data.id }); + notifyLighthouseV2SessionsChanged(); + return result.data.id; + }; + + return { + config, + activeSessionId: options.initialSessionId ?? null, + messages: options.initialMessages ?? [], + streamState: createInitialLighthouseV2StreamState(), + input: options.initialInput ?? "", + feedback: options.initialError ?? null, + blockedByConflict: false, + isSubmitting: false, + isLoadingSession: false, + lastSubmittedText: null, + selectedModelSelection: resolveInitialModelSelection( + connectedConfigurations, + config.modelsByProvider, + ), + modelPreferenceSaving: false, + + setSessionUrlSyncEnabled: (enabled) => { + syncUrlToSession = enabled; + }, + + setInput: (value) => set({ input: value }), + + dismissFeedback: () => set({ feedback: null }), + + selectModel: async (selection) => { + // The selection drives the model used for the next message, so it stays + // applied even if persisting it as the provider's default model fails — + // reverting it would make a connected provider unusable when the save 4xxs. + set({ selectedModelSelection: selection, feedback: null }); + + const configId = connectedConfigurations.find( + (configuration) => + configuration.providerType === selection.providerType, + )?.id; + if (!configId) return; + + set({ modelPreferenceSaving: true }); + + const result = await updateLighthouseV2Configuration(configId, { + defaultModel: selection.modelId, + }); + + set({ modelPreferenceSaving: false }); + + if ("error" in result) { + set({ feedback: result.error }); + } + }, + + submitMessage: async (text) => { + const trimmedText = text.trim(); + if (!trimmedText) return; + if (!get().selectedModelSelection) { + set({ feedback: "Select a model before sending a message." }); + return; + } + if (!selectLighthouseChatCanSend(get())) return; + + set({ isSubmitting: true }); + try { + const sessionId = await ensureSession(trimmedText); + if (!sessionId || destroyed) return; + + const selection = get().selectedModelSelection; + if (!selection) return; + + const provisionalTaskId = `pending-${Date.now()}`; + set((current) => ({ + feedback: null, + blockedByConflict: false, + lastSubmittedText: trimmedText, + input: "", + messages: [ + ...current.messages, + buildOptimisticMessage("user", trimmedText), + ], + streamState: + createInitialLighthouseV2StreamState(provisionalTaskId), + })); + + // Subscribe to the same-origin SSE proxy BEFORE sending the message: + // the backend has no replay buffer, so the listener must be attached + // before the worker starts emitting. + startStream(buildLighthouseV2StreamUrl(sessionId), sessionId); + + const result = await sendLighthouseV2Message({ + sessionId, + text: trimmedText, + provider: selection.providerType, + model: selection.modelId, + }); + if (destroyed) return; + + if ("error" in result) { + // Stale guard: the chat may point at another session by now, so + // this failure must not clobber its stream state or feedback. + if (get().activeSessionId !== sessionId) return; + closeStream(); + set({ + streamState: createInitialLighthouseV2StreamState(), + feedback: result.error, + }); + if (result.status === 409) { + set({ blockedByConflict: true }); + } + // Reconcile the optimistic user message against the server on any + // failure — it may or may not have been persisted. + await refreshMessages(sessionId); + return; + } + + set((current) => ({ + streamState: + current.streamState.activeTaskId === provisionalTaskId + ? { ...current.streamState, activeTaskId: result.data.task.id } + : current.streamState, + })); + notifyLighthouseV2SessionsChanged(); + } finally { + set({ isSubmitting: false }); + } + }, + + openSession: async (sessionId) => { + if (get().activeSessionId === sessionId) return; + sessionIntentVersion += 1; + closeStream(); + set({ + activeSessionId: sessionId, + messages: [], + input: "", + feedback: null, + blockedByConflict: false, + isSubmitting: false, + isLoadingSession: true, + lastSubmittedText: null, + streamState: createInitialLighthouseV2StreamState(), + }); + syncSessionUrl(sessionId); + + const result = await getLighthouseV2Messages(sessionId); + // Stale guard: a reset or another openSession can land mid-fetch. + if (get().activeSessionId !== sessionId) return; + if ("data" in result) { + set({ messages: result.data, isLoadingSession: false }); + } else { + set({ feedback: result.error, isLoadingSession: false }); + } + }, + + resetToNewChat: () => { + sessionIntentVersion += 1; + closeStream(); + set({ + activeSessionId: null, + messages: [], + input: "", + feedback: null, + blockedByConflict: false, + isSubmitting: false, + isLoadingSession: false, + lastSubmittedText: null, + streamState: createInitialLighthouseV2StreamState(), + }); + syncSessionUrl(null); + }, + + handleSessionArchived: (sessionId) => { + // Archiving deletes the session; when it's the open one, fall back to a + // new chat instead of leaving a dead conversation on screen. + if (sessionId === get().activeSessionId) { + get().resetToNewChat(); + } + }, + + destroy: () => { + destroyed = true; + closeStream(); + }, + }; + }); +} + +// Fixed precedence used to pick which connected provider opens the chat. Any +// provider outside this list keeps its relative order behind these. +const LIGHTHOUSE_V2_PROVIDER_PRIORITY = [ + LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI, + LIGHTHOUSE_V2_PROVIDER_TYPE.BEDROCK, + LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI_COMPATIBLE, +] as const; + +// Fallback model per provider when the configuration has no remembered model. +const LIGHTHOUSE_V2_PREFERRED_DEFAULT_MODEL: Partial< + Record +> = { + [LIGHTHOUSE_V2_PROVIDER_TYPE.OPENAI]: "gpt-5.6-terra", +}; + +function resolveInitialModelSelection( + connectedConfigurations: LighthouseV2Configuration[], + modelsByProvider: Record< + LighthouseV2ProviderType, + LighthouseV2SupportedModel[] + >, +): LighthouseV2ModelSelection | null { + const priorityIndex = (providerType: LighthouseV2ProviderType) => { + const index = LIGHTHOUSE_V2_PROVIDER_PRIORITY.indexOf(providerType); + return index === -1 ? LIGHTHOUSE_V2_PROVIDER_PRIORITY.length : index; + }; + // Stable sort keeps providers outside the priority list in their original order. + const orderedConfigurations = [...connectedConfigurations].sort( + (a, b) => priorityIndex(a.providerType) - priorityIndex(b.providerType), + ); + + for (const configuration of orderedConfigurations) { + const providerModels = modelsByProvider[configuration.providerType] ?? []; + if (providerModels.length === 0) continue; + // Prefer the provider's remembered model when it is still supported, then + // the provider's preferred default, then the first supported model. + const rememberedModel = providerModels.find( + (model) => model.id === configuration.defaultModel, + ); + const preferredModel = providerModels.find( + (model) => + model.id === + LIGHTHOUSE_V2_PREFERRED_DEFAULT_MODEL[configuration.providerType], + ); + return { + providerType: configuration.providerType, + modelId: (rememberedModel ?? preferredModel ?? providerModels[0]).id, + }; + } + + return null; +} diff --git a/ui/app/(prowler)/lighthouse/_lib/event-reducer.test.ts b/ui/app/(prowler)/lighthouse/_lib/event-reducer.test.ts index 4f284a5656..89c62ef8bc 100644 --- a/ui/app/(prowler)/lighthouse/_lib/event-reducer.test.ts +++ b/ui/app/(prowler)/lighthouse/_lib/event-reducer.test.ts @@ -62,7 +62,7 @@ describe("event-reducer", () => { state = reduceLighthouseV2Event(state, { type: "tool_call.start", toolCallId: "tool-1", - toolName: "prowler_app_search_security_findings", + toolName: "prowler_search_security_findings", }); state = reduceLighthouseV2Event(state, { type: "tool_call.end", @@ -84,7 +84,7 @@ describe("event-reducer", () => { { id: "tool-1", type: "tool_call", - name: "prowler_app_search_security_findings", + name: "prowler_search_security_findings", status: "completed", outcome: "success", }, diff --git a/ui/app/(prowler)/lighthouse/_lib/load-chat-config.ts b/ui/app/(prowler)/lighthouse/_lib/load-chat-config.ts new file mode 100644 index 0000000000..becc488a34 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_lib/load-chat-config.ts @@ -0,0 +1,86 @@ +import { + getLighthouseV2Configurations, + getLighthouseV2SupportedModels, + getLighthouseV2SupportedProviders, +} from "@/app/(prowler)/lighthouse/_actions"; +import type { LighthouseChatConfig } from "@/app/(prowler)/lighthouse/_lib/chat-store"; +import { loadLighthouseV2ConnectedModels } from "@/app/(prowler)/lighthouse/_lib/model-loading"; + +export const LIGHTHOUSE_CHAT_CONFIG_STATUS = { + ERROR: "error", + NOT_CONFIGURED: "not-configured", + READY: "ready", +} as const; + +interface LighthouseChatConfigError { + status: typeof LIGHTHOUSE_CHAT_CONFIG_STATUS.ERROR; + message: string; +} + +interface LighthouseChatConfigNotConfigured { + status: typeof LIGHTHOUSE_CHAT_CONFIG_STATUS.NOT_CONFIGURED; +} + +interface LighthouseChatConfigReady { + status: typeof LIGHTHOUSE_CHAT_CONFIG_STATUS.READY; + config: LighthouseChatConfig; + modelsError?: string; +} + +export type LighthouseChatConfigResult = + | LighthouseChatConfigError + | LighthouseChatConfigNotConfigured + | LighthouseChatConfigReady; + +// Shared by the /lighthouse server page and the client panel: both need the +// same configurations + providers + connected-models bundle. Server actions +// are callable from either context. Rejections propagate to the caller. +export async function loadLighthouseChatConfig(): Promise { + const [configurationsResult, supportedProvidersResult] = await Promise.all([ + getLighthouseV2Configurations(), + getLighthouseV2SupportedProviders(), + ]); + if ("error" in configurationsResult) { + return { + status: LIGHTHOUSE_CHAT_CONFIG_STATUS.ERROR, + message: configurationsResult.error, + }; + } + if ("error" in supportedProvidersResult) { + return { + status: LIGHTHOUSE_CHAT_CONFIG_STATUS.ERROR, + message: supportedProvidersResult.error, + }; + } + + const configurations = configurationsResult.data; + const hasConnectedProvider = configurations.some( + (configuration) => configuration.connected === true, + ); + if (!hasConnectedProvider) { + return { status: LIGHTHOUSE_CHAT_CONFIG_STATUS.NOT_CONFIGURED }; + } + + const { modelsByProvider, failedModelProviders } = + await loadLighthouseV2ConnectedModels( + configurations, + getLighthouseV2SupportedModels, + ); + // Surface (rather than silently swallow to []) connected providers whose + // models failed to load, so their empty list reads as a real backend + // failure. Disconnected providers are never fetched (see model-loading.ts). + const modelsError = + failedModelProviders.length > 0 + ? `Could not load available models for: ${failedModelProviders.join(", ")}. Try again shortly.` + : undefined; + + return { + status: LIGHTHOUSE_CHAT_CONFIG_STATUS.READY, + config: { + configurations, + modelsByProvider, + supportedProviders: supportedProvidersResult.data, + }, + modelsError, + }; +} diff --git a/ui/app/(prowler)/lighthouse/_lib/panel-chat-message-state.ts b/ui/app/(prowler)/lighthouse/_lib/panel-chat-message-state.ts new file mode 100644 index 0000000000..55661c55bd --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_lib/panel-chat-message-state.ts @@ -0,0 +1,44 @@ +type PanelChatMessageStateListener = () => void; + +const listeners = new Set(); +let hasMessages = false; +let activeSessionId: string | null = null; + +export function getPanelChatHasMessages(): boolean { + return hasMessages; +} + +export function subscribePanelChatHasMessages( + listener: PanelChatMessageStateListener, +): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function getPanelChatActiveSessionId(): string | null { + return activeSessionId; +} + +interface PanelChatMessageState { + hasMessages: boolean; + activeSessionId: string | null; +} + +export function setPanelChatMessageState( + nextState: PanelChatMessageState, +): void { + if ( + hasMessages === nextState.hasMessages && + activeSessionId === nextState.activeSessionId + ) { + return; + } + + hasMessages = nextState.hasMessages; + activeSessionId = nextState.activeSessionId; + listeners.forEach((listener) => listener()); +} + +export function resetPanelChatMessageState(): void { + setPanelChatMessageState({ hasMessages: false, activeSessionId: null }); +} diff --git a/ui/app/(prowler)/lighthouse/_lib/panel-chat-store.ts b/ui/app/(prowler)/lighthouse/_lib/panel-chat-store.ts new file mode 100644 index 0000000000..d435fde2b2 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_lib/panel-chat-store.ts @@ -0,0 +1,57 @@ +import { + createLighthouseChatStore, + type LighthouseChatConfig, + type LighthouseChatStore, +} from "@/app/(prowler)/lighthouse/_lib/chat-store"; + +// Module-level singleton: the global side panel keeps the same conversation +// while switching between Details and Lighthouse AI, across route navigation +// and panel closes. The full-page route can reuse it for the same conversation. +let panelChatStore: LighthouseChatStore | null = null; + +interface PanelChatStoreOptions { + initialError?: string; +} + +export function getOrCreatePanelChatStore( + config: LighthouseChatConfig, + options?: PanelChatStoreOptions, +): LighthouseChatStore { + if (!panelChatStore) { + panelChatStore = createLighthouseChatStore({ + config, + syncUrlToSession: false, + initialError: options?.initialError, + }); + } + return panelChatStore; +} + +// Lets the full-page surface reuse the singleton only when both surfaces point +// at the same conversation. This is intentionally a pure lookup: React may +// run state initializers twice in Strict Mode. +export function getPanelChatStoreForSession( + initialSessionId?: string, +): LighthouseChatStore | null { + if (!panelChatStore) return null; + const expectedSessionId = initialSessionId ?? null; + if (panelChatStore.getState().activeSessionId !== expectedSessionId) { + return null; + } + return panelChatStore; +} + +export function isPanelChatStore(store: LighthouseChatStore): boolean { + return panelChatStore === store; +} + +// The config is captured in the store's closure at creation, so a +// configuration change must tear the singleton down and rebuild it. +export function resetPanelChatStore(): void { + panelChatStore?.getState().destroy(); + panelChatStore = null; +} + +export function resetPanelChatStoreForTests(): void { + resetPanelChatStore(); +} diff --git a/ui/app/(prowler)/lighthouse/_lib/session-events.ts b/ui/app/(prowler)/lighthouse/_lib/session-events.ts index ab23282c49..84c722fcd6 100644 --- a/ui/app/(prowler)/lighthouse/_lib/session-events.ts +++ b/ui/app/(prowler)/lighthouse/_lib/session-events.ts @@ -30,3 +30,43 @@ export function notifyLighthouseV2NewChat() { if (typeof window === "undefined") return; window.dispatchEvent(new Event(LIGHTHOUSE_V2_NEW_CHAT_EVENT)); } + +export const LIGHTHOUSE_V2_CONFIGURATIONS_CHANGED_EVENT = + "lighthouse-v2:configurations-changed"; + +// Fired after provider configuration CRUD so cached chat configs (the panel +// keeps one at module scope) can invalidate and reload. +export function notifyLighthouseV2ConfigurationsChanged() { + if (typeof window === "undefined") return; + window.dispatchEvent(new Event(LIGHTHOUSE_V2_CONFIGURATIONS_CHANGED_EVENT)); +} + +// Typed subscribe helpers: each returns an unsubscribe function so consumers +// never hand-roll addEventListener plus the CustomEvent detail cast. +function subscribe(eventName: string, handler: (event: Event) => void) { + if (typeof window === "undefined") return () => {}; + window.addEventListener(eventName, handler); + return () => window.removeEventListener(eventName, handler); +} + +export function onLighthouseV2SessionsChanged(callback: () => void) { + return subscribe(LIGHTHOUSE_V2_SESSIONS_CHANGED_EVENT, callback); +} + +export function onLighthouseV2SessionArchived( + callback: (sessionId: string) => void, +) { + return subscribe(LIGHTHOUSE_V2_SESSION_ARCHIVED_EVENT, (event) => { + const sessionId = (event as CustomEvent<{ sessionId: string }>).detail + ?.sessionId; + if (sessionId) callback(sessionId); + }); +} + +export function onLighthouseV2NewChat(callback: () => void) { + return subscribe(LIGHTHOUSE_V2_NEW_CHAT_EVENT, callback); +} + +export function onLighthouseV2ConfigurationsChanged(callback: () => void) { + return subscribe(LIGHTHOUSE_V2_CONFIGURATIONS_CHANGED_EVENT, callback); +} diff --git a/ui/app/(prowler)/lighthouse/_lib/testing/event-source-mock.ts b/ui/app/(prowler)/lighthouse/_lib/testing/event-source-mock.ts new file mode 100644 index 0000000000..2672509510 --- /dev/null +++ b/ui/app/(prowler)/lighthouse/_lib/testing/event-source-mock.ts @@ -0,0 +1,49 @@ +import { vi } from "vitest"; + +// Controllable EventSource mock: records each instance so tests can drive +// named SSE events and connection failures, while still being a vi.fn so +// `expect(EventSource).toHaveBeenCalledWith(...)` keeps working. +export interface MockEventSource { + url: string; + readyState: number; + onerror: ((event: Event) => void) | null; + listeners: Map>; + addEventListener: (type: string, cb: EventListener) => void; + close: ReturnType; + emit: (type: string, data: unknown) => void; + fail: (readyState: number) => void; +} + +// The mock never fires "open": the client must POST the message without +// waiting for it (the backend sends no bytes until the worker emits, which +// only happens after the POST). This is the regression guard for the +// open-gate deadlock. +export function stubEventSource(): MockEventSource[] { + const eventSources: MockEventSource[] = []; + const EventSourceMock = vi.fn(function (this: MockEventSource, url: string) { + this.url = url; + this.readyState = 0; + this.onerror = null; + this.listeners = new Map(); + this.addEventListener = (type: string, cb: EventListener) => { + const set = this.listeners.get(type) ?? new Set(); + set.add(cb); + this.listeners.set(type, set); + }; + this.close = vi.fn(() => { + this.readyState = 2; + }); + this.emit = (type: string, data: unknown) => { + const event = new MessageEvent(type, { data: JSON.stringify(data) }); + this.listeners.get(type)?.forEach((cb) => cb(event)); + }; + this.fail = (readyState: number) => { + this.readyState = readyState; + this.onerror?.(new Event("error")); + }; + eventSources.push(this); + }); + Object.assign(EventSourceMock, { CONNECTING: 0, OPEN: 1, CLOSED: 2 }); + vi.stubGlobal("EventSource", EventSourceMock); + return eventSources; +} diff --git a/ui/app/(prowler)/lighthouse/_lib/tool-calls.test.ts b/ui/app/(prowler)/lighthouse/_lib/tool-calls.test.ts index 570e7cc597..5ccc2b4129 100644 --- a/ui/app/(prowler)/lighthouse/_lib/tool-calls.test.ts +++ b/ui/app/(prowler)/lighthouse/_lib/tool-calls.test.ts @@ -11,7 +11,7 @@ describe("getToolCallContent", () => { // Given const content = { tool_call_id: "call_1", - tool_name: "prowler_app_search_security_findings", + tool_name: "prowler_search_security_findings", arguments: { severity: "high" }, result: { count: 3 }, outcome: "success", @@ -23,7 +23,7 @@ describe("getToolCallContent", () => { // Then expect(parsed).toEqual({ toolCallId: "call_1", - toolName: "prowler_app_search_security_findings", + toolName: "prowler_search_security_findings", arguments: { severity: "high" }, result: { count: 3 }, outcome: "success", @@ -53,12 +53,18 @@ describe("getToolCallContent", () => { describe("formatToolName", () => { it("should strip the prowler prefix and title-case", () => { - expect(formatToolName("prowler_app_search_security_findings")).toBe( + expect(formatToolName("prowler_search_security_findings")).toBe( "Search security findings", ); expect(formatToolName("prowler_hub_list_checks")).toBe("List checks"); }); + it("should still strip the legacy prowler_app_ prefix", () => { + expect(formatToolName("prowler_app_search_security_findings")).toBe( + "Search security findings", + ); + }); + it("should humanize prefix-less tools", () => { expect(formatToolName("search_tools")).toBe("Search tools"); }); diff --git a/ui/app/(prowler)/lighthouse/_lib/tool-calls.ts b/ui/app/(prowler)/lighthouse/_lib/tool-calls.ts index 46034102d5..88e478089b 100644 --- a/ui/app/(prowler)/lighthouse/_lib/tool-calls.ts +++ b/ui/app/(prowler)/lighthouse/_lib/tool-calls.ts @@ -1,11 +1,14 @@ import type { LighthouseV2ToolCallContent } from "@/app/(prowler)/lighthouse/_types"; // Prefixes shared by the MCP-sourced tools; stripped for display so a name like -// `prowler_app_search_security_findings` reads as "Search security findings". +// `prowler_search_security_findings` reads as "Search security findings". The +// specific prefixes are listed before the bare `prowler_` catch-all so Hub, Docs, +// and legacy `prowler_app_` records match first and render cleanly. const TOOL_NAME_PREFIXES = [ - "prowler_app_", "prowler_hub_", "prowler_docs_", + "prowler_app_", + "prowler_", ] as const; // Reads the snake_case TOOL_CALL blob the backend persists and normalizes it to diff --git a/ui/app/(prowler)/lighthouse/page.tsx b/ui/app/(prowler)/lighthouse/page.tsx index 625aaf795a..f54be45aed 100644 --- a/ui/app/(prowler)/lighthouse/page.tsx +++ b/ui/app/(prowler)/lighthouse/page.tsx @@ -4,14 +4,12 @@ import { getLighthouseProvidersConfig, isLighthouseConfigured, } from "@/actions/lighthouse-v1/lighthouse"; -import { - getLighthouseV2Configurations, - getLighthouseV2Messages, - getLighthouseV2SupportedModels, - getLighthouseV2SupportedProviders, -} from "@/app/(prowler)/lighthouse/_actions"; +import { getLighthouseV2Messages } from "@/app/(prowler)/lighthouse/_actions"; import { LighthouseV2ChatPage } from "@/app/(prowler)/lighthouse/_components/chat"; -import { loadLighthouseV2ConnectedModels } from "@/app/(prowler)/lighthouse/_lib/model-loading"; +import { + LIGHTHOUSE_CHAT_CONFIG_STATUS, + loadLighthouseChatConfig, +} from "@/app/(prowler)/lighthouse/_lib/load-chat-config"; import { LighthouseIcon } from "@/components/icons/Icons"; import { APP_SIDEBAR_MODE, @@ -36,34 +34,13 @@ export default async function AIChatbot({ typeof params.session === "string" ? params.session : undefined; if (isCloud()) { - const [configurationsResult, supportedProvidersResult] = await Promise.all([ - getLighthouseV2Configurations(), - getLighthouseV2SupportedProviders(), - ]); - const configurations = - "data" in configurationsResult ? configurationsResult.data : []; - const supportedProviders = - "data" in supportedProvidersResult ? supportedProvidersResult.data : []; - const connectedConfigurations = configurations.filter( - (configuration) => configuration.connected === true, - ); - - if (connectedConfigurations.length === 0) { + const chatConfigResult = await loadLighthouseChatConfig(); + // Errors and the not-configured case both land on settings, where the + // user can connect (or fix) a provider. + if (chatConfigResult.status !== LIGHTHOUSE_CHAT_CONFIG_STATUS.READY) { return redirect(LIGHTHOUSE_ROUTE.SETTINGS); } - - const { modelsByProvider, failedModelProviders } = - await loadLighthouseV2ConnectedModels( - configurations, - getLighthouseV2SupportedModels, - ); - // Surface (rather than silently swallow to []) connected providers whose - // models failed to load, so their empty list reads as a real backend - // failure. Disconnected providers are never fetched (see model-loading.ts). - const modelsError = - failedModelProviders.length > 0 - ? `Could not load available models for: ${failedModelProviders.join(", ")}. Try again shortly.` - : undefined; + const { config, modelsError } = chatConfigResult; const initialMessages = activeSessionId ? await getLighthouseV2Messages(activeSessionId) @@ -80,15 +57,15 @@ export default async function AIChatbot({ return ( }> - + {/* [contain:layout] traps streamdown's fixed fullscreen overlay inside the chat area so it never covers the sidebar or navbar. */}
({ Modal: ({ @@ -21,8 +22,6 @@ vi.mock("@/components/shadcn/modal", () => ({ ) : null, })); -import { MuteRuleTargetsModal } from "./mute-rule-targets-modal"; - const longMuteRule: MuteRuleTableData = { type: "mute-rules", id: "mute-rule-1", diff --git a/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx b/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx index e9086cd366..83cfe429bb 100644 --- a/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx +++ b/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx @@ -17,8 +17,8 @@ import { FieldLabel, Input, Textarea, + useToast, } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; import { Modal } from "@/components/shadcn/modal"; import { DOCS_URLS } from "@/lib/external-urls"; diff --git a/ui/app/(prowler)/scans/config/_components/scan-configurations-manager.tsx b/ui/app/(prowler)/scans/config/_components/scan-configurations-manager.tsx index 2599b7291b..618ee1b8d0 100644 --- a/ui/app/(prowler)/scans/config/_components/scan-configurations-manager.tsx +++ b/ui/app/(prowler)/scans/config/_components/scan-configurations-manager.tsx @@ -10,8 +10,7 @@ import { import { AccountsSelector } from "@/app/(prowler)/_overview/_components/accounts-selector"; import { BatchFiltersLayout } from "@/components/filters/batch-filters-layout"; import { ClearFiltersButton } from "@/components/filters/clear-filters-button"; -import { Button, Card } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Button, Card, useToast } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; import { Modal } from "@/components/shadcn/modal"; import { DataTable } from "@/components/shadcn/table"; diff --git a/ui/app/(prowler)/scans/page.tsx b/ui/app/(prowler)/scans/page.tsx index 5bbfbdf213..66635d7443 100644 --- a/ui/app/(prowler)/scans/page.tsx +++ b/ui/app/(prowler)/scans/page.tsx @@ -9,6 +9,7 @@ import { } from "@/actions/scans/scans-filters"; import { getSchedules, getSchedulesPage } from "@/actions/schedules"; import { auth } from "@/auth.config"; +import { ScansPageShell } from "@/components/scans/scans-page-shell"; import { appendPendingScheduleRowsToPage, buildScheduledTabRows, @@ -18,7 +19,6 @@ import { getScanJobsUserFilters, pickScheduleProviderFilters, } from "@/components/scans/scans.utils"; -import { ScansPageShell } from "@/components/scans/scans-page-shell"; import { SkeletonTableScans } from "@/components/scans/table"; import { ScanJobsTable } from "@/components/scans/table/scan-jobs-table"; import { ContentLayout } from "@/components/shadcn/content-layout"; diff --git a/ui/changelog.d/dynamic-provider-alias-delete.changed.md b/ui/changelog.d/dynamic-provider-alias-delete.changed.md deleted file mode 100644 index 75adeeeffb..0000000000 --- a/ui/changelog.d/dynamic-provider-alias-delete.changed.md +++ /dev/null @@ -1 +0,0 @@ -Dynamic providers can now be renamed and deleted from the Providers table diff --git a/ui/changelog.d/enterprise-billing-navigation.fixed.md b/ui/changelog.d/enterprise-billing-navigation.fixed.md new file mode 100644 index 0000000000..0ea2b4d31e --- /dev/null +++ b/ui/changelog.d/enterprise-billing-navigation.fixed.md @@ -0,0 +1 @@ +Billing navigation is hidden when Cloud billing is disabled, including Enterprise deployments diff --git a/ui/changelog.d/findings-scan-filter-selection.fixed.md b/ui/changelog.d/findings-scan-filter-selection.fixed.md deleted file mode 100644 index ff1f70c117..0000000000 --- a/ui/changelog.d/findings-scan-filter-selection.fixed.md +++ /dev/null @@ -1 +0,0 @@ -`Scan ID` filter on the Findings page now shows the active scan when opening findings from a scan's `View Findings` action diff --git a/ui/changelog.d/findings-timeline-y-axis.fixed.md b/ui/changelog.d/findings-timeline-y-axis.fixed.md new file mode 100644 index 0000000000..4dd338070f --- /dev/null +++ b/ui/changelog.d/findings-timeline-y-axis.fixed.md @@ -0,0 +1 @@ +Findings Severity Over Time chart Y-axis labels no longer overflow for large findings counts diff --git a/ui/changelog.d/oci-regionless-provider-e2e.fixed.md b/ui/changelog.d/oci-regionless-provider-e2e.fixed.md new file mode 100644 index 0000000000..2e413a6fa8 --- /dev/null +++ b/ui/changelog.d/oci-regionless-provider-e2e.fixed.md @@ -0,0 +1 @@ +OCI provider E2E tests no longer require or submit a region when adding or updating credentials diff --git a/ui/changelog.d/sidebar-redesign.changed.md b/ui/changelog.d/sidebar-redesign.changed.md deleted file mode 100644 index 9d45d8ffdf..0000000000 --- a/ui/changelog.d/sidebar-redesign.changed.md +++ /dev/null @@ -1 +0,0 @@ -Sidebar navigation with grouped sections, clearer active states, and a responsive mobile overlay diff --git a/ui/changelog.d/ui-sentry-actionability.fixed.md b/ui/changelog.d/ui-sentry-actionability.fixed.md new file mode 100644 index 0000000000..7a4b8ce2ca --- /dev/null +++ b/ui/changelog.d/ui-sentry-actionability.fixed.md @@ -0,0 +1 @@ +UI Sentry alerts now suppress non-actionable warnings and expected API/control-flow noise while preserving actionable runtime failures diff --git a/ui/components/ThemeSwitch.test.tsx b/ui/components/ThemeSwitch.test.tsx new file mode 100644 index 0000000000..5d0ffde75b --- /dev/null +++ b/ui/components/ThemeSwitch.test.tsx @@ -0,0 +1,52 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { ThemeSwitch } from "./ThemeSwitch"; + +const { setThemeMock, themeState } = vi.hoisted(() => ({ + setThemeMock: vi.fn(), + themeState: { current: "light" }, +})); + +vi.mock("next-themes", () => ({ + useTheme: () => ({ theme: themeState.current, setTheme: setThemeMock }), +})); + +describe("ThemeSwitch", () => { + beforeEach(() => { + setThemeMock.mockClear(); + themeState.current = "light"; + }); + + it("exposes an accessible switch reflecting the current mode", () => { + // Given / When + render(); + + // Then + const control = screen.getByRole("switch", { + name: "Switch to dark mode", + }); + expect(control).toHaveAttribute("aria-checked", "true"); + }); + + it("toggles to the opposite theme on click", () => { + // Given + render(); + + // When + fireEvent.click(screen.getByRole("switch")); + + // Then + expect(setThemeMock).toHaveBeenCalledWith("dark"); + }); + + it("renders as a shared ghost icon button, matching the navbar cluster", () => { + // Given / When + render(); + + // Then: same 32px square treatment as the other navbar actions + const control = screen.getByRole("switch"); + expect(control).toHaveClass("size-8"); + expect(control).not.toHaveClass("rounded-full"); + }); +}); diff --git a/ui/components/ThemeSwitch.tsx b/ui/components/ThemeSwitch.tsx index 4979dec541..8de15b3531 100644 --- a/ui/components/ThemeSwitch.tsx +++ b/ui/components/ThemeSwitch.tsx @@ -3,12 +3,12 @@ import { useTheme } from "next-themes"; import { ComponentProps, useSyncExternalStore } from "react"; +import { Button } from "@/components/shadcn/button/button"; import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/shadcn/tooltip"; -import { cn } from "@/lib/utils"; import { MoonFilledIcon, SunFilledIcon } from "./icons"; @@ -30,24 +30,23 @@ export function ThemeSwitch({ className, ...props }: ThemeSwitchProps) { return ( - + {isLightMode ? "Switch to Dark Mode" : "Switch to Light Mode"} diff --git a/ui/components/auth/oss/auth-brand.tsx b/ui/components/auth/oss/auth-brand.tsx new file mode 100644 index 0000000000..396064c52b --- /dev/null +++ b/ui/components/auth/oss/auth-brand.tsx @@ -0,0 +1,14 @@ +import { ProwlerBrand } from "@/components/icons"; +import { cn } from "@/lib/utils"; + +interface AuthBrandProps { + className?: string; +} + +export const AuthBrand = ({ className }: AuthBrandProps) => { + return ( +
+ +
+ ); +}; diff --git a/ui/components/auth/oss/auth-divider.tsx b/ui/components/auth/oss/auth-divider.tsx index 2d9fe56b30..edf4a1c736 100644 --- a/ui/components/auth/oss/auth-divider.tsx +++ b/ui/components/auth/oss/auth-divider.tsx @@ -2,9 +2,9 @@ import { Separator } from "@/components/shadcn"; export const AuthDivider = () => { return ( -
+
-

OR

+

or

); diff --git a/ui/components/auth/oss/auth-layout.test.tsx b/ui/components/auth/oss/auth-layout.test.tsx new file mode 100644 index 0000000000..dd144243da --- /dev/null +++ b/ui/components/auth/oss/auth-layout.test.tsx @@ -0,0 +1,39 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { AuthLayout } from "./auth-layout"; + +describe("AuthLayout", () => { + it("renders the Prowler brand directly above the form card", () => { + render( + +

form content

+ , + ); + + const brand = screen.getByRole("img", { name: /prowler/i }); + const title = screen.getByText("Sign in"); + + expect( + brand.compareDocumentPosition(title) & Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); + }); + + it("renders the footer outside the form card, below it", () => { + render( + footer link

}> +

form content

+ , + ); + + const content = screen.getByText("form content"); + const footer = screen.getByText("footer link"); + const card = content.parentElement!; + + expect(card.contains(footer)).toBe(false); + expect( + content.compareDocumentPosition(footer) & + Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); + }); +}); diff --git a/ui/components/auth/oss/auth-layout.tsx b/ui/components/auth/oss/auth-layout.tsx index 6d90c28844..8c122aa661 100644 --- a/ui/components/auth/oss/auth-layout.tsx +++ b/ui/components/auth/oss/auth-layout.tsx @@ -2,12 +2,15 @@ import { ReactNode } from "react"; import { ThemeSwitch } from "@/components/ThemeSwitch"; +import { AuthBrand } from "./auth-brand"; + interface AuthLayoutProps { title: string; + footer?: ReactNode; children: ReactNode; } -export const AuthLayout = ({ title, children }: AuthLayoutProps) => { +export const AuthLayout = ({ title, footer, children }: AuthLayoutProps) => { return (
@@ -20,6 +23,8 @@ export const AuthLayout = ({ title, children }: AuthLayoutProps) => { }} >
+ + {/* Auth Form Container */}
{/* Header with Title and Theme Toggle */} @@ -31,6 +36,8 @@ export const AuthLayout = ({ title, children }: AuthLayoutProps) => { {/* Content */} {children}
+ + {footer &&
{footer}
}
); diff --git a/ui/components/auth/oss/public-auth-shell.tsx b/ui/components/auth/oss/public-auth-shell.tsx deleted file mode 100644 index 372d2d866a..0000000000 --- a/ui/components/auth/oss/public-auth-shell.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import type { ReactNode } from "react"; - -import { ProwlerBrand } from "@/components/icons"; - -interface PublicAuthShellProps { - children: ReactNode; -} - -export const PublicAuthShell = ({ children }: PublicAuthShellProps) => { - return ( -
-
- -
- {children} -
- ); -}; diff --git a/ui/components/auth/oss/sign-in-form.tsx b/ui/components/auth/oss/sign-in-form.tsx index 64f3352279..7c971913be 100644 --- a/ui/components/auth/oss/sign-in-form.tsx +++ b/ui/components/auth/oss/sign-in-form.tsx @@ -12,8 +12,13 @@ import { AuthDivider } from "@/components/auth/oss/auth-divider"; import { AuthFooterLink } from "@/components/auth/oss/auth-footer-link"; import { AuthLayout } from "@/components/auth/oss/auth-layout"; import { SocialButtons } from "@/components/auth/oss/social-buttons"; -import { Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { + Button, + Tooltip, + TooltipContent, + TooltipTrigger, + useToast, +} from "@/components/shadcn"; import { CustomInput } from "@/components/shadcn/custom"; import { Form } from "@/components/shadcn/form"; import { getSafeCallbackPath } from "@/lib/auth-callback-url"; @@ -142,10 +147,19 @@ export const SignInForm = ({ } }; - const title = isSamlMode ? "Sign in with SAML SSO" : "Sign in"; + const title = isSamlMode ? "Sign in with SAML SSO" : "Welcome back"; return ( - + + } + >
-
+
{!isSamlMode && ( )} - + {isSamlMode ? ( + + ) : ( + + + + + Continue with SAML SSO + + )}
- - ); }; diff --git a/ui/components/auth/oss/sign-up-form.tsx b/ui/components/auth/oss/sign-up-form.tsx index ac0282aec0..68fe4e2c6a 100644 --- a/ui/components/auth/oss/sign-up-form.tsx +++ b/ui/components/auth/oss/sign-up-form.tsx @@ -15,8 +15,7 @@ import { AuthFooterLink } from "@/components/auth/oss/auth-footer-link"; import { AuthLayout } from "@/components/auth/oss/auth-layout"; import { PasswordRequirementsMessage } from "@/components/auth/oss/password-validator"; import { SocialButtons } from "@/components/auth/oss/social-buttons"; -import { Button, Checkbox } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Button, Checkbox, useToast } from "@/components/shadcn"; import { CustomInput } from "@/components/shadcn/custom"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; import { @@ -161,7 +160,16 @@ export const SignUpForm = ({ }; return ( - + + } + > -
+
)} - - ); }; diff --git a/ui/components/auth/oss/social-buttons.test.tsx b/ui/components/auth/oss/social-buttons.test.tsx new file mode 100644 index 0000000000..7a406f4643 --- /dev/null +++ b/ui/components/auth/oss/social-buttons.test.tsx @@ -0,0 +1,41 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import { SocialButtons } from "./social-buttons"; + +// Stub Iconify: the real fetches icon data over the network and its +// retry timers can fire after the jsdom environment is torn down, crashing the +// worker with "window is not defined". +vi.mock("@iconify/react", () => ({ + Icon: ({ icon }: { icon: string }) => , +})); + +describe("SocialButtons", () => { + it("renders icon-only provider links that keep their accessible names", () => { + render( + , + ); + + const google = screen.getByRole("link", { name: "Continue with Google" }); + const github = screen.getByRole("link", { name: "Continue with Github" }); + + expect(google.textContent).toBe(""); + expect(github.textContent).toBe(""); + }); + + it("keeps accessible names on disabled providers", () => { + render(); + + expect( + screen.getByRole("button", { name: "Continue with Google" }), + ).toBeDisabled(); + expect( + screen.getByRole("button", { name: "Continue with Github" }), + ).toBeDisabled(); + }); +}); diff --git a/ui/components/auth/oss/social-buttons.tsx b/ui/components/auth/oss/social-buttons.tsx index b08d830fb5..fd52d0cfb7 100644 --- a/ui/components/auth/oss/social-buttons.tsx +++ b/ui/components/auth/oss/social-buttons.tsx @@ -35,12 +35,13 @@ const SocialButton = ({ const button = ( ); if (!isDisabled) { - return button; + return ( + + {button} + {provider.label} + + ); } return ( - {button} + {button} {provider.isOAuthEnabled ? ( diff --git a/ui/components/compliance/compliance-accordion/client-accordion-wrapper.tsx b/ui/components/compliance/compliance-accordion/client-accordion-wrapper.tsx index e3ab821a55..256431cdfb 100644 --- a/ui/components/compliance/compliance-accordion/client-accordion-wrapper.tsx +++ b/ui/components/compliance/compliance-accordion/client-accordion-wrapper.tsx @@ -2,8 +2,7 @@ import { useRef, useState } from "react"; -import { Button } from "@/components/shadcn"; -import { Accordion, AccordionItemProps } from "@/components/shadcn"; +import { Button, Accordion, AccordionItemProps } from "@/components/shadcn"; import { Card } from "@/components/shadcn/card/card"; export const ClientAccordionWrapper = ({ diff --git a/ui/components/compliance/compliance-card.tsx b/ui/components/compliance/compliance-card.tsx index 6168784f2f..37c7892fda 100644 --- a/ui/components/compliance/compliance-card.tsx +++ b/ui/components/compliance/compliance-card.tsx @@ -19,6 +19,7 @@ import { import { ScanEntity } from "@/types/scans"; import { getComplianceIcon } from "../icons"; + import { ComplianceDownloadContainer } from "./compliance-download-container"; interface ComplianceCardProps { diff --git a/ui/components/compliance/compliance-charts/heatmap-chart.test.tsx b/ui/components/compliance/compliance-charts/heatmap-chart.test.tsx new file mode 100644 index 0000000000..409974af77 --- /dev/null +++ b/ui/components/compliance/compliance-charts/heatmap-chart.test.tsx @@ -0,0 +1,36 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { HeatmapChart } from "./heatmap-chart"; + +vi.mock("next-themes", () => ({ + useTheme: () => ({ theme: "light" }), +})); + +describe("HeatmapChart", () => { + it("portals its pointer-positioned tooltip outside layout containers", async () => { + // Given + const user = userEvent.setup(); + const { container } = render( + , + ); + + // When + await user.hover(screen.getByTitle("Identity")); + + // Then: fixed client coordinates resolve against the viewport, not
+ const tooltip = screen.getByRole("tooltip"); + expect(container).not.toContainElement(tooltip); + expect(tooltip.parentElement).toBe(document.body); + }); +}); diff --git a/ui/components/compliance/compliance-charts/heatmap-chart.tsx b/ui/components/compliance/compliance-charts/heatmap-chart.tsx index 2d7ce547b3..f57fa70aee 100644 --- a/ui/components/compliance/compliance-charts/heatmap-chart.tsx +++ b/ui/components/compliance/compliance-charts/heatmap-chart.tsx @@ -2,6 +2,7 @@ import { useTheme } from "next-themes"; import { useState } from "react"; +import { createPortal } from "react-dom"; import { cn } from "@/lib/utils"; import { CategoryData } from "@/types/compliance"; @@ -115,44 +116,49 @@ export const HeatmapChart = ({ categories = [] }: HeatmapChartProps) => {
{/* Custom Tooltip */} - {hoveredItem && ( -
-
- {capitalizeFirstLetter(hoveredItem.name)} -
-
- - Failure Rate: {hoveredItem.failurePercentage}% - -
-
- - Failed: {hoveredItem.failedRequirements}/ - {hoveredItem.totalRequirements} - -
-
- )} +
+ {capitalizeFirstLetter(hoveredItem.name)} +
+
+ + Failure Rate: {hoveredItem.failurePercentage}% + +
+
+ + Failed: {hoveredItem.failedRequirements}/ + {hoveredItem.totalRequirements} + +
+
, + document.body, + ) + : null}
); diff --git a/ui/components/feeds/feeds-client.tsx b/ui/components/feeds/feeds-client.tsx index e91e16b6d6..6b0e45e367 100644 --- a/ui/components/feeds/feeds-client.tsx +++ b/ui/components/feeds/feeds-client.tsx @@ -58,8 +58,9 @@ export function FeedsClient({ feedData, error }: FeedsClientProps) { -
+ {/* Portaled to body:
is a layout container (container queries), + which would otherwise capture this fixed button and scroll it away + with the content. */} + {typeof document !== "undefined" + ? createPortal( +
+ +
, + document.body, + ) + : null} ); } diff --git a/ui/components/findings/table/column-finding-groups.test.tsx b/ui/components/findings/table/column-finding-groups.test.tsx index 5a8ec51e56..ca6e01bb7b 100644 --- a/ui/components/findings/table/column-finding-groups.test.tsx +++ b/ui/components/findings/table/column-finding-groups.test.tsx @@ -409,7 +409,9 @@ describe("column-finding-groups — accessibility of check title cell", () => { expect( screen.queryByRole("button", { name: "Fallback IaC Check" }), ).not.toBeInTheDocument(); - expect(screen.getByText("Fallback IaC Check")).toBeInTheDocument(); + // The title renders as plain (non-clickable) text and the inline-mocked + // tooltip duplicates it, so exactly both copies must exist. + expect(screen.getAllByText("Fallback IaC Check")).toHaveLength(2); expect(onDrillDown).not.toHaveBeenCalled(); }); }); diff --git a/ui/components/findings/table/column-finding-groups.tsx b/ui/components/findings/table/column-finding-groups.tsx index f200a4ed39..e5cd81cb4d 100644 --- a/ui/components/findings/table/column-finding-groups.tsx +++ b/ui/components/findings/table/column-finding-groups.tsx @@ -199,20 +199,29 @@ export function getColumnFindingGroups({ {providerName} ) : null} -
- {canExpand ? ( - + ) : ( + + {group.checkTitle} + + )} + + {group.checkTitle} - - ) : ( - - {group.checkTitle} - - )} + +
); diff --git a/ui/components/findings/table/column-standalone-findings.tsx b/ui/components/findings/table/column-standalone-findings.tsx index 75887ee9b9..30facaf97f 100644 --- a/ui/components/findings/table/column-standalone-findings.tsx +++ b/ui/components/findings/table/column-standalone-findings.tsx @@ -9,6 +9,11 @@ import { SeverityBadge, StatusFindingBadge, } from "@/components/shadcn/table"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; import { getRegionFlag } from "@/lib/region-flags"; import { getOptionalText } from "@/lib/utils"; import { FindingProps, ProviderType } from "@/types"; @@ -67,11 +72,17 @@ function FindingTitleCell({ finding={finding} defaultOpen={defaultOpen} trigger={ -
-

+ // Single line always: ellipsis beyond the max, full title in the tooltip. + + +

+ {finding.attributes.check_metadata.checktitle} +

+ + {finding.attributes.check_metadata.checktitle} -

-
+ + } /> ); diff --git a/ui/components/findings/table/findings-group-drill-down.tsx b/ui/components/findings/table/findings-group-drill-down.tsx index 0b96a9657e..b71787ef37 100644 --- a/ui/components/findings/table/findings-group-drill-down.tsx +++ b/ui/components/findings/table/findings-group-drill-down.tsx @@ -20,8 +20,9 @@ import { TableHead, TableHeader, TableRow, + SeverityBadge, + StatusFindingBadge, } from "@/components/shadcn/table"; -import { SeverityBadge, StatusFindingBadge } from "@/components/shadcn/table"; import { useFindingGroupResourceState } from "@/hooks/use-finding-group-resource-state"; import { cn, hasHistoricalFindingFilter } from "@/lib"; import { @@ -32,6 +33,7 @@ import { import { FindingGroupRow } from "@/types"; import { FloatingMuteButton } from "../floating-mute-button"; + import { getColumnFindingResources } from "./column-finding-resources"; import { FindingsSelectionContext } from "./findings-selection-context"; import { ImpactedResourcesCell } from "./impacted-resources-cell"; diff --git a/ui/components/findings/table/findings-group-table.tsx b/ui/components/findings/table/findings-group-table.tsx index 10eeff0b62..986cba7f09 100644 --- a/ui/components/findings/table/findings-group-table.tsx +++ b/ui/components/findings/table/findings-group-table.tsx @@ -14,6 +14,7 @@ import { createExploreFindingsTourStepHandlers } from "@/lib/tours/explore-findi import { FindingGroupRow, MetaDataProps } from "@/types"; import { FloatingMuteButton } from "../floating-mute-button"; + import { getColumnFindingGroups } from "./column-finding-groups"; import { canMuteFindingGroup } from "./finding-group-selection"; import { FindingsSelectionContext } from "./findings-selection-context"; diff --git a/ui/components/findings/table/resource-detail-drawer/resource-detail-drawer-content.tsx b/ui/components/findings/table/resource-detail-drawer/resource-detail-drawer-content.tsx index af7e38acc3..f18f0b671a 100644 --- a/ui/components/findings/table/resource-detail-drawer/resource-detail-drawer-content.tsx +++ b/ui/components/findings/table/resource-detail-drawer/resource-detail-drawer-content.tsx @@ -71,9 +71,9 @@ import { type QueryEditorLanguage, } from "@/components/shared/query-code-editor"; import { ResourceMetadataPanel } from "@/components/shared/resource-metadata-panel"; -import { getFailingForLabel } from "@/lib/date-utils"; -import { formatDuration } from "@/lib/date-utils"; +import { getFailingForLabel, formatDuration } from "@/lib/date-utils"; import { shouldRefreshAfterTriageUpdate } from "@/lib/finding-triage"; +import { buildFindingAnalysisPrompt } from "@/lib/lighthouse/prompts"; import { getRegionFlag } from "@/lib/region-flags"; import { getRecommendationLinkLabel } from "@/lib/vulnerability-references"; import type { ComplianceOverviewData } from "@/types/compliance"; @@ -88,6 +88,7 @@ import { FindingTriageStatusCell, } from "../finding-triage-cells"; import { DeltaValues, NotificationIndicator } from "../notification-indicator"; + import { ResourceDetailSkeleton } from "./resource-detail-skeleton"; import type { CheckMeta } from "./use-resource-detail-drawer"; @@ -471,6 +472,16 @@ export function ResourceDetailDrawerContent({ const overviewStatusExtended = currentResource?.statusExtended || f?.statusExtended; const showOverviewStatusExtended = Boolean(overviewStatusExtended); + const findingAnalysisPrompt = buildFindingAnalysisPrompt({ + findingId: currentResource?.findingId ?? f?.id, + providerUid, + resourceUid, + checkId: currentResource?.checkId ?? checkMeta.checkId, + severity: findingSeverity, + status: findingStatus, + detail: overviewStatusExtended, + risk: f?.risk || checkMeta.risk, + }); const handleDrawerTriageUpdate = async (input: UpdateFindingTriageInput) => { await updateFindingTriage(input); @@ -712,7 +723,10 @@ export function ResourceDetailDrawerContent({ <>
{/* Resource info grid — 4 data columns */} -
+
{/* Row 1: Provider, Resource, Service, Region */}
- - - Resource Finding Details - - View finding details for the selected resource - - - - - Close - - {open && ( - - )} - - + + + ); } diff --git a/ui/components/findings/table/resource-detail-drawer/resource-detail-skeleton.tsx b/ui/components/findings/table/resource-detail-drawer/resource-detail-skeleton.tsx index 938b4bc35a..163a2dbd54 100644 --- a/ui/components/findings/table/resource-detail-drawer/resource-detail-skeleton.tsx +++ b/ui/components/findings/table/resource-detail-drawer/resource-detail-skeleton.tsx @@ -8,7 +8,10 @@ import { Skeleton } from "@/components/shadcn/skeleton/skeleton"; export function ResourceDetailSkeleton() { return (
-
+
{/* Row 1: Provider, Resource, Service, Region */}
diff --git a/ui/components/graphs/line-chart.test.tsx b/ui/components/graphs/line-chart.test.tsx new file mode 100644 index 0000000000..3febcd46f3 --- /dev/null +++ b/ui/components/graphs/line-chart.test.tsx @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; + +import { formatYAxisTick } from "./line-chart.utils"; + +describe("formatYAxisTick", () => { + describe("when findings counts are large", () => { + it("should compact six-digit values so Y-axis labels do not overflow", () => { + // Given + const tickValue = 150000; + + // When + const formattedValue = formatYAxisTick(tickValue); + + // Then + expect(formattedValue).toBe("150K"); + }); + + it("should compact million-scale values", () => { + // Given + const tickValue = 1200000; + + // When + const formattedValue = formatYAxisTick(tickValue); + + // Then + expect(formattedValue).toBe("1.2M"); + }); + }); + + describe("when findings counts are small", () => { + it("should keep values below 1000 readable without compact notation", () => { + // Given + const tickValue = 999; + + // When + const formattedValue = formatYAxisTick(tickValue); + + // Then + expect(formattedValue).toBe("999"); + }); + }); +}); diff --git a/ui/components/graphs/line-chart.tsx b/ui/components/graphs/line-chart.tsx index eab0551bb6..1ffdd19dc3 100644 --- a/ui/components/graphs/line-chart.tsx +++ b/ui/components/graphs/line-chart.tsx @@ -17,6 +17,7 @@ import { ChartTooltip, } from "@/components/shadcn/chart/Chart"; +import { formatYAxisTick } from "./line-chart.utils"; import { AlertPill } from "./shared/alert-pill"; import { ChartLegend } from "./shared/chart-legend"; import { CustomActiveDot, PointClickData } from "./shared/custom-active-dot"; @@ -222,6 +223,8 @@ export function LineChart({ tickLine={false} axisLine={false} tickMargin={8} + tickFormatter={formatYAxisTick} + width={56} padding={{ top: 20 }} tick={{ fill: "var(--color-text-neutral-secondary)", diff --git a/ui/components/graphs/line-chart.utils.ts b/ui/components/graphs/line-chart.utils.ts new file mode 100644 index 0000000000..b219529c24 --- /dev/null +++ b/ui/components/graphs/line-chart.utils.ts @@ -0,0 +1,8 @@ +const Y_AXIS_TICK_FORMATTER = new Intl.NumberFormat("en-US", { + notation: "compact", + maximumFractionDigits: 1, +}); + +export function formatYAxisTick(value: number) { + return Y_AXIS_TICK_FORMATTER.format(value); +} diff --git a/ui/components/icons/Icons.tsx b/ui/components/icons/Icons.tsx index a4053e9f55..0a2edb0526 100644 --- a/ui/components/icons/Icons.tsx +++ b/ui/components/icons/Icons.tsx @@ -1129,7 +1129,13 @@ export const LighthouseIcon: React.FC = ({ height, ...props }) => { - const gradientId = (id: string) => (animatedAura ? `${id}_animated` : id); + // Gradient defs are referenced by id, and this icon renders many times per + // page. Duplicate ids resolve against the FIRST instance in the document — + // if that one sits in a display:none subtree (e.g. the desktop sidebar on + // mobile), every other copy loses its fill and turns invisible. useId keeps + // each instance self-contained; strip its delimiters for url(#...) safety. + const uid = React.useId().replace(/[«»:]/g, ""); + const gradientId = (id: string) => `${id}_${uid}`; return ( { + it("keeps gradient ids unique across instances", () => { + // Given: the icon rendered several times on one page (sidebar, navbar, + // overview banner) — with duplicate ids, browsers resolve url(#...) + // against the first instance, which may sit in a display:none subtree + // (the desktop sidebar on mobile) and leave the others unpainted. + const { container } = render( + <> + + + + , + ); + + // Then: every gradient id is unique document-wide + const ids = Array.from( + container.querySelectorAll("linearGradient, radialGradient"), + ).map((gradient) => gradient.id); + expect(ids.length).toBeGreaterThan(0); + expect(new Set(ids).size).toBe(ids.length); + }); + + it("paints every path from its own instance's defs", () => { + // Given + const { container } = render(); + const svg = container.querySelector("svg") as SVGElement; + const localIds = new Set( + Array.from(svg.querySelectorAll("linearGradient, radialGradient")).map( + (gradient) => gradient.id, + ), + ); + + // Then: every fill/stroke reference resolves inside this same svg + const references = Array.from(svg.querySelectorAll("path")) + .flatMap((path) => [ + path.getAttribute("fill"), + path.getAttribute("stroke"), + ]) + .filter((paint): paint is string => paint?.startsWith("url(#") ?? false); + expect(references.length).toBeGreaterThan(0); + for (const reference of references) { + const id = reference.slice("url(#".length, -1); + expect(localIds.has(id)).toBe(true); + } + }); +}); diff --git a/ui/components/integrations/jira/jira-integration-card.tsx b/ui/components/integrations/jira/jira-integration-card.tsx index 4629215f5e..b99cf7cdbe 100644 --- a/ui/components/integrations/jira/jira-integration-card.tsx +++ b/ui/components/integrations/jira/jira-integration-card.tsx @@ -4,11 +4,9 @@ import { SettingsIcon } from "lucide-react"; import Link from "next/link"; import { JiraIcon } from "@/components/icons/services/IconServices"; -import { Button } from "@/components/shadcn"; +import { Button, Card, CardContent, CardHeader } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; -import { Card, CardContent, CardHeader } from "../../shadcn"; - export const JiraIntegrationCard = () => { return ( diff --git a/ui/components/integrations/jira/jira-integrations-manager.tsx b/ui/components/integrations/jira/jira-integrations-manager.tsx index c3f3a1412f..e63b17dac3 100644 --- a/ui/components/integrations/jira/jira-integrations-manager.tsx +++ b/ui/components/integrations/jira/jira-integrations-manager.tsx @@ -15,15 +15,19 @@ import { IntegrationCardHeader, IntegrationSkeleton, } from "@/components/integrations/shared"; -import { Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { + Button, + useToast, + Card, + CardContent, + CardHeader, +} from "@/components/shadcn"; import { Modal } from "@/components/shadcn/modal"; import { DataTablePagination } from "@/components/shadcn/table/data-table-pagination"; import { triggerTestConnectionWithDelay } from "@/lib/integrations/test-connection-helper"; import { MetaDataProps } from "@/types"; import { IntegrationProps } from "@/types/integrations"; -import { Card, CardContent, CardHeader } from "../../shadcn"; import { JiraIntegrationForm } from "./jira-integration-form"; interface JiraIntegrationsManagerProps { diff --git a/ui/components/integrations/s3/s3-integration-card.tsx b/ui/components/integrations/s3/s3-integration-card.tsx index 7e2be1890d..be173c5609 100644 --- a/ui/components/integrations/s3/s3-integration-card.tsx +++ b/ui/components/integrations/s3/s3-integration-card.tsx @@ -4,11 +4,9 @@ import { SettingsIcon } from "lucide-react"; import Link from "next/link"; import { AmazonS3Icon } from "@/components/icons/services/IconServices"; -import { Button } from "@/components/shadcn"; +import { Button, Card, CardContent, CardHeader } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; -import { Card, CardContent, CardHeader } from "../../shadcn"; - export const S3IntegrationCard = () => { return ( diff --git a/ui/components/integrations/s3/s3-integration-form.test.tsx b/ui/components/integrations/s3/s3-integration-form.test.tsx new file mode 100644 index 0000000000..7186cc42ed --- /dev/null +++ b/ui/components/integrations/s3/s3-integration-form.test.tsx @@ -0,0 +1,244 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import type { ComponentProps } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { IntegrationProps } from "@/types/integrations"; +import type { ProviderProps } from "@/types/providers"; + +import { S3IntegrationForm } from "./s3-integration-form"; + +const { createIntegrationMock, toastMock, updateIntegrationMock } = vi.hoisted( + () => ({ + createIntegrationMock: vi.fn(), + toastMock: vi.fn(), + updateIntegrationMock: vi.fn(), + }), +); + +vi.mock("@/actions/integrations", () => ({ + createIntegration: createIntegrationMock, + updateIntegration: updateIntegrationMock, +})); + +vi.mock("next-auth/react", () => ({ + useSession: () => ({ + data: { + tenantId: "tenant-id", + }, + }), +})); + +vi.mock("@/components/shadcn", async (importOriginal) => ({ + ...(await importOriginal>()), + useToast: () => ({ + toast: toastMock, + }), +})); + +interface MockEnhancedMultiSelectProps { + onValueChange: (values: string[]) => void; + options: Array<{ value: string }>; +} + +vi.mock("@/components/shadcn/select/enhanced-multi-select", () => ({ + EnhancedMultiSelect: ({ + onValueChange, + options, + }: MockEnhancedMultiSelectProps) => ( + + ), +})); + +vi.mock( + "@/components/providers/workflow/forms/select-credentials-type/aws/credentials-type/aws-role-credentials-form", + () => ({ + AWSRoleCredentialsForm: ({ + templateLinks, + }: { + templateLinks: { cloudformationQuickLink: string }; + }) => ( + + {templateLinks.cloudformationQuickLink} + + ), + }), +); + +vi.mock("@/lib", () => ({ + getAWSCredentialsTemplateLinks: ( + _externalId: string, + _bucketName: string, + _integrationType: string, + bucketAccountId?: string, + ) => ({ + cloudformation: "https://example.com/cloudformation", + terraform: "https://example.com/terraform", + cloudformationQuickLink: `https://example.com/quick-create?bucketAccountId=${bucketAccountId ?? ""}`, + }), +})); + +function createProvider( + provider: ProviderProps["attributes"]["provider"], + uid: string, +): ProviderProps { + return { + id: `${provider}-provider`, + type: "providers", + attributes: { + provider, + is_dynamic: false, + uid, + alias: `${provider} provider`, + status: "completed", + resources: 0, + connection: { + connected: true, + last_checked_at: "2026-07-16T00:00:00Z", + }, + scanner_args: { + only_logs: false, + excluded_checks: [], + aws_retries_max_attempts: 3, + }, + inserted_at: "2026-07-16T00:00:00Z", + updated_at: "2026-07-16T00:00:00Z", + created_by: { + object: "users", + id: "user-1", + }, + }, + relationships: { + secret: { + data: null, + }, + provider_groups: { + meta: { + count: 0, + }, + data: [], + }, + }, + }; +} + +function renderS3IntegrationForm( + props?: Partial>, +) { + return render( + , + ); +} + +const integration: IntegrationProps = { + type: "integrations", + id: "integration-1", + attributes: { + inserted_at: "2026-07-16T00:00:00Z", + updated_at: "2026-07-16T00:00:00Z", + enabled: true, + connected: true, + connection_last_checked_at: "2026-07-16T00:00:00Z", + integration_type: "amazon_s3", + configuration: { + bucket_name: "prowler-reports", + output_directory: "output", + }, + }, + relationships: { + providers: { + data: [{ type: "providers", id: "aws-provider" }], + }, + }, + links: { + self: "/integrations/integration-1", + }, +}; + +describe("S3IntegrationForm", () => { + beforeEach(() => { + createIntegrationMock.mockReset(); + toastMock.mockReset(); + updateIntegrationMock.mockReset(); + }); + + it("should require the bucket owner account ID when it cannot derive one", async () => { + // Given + const user = userEvent.setup(); + renderS3IntegrationForm({ + providers: [createProvider("azure", "subscription-id")], + }); + + // When + await user.type(screen.getByLabelText(/Bucket name/i), "prowler-reports"); + await user.click(screen.getByRole("button", { name: "Next" })); + + // Then + expect( + await screen.findByText( + "Bucket owner account ID is required when no AWS account is selected", + ), + ).toBeVisible(); + expect( + screen.queryByLabelText("CloudFormation quick link"), + ).not.toBeInTheDocument(); + }); + + it("should derive the bucket owner account ID from the selected AWS provider", async () => { + // Given + const user = userEvent.setup(); + renderS3IntegrationForm({ + providers: [createProvider("aws", "123456789012")], + }); + + // When + await user.click( + screen.getByRole("button", { name: "Select first provider" }), + ); + await user.type(screen.getByLabelText(/Bucket name/i), "prowler-reports"); + await user.click(screen.getByRole("button", { name: "Next" })); + + // Then + expect( + await screen.findByLabelText("CloudFormation quick link"), + ).toHaveTextContent("bucketAccountId=123456789012"); + }); + + it("should not show a bucket account field that configuration updates cannot persist", () => { + // When + renderS3IntegrationForm({ + integration, + providers: [createProvider("aws", "123456789012")], + editMode: "configuration", + }); + + // Then + expect( + screen.queryByLabelText(/Bucket owner account ID/i), + ).not.toBeInTheDocument(); + }); + + it("should allow changing the bucket owner account for credential updates", () => { + // When + renderS3IntegrationForm({ + integration, + providers: [createProvider("aws", "123456789012")], + editMode: "credentials", + }); + + // Then + expect( + screen.getByLabelText(/Bucket owner account ID/i), + ).toBeInTheDocument(); + }); +}); diff --git a/ui/components/integrations/s3/s3-integration-form.tsx b/ui/components/integrations/s3/s3-integration-form.tsx index 670e3c9aae..fab405dd2b 100644 --- a/ui/components/integrations/s3/s3-integration-form.tsx +++ b/ui/components/integrations/s3/s3-integration-form.tsx @@ -4,7 +4,8 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { ArrowLeftIcon, ArrowRightIcon } from "lucide-react"; import { useSession } from "next-auth/react"; import { useState } from "react"; -import { Control, useForm } from "react-hook-form"; +import type { Control } from "react-hook-form"; +import { useForm } from "react-hook-form"; import { createIntegration, updateIntegration } from "@/actions/integrations"; import { @@ -12,8 +13,7 @@ import { ProviderTypeIcon, } from "@/components/icons/providers-badge/provider-type-icon"; import { AWSRoleCredentialsForm } from "@/components/providers/workflow/forms/select-credentials-type/aws/credentials-type/aws-role-credentials-form"; -import { Separator } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Separator, useToast } from "@/components/shadcn"; import { CustomInput } from "@/components/shadcn/custom"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; import { @@ -25,13 +25,13 @@ import { import { FormButtons } from "@/components/shadcn/form/form-buttons"; import { EnhancedMultiSelect } from "@/components/shadcn/select/enhanced-multi-select"; import { getAWSCredentialsTemplateLinks } from "@/lib"; -import { AWSCredentialsRole } from "@/types"; +import type { AWSCredentialsRole } from "@/types"; +import type { IntegrationProps } from "@/types/integrations"; import { editS3IntegrationFormSchema, - IntegrationProps, s3IntegrationFormSchema, } from "@/types/integrations"; -import { ProviderProps } from "@/types/providers"; +import type { ProviderProps } from "@/types/providers"; interface S3IntegrationFormProps { integration?: IntegrationProps | null; @@ -41,6 +41,25 @@ interface S3IntegrationFormProps { editMode?: "configuration" | "credentials" | null; // null means creating new } +const getSelectedAWSAccountId = ( + selectedProviderIds: string[], + providers: ProviderProps[], +): string => { + for (const providerId of selectedProviderIds) { + const provider = providers.find(({ id }) => id === providerId); + const uid = provider?.attributes.uid; + if ( + provider?.attributes.provider === "aws" && + uid && + /^\d{12}$/.test(uid) + ) { + return uid; + } + } + + return ""; +}; + export const S3IntegrationForm = ({ integration, providers, @@ -75,6 +94,7 @@ export const S3IntegrationForm = ({ defaultValues: { integration_type: "amazon_s3" as const, bucket_name: integration?.attributes.configuration.bucket_name || "", + bucket_account_id: "", output_directory: integration?.attributes.configuration.output_directory || "output", providers: @@ -94,6 +114,14 @@ export const S3IntegrationForm = ({ }); const isLoading = form.formState.isSubmitting; + const selectedProviderIds = form.watch("providers") || []; + const bucketAccountIdOverride = form.watch("bucket_account_id")?.trim() || ""; + const derivedBucketAccountId = getSelectedAWSAccountId( + selectedProviderIds, + providers, + ); + const resolvedBucketAccountId = + bucketAccountIdOverride || derivedBucketAccountId; const handleNext = async (e: React.FormEvent) => { e.preventDefault(); @@ -103,18 +131,36 @@ export const S3IntegrationForm = ({ return; } - // Validate current step fields for creation flow + // Validate current step fields for creation flow. bucket_account_id is + // validated here, while its input is visible, so a malformed value surfaces + // its error instead of silently blocking the step 1 submit. const stepFields = currentStep === 0 - ? (["bucket_name", "output_directory", "providers"] as const) + ? ([ + "bucket_name", + "output_directory", + "providers", + "bucket_account_id", + ] as const) : // Step 1: No required fields since role_arn and external_id are optional []; const isValid = stepFields.length === 0 || (await form.trigger(stepFields)); - if (isValid) { - setCurrentStep(1); + if (!isValid) { + return; } + + if (!resolvedBucketAccountId) { + form.setError("bucket_account_id", { + message: + "Bucket owner account ID is required when no AWS account is selected", + }); + return; + } + + form.clearErrors("bucket_account_id"); + setCurrentStep(1); }; const handleBack = () => { @@ -255,6 +301,30 @@ export const S3IntegrationForm = ({ } }; + const renderBucketAccountIdField = () => ( +
+ +

+ {derivedBucketAccountId + ? `Leave empty to use selected AWS account ${derivedBucketAccountId}, or enter another bucket owner account ID.` + : "Required because the selected provider does not identify the AWS account that owns the bucket."} +

+
+ ); + const renderStepContent = () => { // If editing credentials, show only credentials form if (isEditingCredentials || currentStep === 1) { @@ -265,17 +335,21 @@ export const S3IntegrationForm = ({ externalId, bucketName, "amazon_s3", + resolvedBucketAccountId, ); return ( - } - setValue={form.setValue as any} - externalId={externalId} - templateLinks={templateLinks} - type="integrations" - integrationType="amazon_s3" - /> +
+ {isEditingCredentials && renderBucketAccountIdField()} + } + setValue={form.setValue as any} + externalId={externalId} + templateLinks={templateLinks} + type="integrations" + integrationType="amazon_s3" + /> +
); } @@ -346,6 +420,8 @@ export const S3IntegrationForm = ({ variant="bordered" isRequired /> + + {!isEditingConfig && renderBucketAccountIdField()}
); diff --git a/ui/components/integrations/s3/s3-integrations-manager.tsx b/ui/components/integrations/s3/s3-integrations-manager.tsx index 1e75d1b43c..03ec525abc 100644 --- a/ui/components/integrations/s3/s3-integrations-manager.tsx +++ b/ui/components/integrations/s3/s3-integrations-manager.tsx @@ -15,8 +15,13 @@ import { IntegrationCardHeader, IntegrationSkeleton, } from "@/components/integrations/shared"; -import { Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { + Button, + useToast, + Card, + CardContent, + CardHeader, +} from "@/components/shadcn"; import { Modal } from "@/components/shadcn/modal"; import { DataTablePagination } from "@/components/shadcn/table/data-table-pagination"; import { triggerTestConnectionWithDelay } from "@/lib/integrations/test-connection-helper"; @@ -24,7 +29,6 @@ import { MetaDataProps } from "@/types"; import { IntegrationProps } from "@/types/integrations"; import { ProviderProps } from "@/types/providers"; -import { Card, CardContent, CardHeader } from "../../shadcn"; import { S3IntegrationForm } from "./s3-integration-form"; interface S3IntegrationsManagerProps { diff --git a/ui/components/integrations/saml/saml-config-form.tsx b/ui/components/integrations/saml/saml-config-form.tsx index db713252a8..82d5fccfd5 100644 --- a/ui/components/integrations/saml/saml-config-form.tsx +++ b/ui/components/integrations/saml/saml-config-form.tsx @@ -12,8 +12,13 @@ import { z } from "zod"; import { createSamlConfig, updateSamlConfig } from "@/actions/integrations"; import { AddIcon } from "@/components/icons"; -import { Button, Card, CardContent, CardHeader } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { + Button, + Card, + CardContent, + CardHeader, + useToast, +} from "@/components/shadcn"; import { CodeSnippet } from "@/components/shadcn/code-snippet/code-snippet"; import { CustomServerInput } from "@/components/shadcn/custom"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; diff --git a/ui/components/integrations/security-hub/security-hub-integration-card.tsx b/ui/components/integrations/security-hub/security-hub-integration-card.tsx index 4003f8c286..3861702fe0 100644 --- a/ui/components/integrations/security-hub/security-hub-integration-card.tsx +++ b/ui/components/integrations/security-hub/security-hub-integration-card.tsx @@ -4,11 +4,9 @@ import { SettingsIcon } from "lucide-react"; import Link from "next/link"; import { AWSSecurityHubIcon } from "@/components/icons/services/IconServices"; -import { Button } from "@/components/shadcn"; +import { Button, Card, CardContent, CardHeader } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; -import { Card, CardContent, CardHeader } from "../../shadcn"; - export const SecurityHubIntegrationCard = () => { return ( diff --git a/ui/components/integrations/security-hub/security-hub-integration-form.tsx b/ui/components/integrations/security-hub/security-hub-integration-form.tsx index b7fac10706..24e88b744f 100644 --- a/ui/components/integrations/security-hub/security-hub-integration-form.tsx +++ b/ui/components/integrations/security-hub/security-hub-integration-form.tsx @@ -12,8 +12,7 @@ import { ProviderTypeIcon, } from "@/components/icons/providers-badge/provider-type-icon"; import { AWSRoleCredentialsForm } from "@/components/providers/workflow/forms/select-credentials-type/aws/credentials-type/aws-role-credentials-form"; -import { Checkbox, Separator } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Checkbox, Separator, useToast } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; import { Form, diff --git a/ui/components/integrations/security-hub/security-hub-integrations-manager.tsx b/ui/components/integrations/security-hub/security-hub-integrations-manager.tsx index 34c2d9e303..c0c5f9f02e 100644 --- a/ui/components/integrations/security-hub/security-hub-integrations-manager.tsx +++ b/ui/components/integrations/security-hub/security-hub-integrations-manager.tsx @@ -15,8 +15,14 @@ import { IntegrationCardHeader, IntegrationSkeleton, } from "@/components/integrations/shared"; -import { Badge, Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { + Badge, + Button, + useToast, + Card, + CardContent, + CardHeader, +} from "@/components/shadcn"; import { Modal } from "@/components/shadcn/modal"; import { DataTablePagination } from "@/components/shadcn/table/data-table-pagination"; import { triggerTestConnectionWithDelay } from "@/lib/integrations/test-connection-helper"; @@ -24,7 +30,6 @@ import { MetaDataProps } from "@/types"; import { IntegrationProps } from "@/types/integrations"; import { ProviderProps } from "@/types/providers"; -import { Card, CardContent, CardHeader } from "../../shadcn"; import { SecurityHubIntegrationForm } from "./security-hub-integration-form"; interface SecurityHubIntegrationsManagerProps { diff --git a/ui/components/integrations/shared/link-card.tsx b/ui/components/integrations/shared/link-card.tsx index ceb74d6a1b..212c95ed39 100644 --- a/ui/components/integrations/shared/link-card.tsx +++ b/ui/components/integrations/shared/link-card.tsx @@ -3,11 +3,9 @@ import { ExternalLinkIcon, LucideIcon } from "lucide-react"; import Link from "next/link"; -import { Button } from "@/components/shadcn"; +import { Button, Card, CardContent, CardHeader } from "@/components/shadcn"; import { CustomLink } from "@/components/shadcn/custom/custom-link"; -import { Card, CardContent, CardHeader } from "../../shadcn"; - interface LinkCardProps { icon: LucideIcon; title: string; diff --git a/ui/components/invitations/forms/delete-form.tsx b/ui/components/invitations/forms/delete-form.tsx index 58618b9dc6..e34a5e30bd 100644 --- a/ui/components/invitations/forms/delete-form.tsx +++ b/ui/components/invitations/forms/delete-form.tsx @@ -7,8 +7,7 @@ import * as z from "zod"; import { revokeInvite } from "@/actions/invitations/invitation"; import { DeleteIcon } from "@/components/icons"; -import { Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Button, useToast } from "@/components/shadcn"; import { Form } from "@/components/shadcn/form"; const formSchema = z.object({ diff --git a/ui/components/invitations/forms/edit-form.tsx b/ui/components/invitations/forms/edit-form.tsx index b8bfaf6f70..bbfc57b2fb 100644 --- a/ui/components/invitations/forms/edit-form.tsx +++ b/ui/components/invitations/forms/edit-form.tsx @@ -5,7 +5,7 @@ import { Controller, useForm } from "react-hook-form"; import * as z from "zod"; import { updateInvite } from "@/actions/invitations/invitation"; -import { useToast } from "@/components/shadcn"; +import { useToast, Card, CardContent } from "@/components/shadcn"; import { CustomInput } from "@/components/shadcn/custom"; import { Form, FormButtons } from "@/components/shadcn/form"; import { @@ -17,8 +17,6 @@ import { } from "@/components/shadcn/select/select"; import { editInviteFormSchema } from "@/types"; -import { Card, CardContent } from "../../shadcn"; - export const EditForm = ({ invitationId, invitationEmail, diff --git a/ui/components/invitations/workflow/forms/send-invitation-form.tsx b/ui/components/invitations/workflow/forms/send-invitation-form.tsx index 67e6938d4a..ce98cfdc97 100644 --- a/ui/components/invitations/workflow/forms/send-invitation-form.tsx +++ b/ui/components/invitations/workflow/forms/send-invitation-form.tsx @@ -7,8 +7,7 @@ import { Controller, useForm } from "react-hook-form"; import * as z from "zod"; import { sendInvite } from "@/actions/invitations/invitation"; -import { Button } from "@/components/shadcn"; -import { useToast } from "@/components/shadcn"; +import { Button, useToast } from "@/components/shadcn"; import { CustomInput } from "@/components/shadcn/custom"; import { Form } from "@/components/shadcn/form"; import { diff --git a/ui/components/invitations/workflow/vertical-steps.tsx b/ui/components/invitations/workflow/vertical-steps.tsx index 17abec63d2..79fb77678c 100644 --- a/ui/components/invitations/workflow/vertical-steps.tsx +++ b/ui/components/invitations/workflow/vertical-steps.tsx @@ -2,19 +2,18 @@ import { useControlledState } from "@react-stately/utils"; import { domAnimation, LazyMotion, m } from "framer-motion"; -import type { ComponentProps } from "react"; -import React from "react"; +import { forwardRef, useMemo } from "react"; +import type { ComponentProps, HTMLAttributes, ReactNode } from "react"; import { cn } from "@/lib/utils"; export type VerticalStepProps = { className?: string; - description?: React.ReactNode; - title?: React.ReactNode; + description?: ReactNode; + title?: ReactNode; }; -export interface VerticalStepsProps - extends React.HTMLAttributes { +export interface VerticalStepsProps extends HTMLAttributes { /** * An array of steps. * @@ -89,10 +88,7 @@ function CheckIcon(props: ComponentProps<"svg">) { ); } -export const VerticalSteps = React.forwardRef< - HTMLButtonElement, - VerticalStepsProps ->( +export const VerticalSteps = forwardRef( ( { color = "primary", @@ -113,7 +109,7 @@ export const VerticalSteps = React.forwardRef< onStepChange, ); - const colors = React.useMemo(() => { + const colors = useMemo(() => { let userColor; let fgColor; diff --git a/ui/components/layout/app-sidebar/app-sidebar-content.tsx b/ui/components/layout/app-sidebar/app-sidebar-content.tsx index c32a7d5048..ca2b28949d 100644 --- a/ui/components/layout/app-sidebar/app-sidebar-content.tsx +++ b/ui/components/layout/app-sidebar/app-sidebar-content.tsx @@ -24,10 +24,15 @@ interface AppSidebarContentProps { export function AppSidebarContent({ onSelect }: AppSidebarContentProps) { const pathname = usePathname(); const { permissions } = useAuth(); - const { apiDocsUrl } = useRuntimeConfig(); + const { apiDocsUrl, cloudBillingEnabled } = useRuntimeConfig(); const mode = useAppSidebarMode((state) => state.mode); const isCloudEnvironment = isCloud(); - const sections = getNavigationConfig({ pathname, apiDocsUrl, permissions }); + const sections = getNavigationConfig({ + pathname, + apiDocsUrl, + cloudBillingEnabled, + permissions, + }); const showChat = isCloudEnvironment && mode === APP_SIDEBAR_MODE.CHAT; return ( diff --git a/ui/components/layout/app-sidebar/app-sidebar-mode-sync.test.tsx b/ui/components/layout/app-sidebar/app-sidebar-mode-sync.test.tsx index 59caaae738..7eb39455b4 100644 --- a/ui/components/layout/app-sidebar/app-sidebar-mode-sync.test.tsx +++ b/ui/components/layout/app-sidebar/app-sidebar-mode-sync.test.tsx @@ -1,6 +1,8 @@ import { render } from "@testing-library/react"; import { beforeEach, describe, expect, it } from "vitest"; +import { useSidePanelStore } from "@/store/side-panel"; + import { useAppSidebarMode } from "./app-sidebar-mode-store"; import { AppSidebarModeSync } from "./app-sidebar-mode-sync"; import { APP_SIDEBAR_MODE } from "./types"; @@ -8,6 +10,7 @@ import { APP_SIDEBAR_MODE } from "./types"; describe("AppSidebarModeSync", () => { beforeEach(() => { useAppSidebarMode.setState({ mode: APP_SIDEBAR_MODE.CHAT }); + useSidePanelStore.setState({ isOpen: false }); }); it("restores the requested sidebar mode when a route mounts", () => { @@ -17,4 +20,26 @@ describe("AppSidebarModeSync", () => { // Then expect(useAppSidebarMode.getState().mode).toBe(APP_SIDEBAR_MODE.BROWSE); }); + + it("keeps the side panel open by default", () => { + // Given + useSidePanelStore.setState({ isOpen: true }); + + // When + render(); + + // Then + expect(useSidePanelStore.getState().isOpen).toBe(true); + }); + + it("closes the side panel when the full-page chat mounts", () => { + // Given + useSidePanelStore.setState({ isOpen: true }); + + // When + render(); + + // Then + expect(useSidePanelStore.getState().isOpen).toBe(false); + }); }); diff --git a/ui/components/layout/app-sidebar/app-sidebar-mode-sync.tsx b/ui/components/layout/app-sidebar/app-sidebar-mode-sync.tsx index c97d84a14d..b4d7887dfb 100644 --- a/ui/components/layout/app-sidebar/app-sidebar-mode-sync.tsx +++ b/ui/components/layout/app-sidebar/app-sidebar-mode-sync.tsx @@ -1,19 +1,29 @@ "use client"; import { useMountEffect } from "@/hooks/use-mount-effect"; +import { useSidePanelStore } from "@/store/side-panel"; import { useAppSidebarMode } from "./app-sidebar-mode-store"; import type { AppSidebarMode } from "./types"; interface AppSidebarModeSyncProps { mode: AppSidebarMode; + // The full-page chat dismisses the side panel: the chat lives in one place + // or the other, never both. + closeSidePanel?: boolean; } -export function AppSidebarModeSync({ mode }: AppSidebarModeSyncProps) { +export function AppSidebarModeSync({ + mode, + closeSidePanel = false, +}: AppSidebarModeSyncProps) { const setMode = useAppSidebarMode((state) => state.setMode); useMountEffect(() => { setMode(mode); + if (closeSidePanel) { + useSidePanelStore.getState().closePanel(); + } }); return null; diff --git a/ui/components/layout/app-sidebar/mobile-app-sidebar.test.tsx b/ui/components/layout/app-sidebar/mobile-app-sidebar.test.tsx index 75aae87f5a..343a6aef72 100644 --- a/ui/components/layout/app-sidebar/mobile-app-sidebar.test.tsx +++ b/ui/components/layout/app-sidebar/mobile-app-sidebar.test.tsx @@ -45,6 +45,17 @@ describe("MobileAppSidebar", () => { expect(openButton).toHaveFocus(); }); + it("hides the trigger based on the viewport, not the narrowed content", () => { + // Given / When + render(); + + // Then: lg: is a container query inside
; the side panel squeezing + // the page must not surface the mobile menu on desktop. + expect(screen.getByRole("button", { name: "Open menu" })).toHaveClass( + "min-[64rem]:hidden", + ); + }); + it("closes after selecting an item from the shared sidebar content", async () => { // Given const user = userEvent.setup(); diff --git a/ui/components/layout/app-sidebar/mobile-app-sidebar.tsx b/ui/components/layout/app-sidebar/mobile-app-sidebar.tsx index abb6fbbfe7..0a3f9afdeb 100644 --- a/ui/components/layout/app-sidebar/mobile-app-sidebar.tsx +++ b/ui/components/layout/app-sidebar/mobile-app-sidebar.tsx @@ -35,7 +35,9 @@ export function MobileAppSidebar() { variant="bare" size="icon-sm" aria-label="Open menu" - className={cn("lg:hidden", open && "invisible")} + // min-[64rem] (not lg:): inside
, lg is a container query and + // the side panel squeezing the page would surface the mobile menu. + className={cn("min-[64rem]:hidden", open && "invisible")} >