From bf9c53a4c3ddc560f4f92950aa7c130139997c6a Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Thu, 9 Jul 2026 15:41:51 +0200 Subject: [PATCH 01/20] fix(api): keep auth tests order-safe after token revocation (#11919) --- .../tests/integration/test_authentication.py | 35 +++++++++++++------ api/src/backend/conftest.py | 7 ++-- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/api/src/backend/api/tests/integration/test_authentication.py b/api/src/backend/api/tests/integration/test_authentication.py index 23cf07afcb..67f498b238 100644 --- a/api/src/backend/api/tests/integration/test_authentication.py +++ b/api/src/backend/api/tests/integration/test_authentication.py @@ -14,6 +14,19 @@ from rest_framework_simplejwt.token_blacklist.models import ( OutstandingToken, ) +PASSWORD_CHANGE_PASSWORD = "InitialSecret123@" + + +@pytest.fixture +def password_change_user(tenants_fixture): + user = User.objects.create_user( + name="password_change_user", + email=f"password-change-{uuid4()}@prowler.com", + password=PASSWORD_CHANGE_PASSWORD, + ) + Membership.objects.create(user=user, tenant=tenants_fixture[0]) + return user + @pytest.mark.django_db def test_basic_authentication(): @@ -109,16 +122,16 @@ def test_refresh_token(create_test_user, tenants_fixture): @pytest.mark.django_db -def test_password_change_invalidates_existing_tokens(create_test_user, tenants_fixture): +def test_password_change_invalidates_existing_tokens(password_change_user): client = APIClient() new_password = "ChangedSecret123@" access_token, refresh_token = get_api_tokens( - client, create_test_user.email, TEST_PASSWORD + client, password_change_user.email, PASSWORD_CHANGE_PASSWORD ) auth_headers = get_authorization_header(access_token) outstanding_token_ids = list( - OutstandingToken.objects.filter(user=create_test_user).values_list( + OutstandingToken.objects.filter(user=password_change_user).values_list( "id", flat=True ) ) @@ -130,12 +143,12 @@ def test_password_change_invalidates_existing_tokens(create_test_user, tenants_f password_change_payload = { "data": { "type": "users", - "id": str(create_test_user.id), + "id": str(password_change_user.id), "attributes": {"password": new_password}, } } password_change_response = client.patch( - reverse("user-detail", kwargs={"pk": create_test_user.id}), + reverse("user-detail", kwargs={"pk": password_change_user.id}), data=json.dumps(password_change_payload), headers=auth_headers, content_type="application/vnd.api+json", @@ -160,7 +173,9 @@ def test_password_change_invalidates_existing_tokens(create_test_user, tenants_f ) assert old_refresh_response.status_code == 400 - new_access_token, _ = get_api_tokens(client, create_test_user.email, new_password) + new_access_token, _ = get_api_tokens( + client, password_change_user.email, new_password + ) new_access_response = client.get( reverse("user-me"), headers=get_authorization_header(new_access_token) ) @@ -169,13 +184,13 @@ def test_password_change_invalidates_existing_tokens(create_test_user, tenants_f @pytest.mark.django_db def test_password_change_invalidates_rotated_refresh_token( - create_test_user, tenants_fixture + password_change_user, ): client = APIClient() new_password = "ChangedSecret123@" access_token, refresh_token = get_api_tokens( - client, create_test_user.email, TEST_PASSWORD + client, password_change_user.email, PASSWORD_CHANGE_PASSWORD ) rotated_refresh_response = client.post( reverse("token-refresh"), @@ -195,12 +210,12 @@ def test_password_change_invalidates_rotated_refresh_token( password_change_payload = { "data": { "type": "users", - "id": str(create_test_user.id), + "id": str(password_change_user.id), "attributes": {"password": new_password}, } } password_change_response = client.patch( - reverse("user-detail", kwargs={"pk": create_test_user.id}), + reverse("user-detail", kwargs={"pk": password_change_user.id}), data=json.dumps(password_change_payload), headers=get_authorization_header(access_token), content_type="application/vnd.api+json", diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index d5bf179b16..f4e398d76a 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -231,14 +231,15 @@ def create_test_user(_session_test_user, django_db_blocker): """Re-create the session-scoped test user when a TransactionTestCase has truncated the users table.""" with django_db_blocker.unblock(): - if not User.objects.filter(pk=_session_test_user.pk).exists(): - User.objects.create_user( + user = User.objects.filter(pk=_session_test_user.pk).first() + if user is None: + user = User.objects.create_user( id=_session_test_user.pk, name="testing", email=TEST_USER, password=TEST_PASSWORD, ) - return _session_test_user + return user @pytest.fixture(scope="function") From 0b36d08b92ba7fc18b48eb92daea68cbac552c61 Mon Sep 17 00:00:00 2001 From: Yinka Metrics <115735904+YinkaMetrics@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:21:05 +0100 Subject: [PATCH 02/20] feat(sdk): add Data Pipeline secret scanning check (#11821) Co-authored-by: Daniel Barranquero --- permissions/prowler-additions-policy.json | 3 + .../cloudformation/prowler-scan-role.yml | 3 + .../datapipeline-secrets-check.added.md | 1 + .../aws/services/datapipeline/__init__.py | 0 .../datapipeline/datapipeline_client.py | 6 + .../__init__.py | 0 ...ine_no_secrets_in_definition.metadata.json | 43 +++ ...eline_pipeline_no_secrets_in_definition.py | 112 ++++++++ .../datapipeline/datapipeline_service.py | 96 +++++++ ..._pipeline_no_secrets_in_definition_test.py | 254 ++++++++++++++++++ .../datapipeline/datapipeline_service_test.py | 121 +++++++++ 11 files changed, 639 insertions(+) create mode 100644 prowler/changelog.d/datapipeline-secrets-check.added.md create mode 100644 prowler/providers/aws/services/datapipeline/__init__.py create mode 100644 prowler/providers/aws/services/datapipeline/datapipeline_client.py create mode 100644 prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/__init__.py create mode 100644 prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.metadata.json create mode 100644 prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.py create mode 100644 prowler/providers/aws/services/datapipeline/datapipeline_service.py create mode 100644 tests/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition_test.py create mode 100644 tests/providers/aws/services/datapipeline/datapipeline_service_test.py diff --git a/permissions/prowler-additions-policy.json b/permissions/prowler-additions-policy.json index eb140e7c09..ba56513732 100644 --- a/permissions/prowler-additions-policy.json +++ b/permissions/prowler-additions-policy.json @@ -15,6 +15,9 @@ "codebuild:BatchGet*", "codebuild:ListReportGroups", "cognito-idp:GetUserPoolMfaConfig", + "datapipeline:DescribePipelines", + "datapipeline:GetPipelineDefinition", + "datapipeline:ListPipelines", "dlm:Get*", "drs:Describe*", "ds:Get*", diff --git a/permissions/templates/cloudformation/prowler-scan-role.yml b/permissions/templates/cloudformation/prowler-scan-role.yml index a7d91b0071..744c8d3e23 100644 --- a/permissions/templates/cloudformation/prowler-scan-role.yml +++ b/permissions/templates/cloudformation/prowler-scan-role.yml @@ -109,6 +109,9 @@ Resources: - "codebuild:BatchGet*" - "codebuild:ListReportGroups" - "cognito-idp:GetUserPoolMfaConfig" + - "datapipeline:DescribePipelines" + - "datapipeline:GetPipelineDefinition" + - "datapipeline:ListPipelines" - "dlm:Get*" - "drs:Describe*" - "ds:Get*" diff --git a/prowler/changelog.d/datapipeline-secrets-check.added.md b/prowler/changelog.d/datapipeline-secrets-check.added.md new file mode 100644 index 0000000000..e55e3338bd --- /dev/null +++ b/prowler/changelog.d/datapipeline-secrets-check.added.md @@ -0,0 +1 @@ +`datapipeline_pipeline_no_secrets_in_definition` check for AWS provider, scanning Data Pipeline object fields, parameter objects, and parameter values for hardcoded secrets with Kingfisher diff --git a/prowler/providers/aws/services/datapipeline/__init__.py b/prowler/providers/aws/services/datapipeline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/aws/services/datapipeline/datapipeline_client.py b/prowler/providers/aws/services/datapipeline/datapipeline_client.py new file mode 100644 index 0000000000..4732a559a5 --- /dev/null +++ b/prowler/providers/aws/services/datapipeline/datapipeline_client.py @@ -0,0 +1,6 @@ +from prowler.providers.aws.services.datapipeline.datapipeline_service import ( + DataPipeline, +) +from prowler.providers.common.provider import Provider + +datapipeline_client = DataPipeline(Provider.get_global_provider()) diff --git a/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/__init__.py b/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.metadata.json b/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.metadata.json new file mode 100644 index 0000000000..cadafedfa9 --- /dev/null +++ b/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.metadata.json @@ -0,0 +1,43 @@ +{ + "Provider": "aws", + "CheckID": "datapipeline_pipeline_no_secrets_in_definition", + "CheckTitle": "Data Pipeline definition has no sensitive credentials", + "CheckType": [ + "Software and Configuration Checks/AWS Security Best Practices", + "TTPs/Credential Access", + "Effects/Data Exposure", + "Sensitive Data Identifications/Security" + ], + "ServiceName": "datapipeline", + "SubServiceName": "", + "ResourceIdTemplate": "arn:partition:datapipeline:region:account-id:pipeline/pipeline-id", + "Severity": "high", + "ResourceType": "AwsDataPipelinePipeline", + "ResourceGroup": "analytics", + "Description": "AWS Data Pipeline definitions are inspected for hardcoded secrets, such as keys, tokens, passwords, or database credentials embedded directly in pipeline objects, parameters, or parameter values.", + "Risk": "Plaintext secrets in Data Pipeline definitions can be viewed by users with pipeline read permissions and may leak through scripts, SQL commands, logs, or exported definitions. Exposed credentials can enable unauthorized access to databases, storage, and downstream systems.", + "RelatedUrl": "", + "AdditionalURLs": [ + "https://docs.aws.amazon.com/datapipeline/latest/APIReference/API_GetPipelineDefinition.html", + "https://docs.aws.amazon.com/datapipeline/latest/DeveloperGuide/dp-object-pipeline.html", + "https://docs.prowler.com/developer-guide/secret-scanning-checks" + ], + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "1. Review the Data Pipeline definition.\n2. Remove hardcoded credentials from pipeline objects, parameter objects, and parameter values.\n3. Store secrets in AWS Secrets Manager or AWS Systems Manager Parameter Store.\n4. Grant the pipeline role least-privilege access to retrieve secrets securely at runtime.\n5. Rotate any exposed credentials.", + "Terraform": "" + }, + "Recommendation": { + "Text": "Avoid embedding secrets in AWS Data Pipeline definitions. Store sensitive values in AWS Secrets Manager or AWS Systems Manager Parameter Store and reference them securely at runtime with least-privilege IAM permissions.", + "Url": "https://hub.prowler.com/check/datapipeline_pipeline_no_secrets_in_definition" + } + }, + "Categories": [ + "secrets" + ], + "DependsOn": [], + "RelatedTo": [], + "Notes": "AWS Data Pipeline has been closed to new customers since July 25, 2024. Accounts without pre-existing pipelines will not return findings for this check, as the service cannot be used by new customers." +} diff --git a/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.py b/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.py new file mode 100644 index 0000000000..1e6a60facd --- /dev/null +++ b/prowler/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition.py @@ -0,0 +1,112 @@ +import json + +from prowler.lib.check.models import Check, Check_Report_AWS +from prowler.lib.utils.utils import ( + SecretsScanError, + annotate_verified_secrets, + detect_secrets_scan_batch, +) +from prowler.providers.aws.services.datapipeline.datapipeline_client import ( + datapipeline_client, +) + + +class datapipeline_pipeline_no_secrets_in_definition(Check): + """Check that AWS Data Pipeline definitions contain no hardcoded secrets.""" + + def execute(self) -> list[Check_Report_AWS]: + """Execute the Data Pipeline definition secret scan.""" + findings = [] + secrets_ignore_patterns = datapipeline_client.audit_config.get( + "secrets_ignore_patterns", [] + ) + validate = datapipeline_client.audit_config.get("secrets_validate", False) + pipelines = list(datapipeline_client.pipelines.values()) + line_context_by_pipeline = {} + payloads = [] + for pipeline_index, pipeline in enumerate(pipelines): + payload, line_context = _build_definition_payload(pipeline.definition) + line_context_by_pipeline[pipeline_index] = line_context + if payload: + payloads.append((pipeline_index, payload)) + + scan_error = None + try: + batch_results = detect_secrets_scan_batch( + payloads, excluded_secrets=secrets_ignore_patterns, validate=validate + ) + except SecretsScanError as error: + batch_results = {} + scan_error = error + + for pipeline_index, pipeline in enumerate(pipelines): + report = Check_Report_AWS(metadata=self.metadata(), resource=pipeline) + report.resource_tags = pipeline.tags + report.status = "PASS" + report.status_extended = ( + f"No secrets found in Data Pipeline {pipeline.name} definition." + ) + + line_context = line_context_by_pipeline.get(pipeline_index, {}) + if line_context: + if scan_error: + report.status = "MANUAL" + report.status_extended = ( + f"Could not scan Data Pipeline {pipeline.name} definition " + f"for secrets: {scan_error}; manual review is required." + ) + findings.append(report) + continue + + detect_secrets_output = batch_results.get(pipeline_index) + if detect_secrets_output: + secrets_string = ", ".join( + [ + f"{secret['type']} in {line_context.get(secret['line_number'], 'definition')}" + for secret in detect_secrets_output + ] + ) + report.status = "FAIL" + report.status_extended = ( + f"Potential {'secrets' if len(detect_secrets_output) > 1 else 'secret'} " + f"found in Data Pipeline {pipeline.name} definition -> {secrets_string}." + ) + annotate_verified_secrets(report, detect_secrets_output) + + findings.append(report) + return findings + + +def _build_definition_payload(definition: dict) -> tuple[str, dict[int, str]]: + """Build a line-oriented scan payload and map each line to a definition field.""" + lines = [] + line_context = {} + + def add_line(context: str, value) -> None: + if value is None: + return + lines.append(json.dumps({context: value})) + line_context[len(lines)] = context + + for pipeline_object in definition.get("pipelineObjects", []): + object_name = pipeline_object.get("name") or pipeline_object.get("id") + for field in pipeline_object.get("fields", []): + field_name = field.get("key") + field_value = field.get("stringValue") or field.get("refValue") + add_line(f"object {object_name} field {field_name}", field_value) + + for parameter_object in definition.get("parameterObjects", []): + parameter_name = parameter_object.get("id") + for attribute in parameter_object.get("attributes", []): + attribute_name = attribute.get("key") + attribute_value = attribute.get("stringValue") + add_line( + f"parameter object {parameter_name} attribute {attribute_name}", + attribute_value, + ) + + for parameter_value in definition.get("parameterValues", []): + parameter_id = parameter_value.get("id") + add_line(f"parameter value {parameter_id}", parameter_value.get("stringValue")) + + return "\n".join(lines), line_context diff --git a/prowler/providers/aws/services/datapipeline/datapipeline_service.py b/prowler/providers/aws/services/datapipeline/datapipeline_service.py new file mode 100644 index 0000000000..c893dd0557 --- /dev/null +++ b/prowler/providers/aws/services/datapipeline/datapipeline_service.py @@ -0,0 +1,96 @@ +from botocore.exceptions import ClientError +from pydantic.v1 import BaseModel, Field + +from prowler.lib.logger import logger +from prowler.lib.scan_filters.scan_filters import is_resource_filtered +from prowler.providers.aws.lib.service.service import AWSService + + +class Pipeline(BaseModel): + """Represents an AWS Data Pipeline pipeline.""" + + id: str + name: str + arn: str + region: str + definition: dict = Field(default_factory=dict) + tags: list[dict] = Field(default_factory=list) + + +class DataPipeline(AWSService): + """AWS Data Pipeline service class to list pipelines and definitions.""" + + def __init__(self, provider): + """Initialize the AWS Data Pipeline service.""" + super().__init__(__class__.__name__, provider) + self.pipelines = {} + self.__threading_call__(self._list_pipelines) + if self.pipelines: + self.__threading_call__( + self._get_pipeline_definition, self.pipelines.values() + ) + + def _list_pipelines(self, regional_client) -> None: + """List AWS Data Pipeline pipelines in a region.""" + logger.info("DataPipeline - Listing pipelines...") + try: + list_pipelines_paginator = regional_client.get_paginator("list_pipelines") + for page in list_pipelines_paginator.paginate(): + for pipeline in page.get("pipelineIdList", []): + pipeline_id = pipeline.get("id") + pipeline_name = pipeline.get("name", pipeline_id) + pipeline_arn = ( + f"arn:{self.audited_partition}:datapipeline:" + f"{regional_client.region}:{self.audited_account}:pipeline/{pipeline_id}" + ) + if not self.audit_resources or is_resource_filtered( + pipeline_arn, self.audit_resources + ): + self.pipelines[pipeline_arn] = Pipeline( + id=pipeline_id, + name=pipeline_name, + arn=pipeline_arn, + region=regional_client.region, + ) + except ClientError as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + def _get_pipeline_definition(self, pipeline: Pipeline) -> None: + """Get the full definition for an AWS Data Pipeline pipeline.""" + logger.info(f"DataPipeline - Getting definition for pipeline {pipeline.id}...") + try: + regional_client = self.regional_clients[pipeline.region] + try: + pipeline_descriptions = regional_client.describe_pipelines( + pipelineIds=[pipeline.id] + ).get("pipelineDescriptionList", []) + if pipeline_descriptions: + pipeline.tags = pipeline_descriptions[0].get("tags", []) + except ClientError as error: + logger.error( + f"{pipeline.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + except Exception as error: + logger.error( + f"{pipeline.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + definition = regional_client.get_pipeline_definition(pipelineId=pipeline.id) + pipeline.definition = { + "pipelineObjects": definition.get("pipelineObjects", []), + "parameterObjects": definition.get("parameterObjects", []), + "parameterValues": definition.get("parameterValues", []), + } + except ClientError as error: + logger.error( + f"{pipeline.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + except Exception as error: + logger.error( + f"{pipeline.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) diff --git a/tests/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition_test.py b/tests/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition_test.py new file mode 100644 index 0000000000..fb550efbf3 --- /dev/null +++ b/tests/providers/aws/services/datapipeline/datapipeline_pipeline_no_secrets_in_definition/datapipeline_pipeline_no_secrets_in_definition_test.py @@ -0,0 +1,254 @@ +from unittest import mock + +from prowler.lib.utils.utils import SecretsScanError +from prowler.providers.aws.services.datapipeline.datapipeline_service import Pipeline +from tests.providers.aws.utils import ( + AWS_ACCOUNT_NUMBER, + AWS_REGION_US_EAST_1, + set_mocked_aws_provider, +) + + +class Test_datapipeline_pipeline_no_secrets_in_definition: + def test_no_pipelines(self): + datapipeline_client = mock.MagicMock() + datapipeline_client.pipelines = {} + datapipeline_client.audit_config = {"secrets_ignore_patterns": []} + + result = _execute_check(datapipeline_client) + + assert len(result) == 0 + + def test_pipeline_with_no_secrets_in_definition(self): + pipeline = _build_pipeline( + definition={ + "pipelineObjects": [ + { + "id": "Default", + "name": "Default", + "fields": [ + {"key": "type", "stringValue": "Default"}, + {"key": "scheduleType", "stringValue": "cron"}, + ], + } + ] + } + ) + datapipeline_client = mock.MagicMock() + datapipeline_client.pipelines = {pipeline.arn: pipeline} + datapipeline_client.audit_config = {"secrets_ignore_patterns": []} + + result = _execute_check(datapipeline_client) + + assert len(result) == 1 + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == "No secrets found in Data Pipeline test-pipeline definition." + ) + assert result[0].region == AWS_REGION_US_EAST_1 + assert result[0].resource_id == "df-1234567890" + assert result[0].resource_arn == pipeline.arn + + def test_pipeline_with_secrets_in_object_field(self): + pipeline = _build_pipeline( + definition={ + "pipelineObjects": [ + { + "id": "SqlActivity", + "name": "SqlActivity", + "fields": [ + {"key": "type", "stringValue": "SqlActivity"}, + { + "key": "script", + "stringValue": "select * from users where token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'", + }, + ], + } + ] + } + ) + datapipeline_client = mock.MagicMock() + datapipeline_client.pipelines = {pipeline.arn: pipeline} + datapipeline_client.audit_config = {"secrets_ignore_patterns": []} + + result = _execute_check(datapipeline_client) + + assert len(result) == 1 + assert result[0].status == "FAIL" + assert "test-pipeline" in result[0].status_extended + assert "object SqlActivity field script" in result[0].status_extended + assert "eyJhbGciOiJIUzI1Ni" not in result[0].status_extended + assert result[0].region == AWS_REGION_US_EAST_1 + assert result[0].resource_id == "df-1234567890" + assert result[0].resource_arn == pipeline.arn + + def test_pipeline_with_secrets_in_parameter_value(self): + pipeline = _build_pipeline( + definition={ + "parameterValues": [ + { + "id": "databasePassword", + "stringValue": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXNzd29yZCI6InN1cGVyLXNlY3JldCJ9.zZ7_9wzLQfPy4TAAkpS6I8nRcEvuTnbwN7gGr1pH5fQ", + } + ] + } + ) + datapipeline_client = mock.MagicMock() + datapipeline_client.pipelines = {pipeline.arn: pipeline} + datapipeline_client.audit_config = {"secrets_ignore_patterns": []} + + result = _execute_check(datapipeline_client) + + assert len(result) == 1 + assert result[0].status == "FAIL" + assert "parameter value databasePassword" in result[0].status_extended + assert "super-secret" not in result[0].status_extended + + def test_pipeline_with_verified_secret_escalates_severity(self): + from prowler.lib.check.models import Severity + + pipeline = _build_pipeline( + definition={ + "parameterValues": [ + { + "id": "databasePassword", + "stringValue": "verified-secret-value", + } + ] + } + ) + datapipeline_client = mock.MagicMock() + datapipeline_client.pipelines = {pipeline.arn: pipeline} + datapipeline_client.audit_config = { + "secrets_ignore_patterns": [], + "secrets_validate": True, + } + + result, scan_batch = _execute_check_with_mocked_scan( + datapipeline_client, + return_value={ + 0: [ + { + "type": "Generic Password", + "line_number": 1, + "filename": "data", + "hashed_secret": "x", + "is_verified": True, + } + ] + }, + ) + + assert scan_batch.call_args.kwargs.get("validate") is True + assert len(result) == 1 + assert result[0].status == "FAIL" + assert result[0].check_metadata.Severity == Severity.critical + assert "confirmed to be live" in result[0].status_extended + + def test_scan_error_marks_all_scannable_pipelines_manual(self): + first_pipeline = _build_pipeline( + pipeline_id="df-first", + pipeline_name="first-pipeline", + definition={ + "pipelineObjects": [ + { + "id": "Default", + "name": "Default", + "fields": [{"key": "type", "stringValue": "Default"}], + } + ] + }, + ) + second_pipeline = _build_pipeline( + pipeline_id="df-second", + pipeline_name="second-pipeline", + definition={ + "parameterValues": [ + {"id": "databasePassword", "stringValue": "secret-value"} + ] + }, + ) + datapipeline_client = mock.MagicMock() + datapipeline_client.pipelines = { + first_pipeline.arn: first_pipeline, + second_pipeline.arn: second_pipeline, + } + datapipeline_client.audit_config = {"secrets_ignore_patterns": []} + + result, scan_batch = _execute_check_with_mocked_scan( + datapipeline_client, + side_effect=SecretsScanError("scanner unavailable"), + ) + + scan_payloads = scan_batch.call_args.args[0] + assert len(scan_payloads) == 2 + assert len(result) == 2 + assert {report.status for report in result} == {"MANUAL"} + assert all( + "manual review is required" in report.status_extended for report in result + ) + + +def _build_pipeline( + definition: dict, + pipeline_id: str = "df-1234567890", + pipeline_name: str = "test-pipeline", +) -> Pipeline: + pipeline_arn = ( + f"arn:aws:datapipeline:{AWS_REGION_US_EAST_1}:" + f"{AWS_ACCOUNT_NUMBER}:pipeline/{pipeline_id}" + ) + return Pipeline( + id=pipeline_id, + name=pipeline_name, + arn=pipeline_arn, + region=AWS_REGION_US_EAST_1, + definition=definition, + ) + + +def _execute_check(datapipeline_client): + aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1]) + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch( + "prowler.providers.aws.services.datapipeline.datapipeline_pipeline_no_secrets_in_definition.datapipeline_pipeline_no_secrets_in_definition.datapipeline_client", + datapipeline_client, + ), + ): + from prowler.providers.aws.services.datapipeline.datapipeline_pipeline_no_secrets_in_definition.datapipeline_pipeline_no_secrets_in_definition import ( + datapipeline_pipeline_no_secrets_in_definition, + ) + + check = datapipeline_pipeline_no_secrets_in_definition() + return check.execute() + + +def _execute_check_with_mocked_scan( + datapipeline_client, return_value=None, side_effect=None +): + aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1]) + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=aws_provider, + ), + mock.patch( + "prowler.providers.aws.services.datapipeline.datapipeline_pipeline_no_secrets_in_definition.datapipeline_pipeline_no_secrets_in_definition.datapipeline_client", + datapipeline_client, + ), + ): + import prowler.providers.aws.services.datapipeline.datapipeline_pipeline_no_secrets_in_definition.datapipeline_pipeline_no_secrets_in_definition as check_module + + with mock.patch.object( + check_module, + "detect_secrets_scan_batch", + return_value=return_value, + side_effect=side_effect, + ) as scan_batch: + check = check_module.datapipeline_pipeline_no_secrets_in_definition() + return check.execute(), scan_batch diff --git a/tests/providers/aws/services/datapipeline/datapipeline_service_test.py b/tests/providers/aws/services/datapipeline/datapipeline_service_test.py new file mode 100644 index 0000000000..12637d11d9 --- /dev/null +++ b/tests/providers/aws/services/datapipeline/datapipeline_service_test.py @@ -0,0 +1,121 @@ +from unittest.mock import patch + +import botocore +from moto import mock_aws + +from prowler.providers.aws.services.datapipeline.datapipeline_service import ( + DataPipeline, + Pipeline, +) +from tests.providers.aws.utils import ( + AWS_ACCOUNT_NUMBER, + AWS_REGION_US_EAST_1, + set_mocked_aws_provider, +) + +pipeline_id = "df-1234567890" +pipeline_name = "test-pipeline" +pipeline_arn = ( + f"arn:aws:datapipeline:{AWS_REGION_US_EAST_1}:" + f"{AWS_ACCOUNT_NUMBER}:pipeline/{pipeline_id}" +) +pipeline_definition = { + "pipelineObjects": [ + { + "id": "Default", + "name": "Default", + "fields": [{"key": "type", "stringValue": "Default"}], + } + ], + "parameterObjects": [], + "parameterValues": [], +} +pipeline_tags = [{"key": "Environment", "value": "test"}] + +make_api_call = botocore.client.BaseClient._make_api_call + + +def mock_make_api_call(self, operation_name, kwarg): + if operation_name == "ListPipelines": + return {"pipelineIdList": [{"id": pipeline_id, "name": pipeline_name}]} + if operation_name == "DescribePipelines": + return { + "pipelineDescriptionList": [ + { + "pipelineId": pipeline_id, + "name": pipeline_name, + "tags": pipeline_tags, + } + ] + } + if operation_name == "GetPipelineDefinition": + return pipeline_definition + return make_api_call(self, operation_name, kwarg) + + +def mock_make_api_call_describe_fails(self, operation_name, kwarg): + if operation_name == "ListPipelines": + return {"pipelineIdList": [{"id": pipeline_id, "name": pipeline_name}]} + if operation_name == "DescribePipelines": + raise botocore.exceptions.ClientError( + { + "Error": { + "Code": "AccessDeniedException", + "Message": "Access denied", + } + }, + operation_name, + ) + if operation_name == "GetPipelineDefinition": + return pipeline_definition + return make_api_call(self, operation_name, kwarg) + + +def mock_generate_regional_clients(provider, service): + regional_client = provider._session.current_session.client( + service, region_name=AWS_REGION_US_EAST_1 + ) + regional_client.region = AWS_REGION_US_EAST_1 + return {AWS_REGION_US_EAST_1: regional_client} + + +@patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call) +@patch( + "prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients", + new=mock_generate_regional_clients, +) +class TestDataPipelineService: + @mock_aws + def test_datapipeline_service(self): + datapipeline = DataPipeline(set_mocked_aws_provider([AWS_REGION_US_EAST_1])) + + assert datapipeline.session.__class__.__name__ == "Session" + assert datapipeline.service == "datapipeline" + assert len(datapipeline.pipelines) == 1 + assert isinstance(datapipeline.pipelines[pipeline_arn], Pipeline) + + pipeline = datapipeline.pipelines[pipeline_arn] + assert pipeline.id == pipeline_id + assert pipeline.name == pipeline_name + assert pipeline.arn == pipeline_arn + assert pipeline.region == AWS_REGION_US_EAST_1 + assert pipeline.definition == pipeline_definition + assert pipeline.tags == pipeline_tags + + +@patch( + "botocore.client.BaseClient._make_api_call", new=mock_make_api_call_describe_fails +) +@patch( + "prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients", + new=mock_generate_regional_clients, +) +class TestDataPipelineServiceDescribeFailure: + @mock_aws + def test_datapipeline_service_gets_definition_when_describe_fails(self): + datapipeline = DataPipeline(set_mocked_aws_provider([AWS_REGION_US_EAST_1])) + + assert len(datapipeline.pipelines) == 1 + pipeline = datapipeline.pipelines[pipeline_arn] + assert pipeline.definition == pipeline_definition + assert pipeline.tags == [] From 01c004a7af6ac2bcb6a0ee9d86d65d3a9163c4a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Mart=C3=ADn?= Date: Thu, 9 Jul 2026 16:41:51 +0200 Subject: [PATCH 03/20] fix(ui): handle level 1 requirements for M365 CIS (#11921) --- ui/changelog.d/cis-profile-level-filter-m365.fixed.md | 1 + ui/lib/compliance/cis.tsx | 7 +++++-- ui/types/compliance.ts | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 ui/changelog.d/cis-profile-level-filter-m365.fixed.md diff --git a/ui/changelog.d/cis-profile-level-filter-m365.fixed.md b/ui/changelog.d/cis-profile-level-filter-m365.fixed.md new file mode 100644 index 0000000000..d6a07fd36f --- /dev/null +++ b/ui/changelog.d/cis-profile-level-filter-m365.fixed.md @@ -0,0 +1 @@ +CIS Level 1 and Level 2 compliance filters now match profiles prefixed with a license tier (e.g. "E3 Level 1"), so M365 CIS requirements are no longer hidden diff --git a/ui/lib/compliance/cis.tsx b/ui/lib/compliance/cis.tsx index cf7b9f8841..d52c89dc81 100644 --- a/ui/lib/compliance/cis.tsx +++ b/ui/lib/compliance/cis.tsx @@ -38,8 +38,11 @@ export const mapComplianceData = ( const attrs = metadataArray?.[0]; if (!attrs) continue; - // Apply profile filter - if (filter === "Level 1" && attrs.Profile !== "Level 1") { + // Apply profile filter. + // Most CIS benchmarks use "Level 1"/"Level 2" as the Profile, but some + // (e.g. M365) prefix it with the license tier: "E3 Level 1", "E5 Level 2". + // Match by suffix so the Level 1 filter works across all CIS variants. + if (filter === "Level 1" && !attrs.Profile?.endsWith("Level 1")) { continue; // Skip Level 2 requirements when Level 1 is selected } diff --git a/ui/types/compliance.ts b/ui/types/compliance.ts index 0ce94b2d17..94a2486369 100644 --- a/ui/types/compliance.ts +++ b/ui/types/compliance.ts @@ -127,7 +127,7 @@ export interface ISO27001AttributesMetadata { export interface CISAttributesMetadata { Section: string; SubSection: string | null; - Profile: string; // "Level 1" or "Level 2" + Profile: string; // "Level 1"/"Level 2" (M365 prefixes the tier: "E3 Level 1", "E5 Level 2") AssessmentStatus: string; // "Manual" or "Automated" Description: string; RationaleStatement: string; From be78fe8374f6e521af38fa14ddb557c9044df1b3 Mon Sep 17 00:00:00 2001 From: Hugo Pereira Brito <101209179+HugoPBrito@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:09:12 +0100 Subject: [PATCH 04/20] fix(jira): surface dispatch failures (#11918) --- api/changelog.d/jira-dispatch-errors.fixed.md | 1 + api/src/backend/tasks/jobs/integrations.py | 73 +++--- .../backend/tasks/tests/test_integrations.py | 211 +++++++++++++++++- .../changelog.d/jira-dispatch-errors.fixed.md | 1 + prowler/lib/outputs/jira/jira.py | 84 +++++-- tests/lib/outputs/jira/jira_test.py | 163 ++++++++++++-- ui/actions/integrations/jira-dispatch.test.ts | 152 +++++++++++++ ui/actions/integrations/jira-dispatch.ts | 10 +- ui/changelog.d/jira-dispatch-errors.fixed.md | 1 + ui/types/integrations.ts | 2 + 10 files changed, 632 insertions(+), 66 deletions(-) create mode 100644 api/changelog.d/jira-dispatch-errors.fixed.md create mode 100644 prowler/changelog.d/jira-dispatch-errors.fixed.md create mode 100644 ui/actions/integrations/jira-dispatch.test.ts create mode 100644 ui/changelog.d/jira-dispatch-errors.fixed.md diff --git a/api/changelog.d/jira-dispatch-errors.fixed.md b/api/changelog.d/jira-dispatch-errors.fixed.md new file mode 100644 index 0000000000..8aa080e8e9 --- /dev/null +++ b/api/changelog.d/jira-dispatch-errors.fixed.md @@ -0,0 +1 @@ +Jira dispatch task results now surface user-facing Jira failure messages diff --git a/api/src/backend/tasks/jobs/integrations.py b/api/src/backend/tasks/jobs/integrations.py index 25722686cc..c77ba22b7f 100644 --- a/api/src/backend/tasks/jobs/integrations.py +++ b/api/src/backend/tasks/jobs/integrations.py @@ -14,6 +14,7 @@ from prowler.lib.outputs.compliance.generic.generic import GenericCompliance from prowler.lib.outputs.csv.csv import CSV from prowler.lib.outputs.finding import Finding as FindingOutput from prowler.lib.outputs.html.html import HTML +from prowler.lib.outputs.jira.exceptions.exceptions import JiraBaseException from prowler.lib.outputs.ocsf.ocsf import OCSF from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.s3.s3 import S3 @@ -26,6 +27,8 @@ from tasks.utils import batched logger = get_task_logger(__name__) +JIRA_GENERIC_SEND_ERROR = "Failed to create Jira issue." + def get_s3_client_from_integration( integration: Integration, @@ -483,6 +486,7 @@ def send_findings_to_jira( jira_integration = initialize_prowler_integration(integration) num_tickets_created = 0 + error_messages = [] for finding_id in finding_ids: with rls_transaction(tenant_id): finding_instance = ( @@ -512,35 +516,54 @@ def send_findings_to_jira( recommendation = remediation.get("recommendation", {}) remediation_code = remediation.get("code", {}) - # Send the individual finding to Jira - result = jira_integration.send_finding( - check_id=finding_instance.check_id, - check_title=check_metadata.get("checktitle", ""), - severity=finding_instance.severity, - status=finding_instance.status, - status_extended=finding_instance.status_extended or "", - provider=finding_instance.scan.provider.provider, - region=region, - resource_uid=resource_uid, - resource_name=resource_name, - risk=check_metadata.get("risk", ""), - recommendation_text=recommendation.get("text", ""), - recommendation_url=recommendation.get("url", ""), - remediation_code_native_iac=remediation_code.get("nativeiac", ""), - remediation_code_terraform=remediation_code.get("terraform", ""), - remediation_code_cli=remediation_code.get("cli", ""), - remediation_code_other=remediation_code.get("other", ""), - resource_tags=resource_tags, - compliance=finding_instance.compliance or {}, - project_key=project_key, - issue_type=issue_type, - ) + try: + # Send the individual finding to Jira + result = jira_integration.send_finding( + check_id=finding_instance.check_id, + check_title=check_metadata.get("checktitle", ""), + severity=finding_instance.severity, + status=finding_instance.status, + status_extended=finding_instance.status_extended or "", + provider=finding_instance.scan.provider.provider, + region=region, + resource_uid=resource_uid, + resource_name=resource_name, + risk=check_metadata.get("risk", ""), + recommendation_text=recommendation.get("text", ""), + recommendation_url=recommendation.get("url", ""), + remediation_code_native_iac=remediation_code.get("nativeiac", ""), + remediation_code_terraform=remediation_code.get("terraform", ""), + remediation_code_cli=remediation_code.get("cli", ""), + remediation_code_other=remediation_code.get("other", ""), + resource_tags=resource_tags, + compliance=finding_instance.compliance or {}, + project_key=project_key, + issue_type=issue_type, + ) + except JiraBaseException as error: + error_message = error.message or JIRA_GENERIC_SEND_ERROR + logger.exception( + "Failed to send finding %s to Jira: %s", finding_id, error_message + ) + error_messages.append(error_message) + continue + except Exception: + logger.exception("Failed to send finding %s to Jira", finding_id) + error_messages.append(JIRA_GENERIC_SEND_ERROR) + continue + if result: num_tickets_created += 1 else: - logger.error(f"Failed to send finding {finding_id} to Jira") + error_message = JIRA_GENERIC_SEND_ERROR + logger.error(error_message) + error_messages.append(error_message) - return { + result = { "created_count": num_tickets_created, "failed_count": len(finding_ids) - num_tickets_created, } + if error_messages: + result["error"] = "; ".join(dict.fromkeys(error_messages)) + + return result diff --git a/api/src/backend/tasks/tests/test_integrations.py b/api/src/backend/tasks/tests/test_integrations.py index 9cb727e8d0..bebb813feb 100644 --- a/api/src/backend/tasks/tests/test_integrations.py +++ b/api/src/backend/tasks/tests/test_integrations.py @@ -5,6 +5,10 @@ from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.models import Integration from api.utils import prowler_integration_connection_test from django.db import OperationalError +from prowler.lib.outputs.jira.exceptions.exceptions import ( + JiraRefreshTokenError, + JiraRequiredCustomFieldsError, +) from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection from prowler.providers.common.models import Connection from tasks.jobs.integrations import ( @@ -1830,10 +1834,213 @@ class TestJiraIntegration: ) # Assertions - assert result == {"created_count": 2, "failed_count": 1} + assert result == { + "created_count": 2, + "failed_count": 1, + "error": "Failed to create Jira issue.", + } # Verify error was logged for the failed finding - mock_logger.error.assert_called_with("Failed to send finding finding-2 to Jira") + mock_logger.error.assert_called_with("Failed to create Jira issue.") + + @patch("tasks.jobs.integrations.rls_transaction") + @patch("tasks.jobs.integrations.Finding") + @patch("tasks.jobs.integrations.Integration") + @patch("tasks.jobs.integrations.initialize_prowler_integration") + @patch("tasks.jobs.integrations.logger") + def test_send_findings_to_jira_preserves_exception_message( + self, + mock_logger, + mock_initialize_integration, + mock_integration_model, + mock_finding_model, + mock_rls_transaction, + ): + """Test Jira send exceptions are returned for UI polling.""" + tenant_id = "tenant-123" + integration_id = "integration-456" + project_key = "PROJ" + issue_type = "Task" + finding_ids = ["finding-1"] + error_message = "Jira project requires custom fields: Team is required" + + mock_rls_transaction.return_value.__enter__ = MagicMock() + mock_rls_transaction.return_value.__exit__ = MagicMock() + + integration = MagicMock() + mock_integration_model.objects.get.return_value = integration + + mock_jira_integration = MagicMock() + + mock_jira_integration.send_finding.side_effect = JiraRequiredCustomFieldsError( + message=error_message + ) + mock_initialize_integration.return_value = mock_jira_integration + + finding = MagicMock() + finding.id = "finding-1" + finding.check_id = "check_001" + finding.severity = "high" + finding.status = "FAIL" + finding.status_extended = "Resource is not compliant" + finding.compliance = {} + finding.resources.exists.return_value = False + finding.resources.first.return_value = None + finding.scan.provider.provider = "aws" + finding.check_metadata = { + "checktitle": "Check Title", + "risk": "High risk", + "remediation": {"recommendation": {}, "code": {}}, + } + mock_select_related = mock_finding_model.all_objects.select_related.return_value + mock_finding_query = mock_select_related.prefetch_related.return_value + mock_finding_query.get.return_value = finding + + result = send_findings_to_jira( + tenant_id, integration_id, project_key, issue_type, finding_ids + ) + + assert result == { + "created_count": 0, + "failed_count": 1, + "error": error_message, + } + mock_logger.exception.assert_called_with( + "Failed to send finding %s to Jira: %s", + "finding-1", + error_message, + ) + + @patch("tasks.jobs.integrations.rls_transaction") + @patch("tasks.jobs.integrations.Finding") + @patch("tasks.jobs.integrations.Integration") + @patch("tasks.jobs.integrations.initialize_prowler_integration") + @patch("tasks.jobs.integrations.logger") + def test_send_findings_to_jira_preserves_refresh_token_error_message( + self, + mock_logger, + mock_initialize_integration, + mock_integration_model, + mock_finding_model, + mock_rls_transaction, + ): + """Test Jira refresh token exceptions return their UI-friendly message.""" + tenant_id = "tenant-123" + integration_id = "integration-456" + project_key = "PROJ" + issue_type = "Task" + finding_ids = ["finding-1"] + error_message = "Failed to refresh the access token" + + mock_rls_transaction.return_value.__enter__ = MagicMock() + mock_rls_transaction.return_value.__exit__ = MagicMock() + + integration = MagicMock() + mock_integration_model.objects.get.return_value = integration + + mock_jira_integration = MagicMock() + + mock_jira_integration.send_finding.side_effect = JiraRefreshTokenError( + message=error_message + ) + mock_initialize_integration.return_value = mock_jira_integration + + finding = MagicMock() + finding.id = "finding-1" + finding.check_id = "check_001" + finding.severity = "high" + finding.status = "FAIL" + finding.status_extended = "Resource is not compliant" + finding.compliance = {} + finding.resources.exists.return_value = False + finding.resources.first.return_value = None + finding.scan.provider.provider = "aws" + finding.check_metadata = { + "checktitle": "Check Title", + "risk": "High risk", + "remediation": {"recommendation": {}, "code": {}}, + } + mock_select_related = mock_finding_model.all_objects.select_related.return_value + mock_finding_query = mock_select_related.prefetch_related.return_value + mock_finding_query.get.return_value = finding + + result = send_findings_to_jira( + tenant_id, integration_id, project_key, issue_type, finding_ids + ) + + assert result == { + "created_count": 0, + "failed_count": 1, + "error": error_message, + } + mock_logger.exception.assert_called_with( + "Failed to send finding %s to Jira: %s", + "finding-1", + error_message, + ) + + @patch("tasks.jobs.integrations.rls_transaction") + @patch("tasks.jobs.integrations.Finding") + @patch("tasks.jobs.integrations.Integration") + @patch("tasks.jobs.integrations.initialize_prowler_integration") + @patch("tasks.jobs.integrations.logger") + def test_send_findings_to_jira_sanitizes_unexpected_exception_message( + self, + mock_logger, + mock_initialize_integration, + mock_integration_model, + mock_finding_model, + mock_rls_transaction, + ): + """Test unexpected Jira send exceptions do not leak raw details to UI.""" + tenant_id = "tenant-123" + integration_id = "integration-456" + project_key = "PROJ" + issue_type = "Task" + finding_ids = ["finding-1"] + + mock_rls_transaction.return_value.__enter__ = MagicMock() + mock_rls_transaction.return_value.__exit__ = MagicMock() + + integration = MagicMock() + mock_integration_model.objects.get.return_value = integration + + mock_jira_integration = MagicMock() + mock_jira_integration.send_finding.side_effect = Exception("token=secret-value") + mock_initialize_integration.return_value = mock_jira_integration + + finding = MagicMock() + finding.id = "finding-1" + finding.check_id = "check_001" + finding.severity = "high" + finding.status = "FAIL" + finding.status_extended = "Resource is not compliant" + finding.compliance = {} + finding.resources.exists.return_value = False + finding.resources.first.return_value = None + finding.scan.provider.provider = "aws" + finding.check_metadata = { + "checktitle": "Check Title", + "risk": "High risk", + "remediation": {"recommendation": {}, "code": {}}, + } + mock_select_related = mock_finding_model.all_objects.select_related.return_value + mock_finding_query = mock_select_related.prefetch_related.return_value + mock_finding_query.get.return_value = finding + + result = send_findings_to_jira( + tenant_id, integration_id, project_key, issue_type, finding_ids + ) + + assert result == { + "created_count": 0, + "failed_count": 1, + "error": "Failed to create Jira issue.", + } + assert "secret-value" not in result["error"] + mock_logger.exception.assert_called_with( + "Failed to send finding %s to Jira", "finding-1" + ) @patch("tasks.jobs.integrations.rls_transaction") @patch("tasks.jobs.integrations.Finding") diff --git a/prowler/changelog.d/jira-dispatch-errors.fixed.md b/prowler/changelog.d/jira-dispatch-errors.fixed.md new file mode 100644 index 0000000000..c626872ecd --- /dev/null +++ b/prowler/changelog.d/jira-dispatch-errors.fixed.md @@ -0,0 +1 @@ +Jira issue creation failures now preserve safe structured response details from Jira diff --git a/prowler/lib/outputs/jira/jira.py b/prowler/lib/outputs/jira/jira.py index 9005b5274e..59b4d015aa 100644 --- a/prowler/lib/outputs/jira/jira.py +++ b/prowler/lib/outputs/jira/jira.py @@ -51,6 +51,39 @@ class JiraConnection(Connection): issue_types: dict = None +def _format_jira_issue_creation_error(response_json: object, status_code: int) -> str: + """Build a safe Jira issue creation error message from structured fields. + + Args: + response_json: Parsed Jira response body. + status_code: HTTP status code returned by Jira. + + Returns: + Safe issue creation error message for user-facing propagation. + """ + message_parts = [] + + if not isinstance(response_json, dict): + return f"Failed to create Jira issue: Jira returned status code {status_code}." + + errors = response_json.get("errors") + if isinstance(errors, dict): + message_parts.extend( + f"'{field}': '{message}'" for field, message in errors.items() if message + ) + + error_messages = response_json.get("errorMessages") + if isinstance(error_messages, list): + message_parts.extend(str(message) for message in error_messages if message) + elif isinstance(error_messages, str) and error_messages: + message_parts.append(error_messages) + + if message_parts: + return f"Failed to create Jira issue: {'; '.join(message_parts)}" + + return f"Failed to create Jira issue: Jira returned status code {status_code}." + + class MarkdownToADFConverter: """Helper to convert Markdown strings into Atlassian Document Format blocks.""" @@ -2060,6 +2093,7 @@ class Jira: Raises: - JiraRefreshTokenError: Failed to refresh the access token - JiraRefreshTokenResponseError: Failed to refresh the access token, response code did not match 200 + - JiraNoTokenError: Failed to get an access token - JiraCreateIssueError: Failed to create an issue in Jira - JiraSendFindingsResponseError: Failed to send the finding to Jira - JiraRequiredCustomFieldsError: Jira project requires custom fields that are not supported @@ -2155,31 +2189,45 @@ class Jira: try: response_json = response.json() except (ValueError, requests.exceptions.JSONDecodeError): - response_error = f"Failed to send finding: {response.status_code} - {response.text}" + response_error = _format_jira_issue_creation_error( + {}, response.status_code + ) logger.error(response_error) - return False + raise JiraSendFindingsResponseError( + message=response_error, file=os.path.basename(__file__) + ) # Check if the error is due to required custom fields - if response.status_code == 400 and "errors" in response_json: + if ( + response.status_code == 400 + and isinstance(response_json, dict) + and "errors" in response_json + ): errors = response_json.get("errors", {}) # Look for custom field errors (fields starting with "customfield_") - custom_field_errors = { - k: v for k, v in errors.items() if k.startswith("customfield_") - } + custom_field_errors = {} + if isinstance(errors, dict): + custom_field_errors = { + k: v + for k, v in errors.items() + if k.startswith("customfield_") + } if custom_field_errors: custom_fields_formatted = ", ".join( [f"'{k}': '{v}'" for k, v in custom_field_errors.items()] ) - logger.error( - f"Jira project requires custom fields that are not supported: {custom_fields_formatted}" + raise JiraRequiredCustomFieldsError( + message=f"Jira project requires custom fields that are not supported: {custom_fields_formatted}", + file=os.path.basename(__file__), ) - return False - response_error = ( - f"Failed to send finding: {response.status_code} - {response_json}" + response_error = _format_jira_issue_creation_error( + response_json, response.status_code ) logger.error(response_error) - return False + raise JiraSendFindingsResponseError( + message=response_error, file=os.path.basename(__file__) + ) else: try: response_json = response.json() @@ -2191,13 +2239,17 @@ class Jira: return True except JiraRequiredCustomFieldsError as custom_fields_error: logger.error(f"Custom fields error: {custom_fields_error}") - return False + raise custom_fields_error + except JiraSendFindingsResponseError as response_error: + logger.error(f"Jira response error: {response_error}") + raise response_error except JiraRefreshTokenError as refresh_error: logger.error(f"Token refresh error: {refresh_error}") - return False + raise refresh_error except JiraRefreshTokenResponseError as response_error: - logger.error(f"Token response error: {response_error}") - return False + raise response_error + except JiraNoTokenError as no_token_error: + raise no_token_error except Exception as e: logger.error(f"Failed to send finding: {e}") return False diff --git a/tests/lib/outputs/jira/jira_test.py b/tests/lib/outputs/jira/jira_test.py index 76352cb297..b71a58a570 100644 --- a/tests/lib/outputs/jira/jira_test.py +++ b/tests/lib/outputs/jira/jira_test.py @@ -16,8 +16,11 @@ from prowler.lib.outputs.jira.exceptions.exceptions import ( JiraGetProjectsError, JiraGetProjectsResponseError, JiraNoProjectsError, + JiraNoTokenError, JiraRefreshTokenError, + JiraRefreshTokenResponseError, JiraRequiredCustomFieldsError, + JiraSendFindingsResponseError, JiraTestConnectionError, ) from prowler.lib.outputs.jira.jira import Jira @@ -1692,7 +1695,7 @@ class TestJiraIntegration: mock_cloud_id, mock_get_access_token, ): - """Test that send_finding returns False when the request fails.""" + """Test that send_finding raises with Jira JSON error details.""" # To disable vulture mock_cloud_id = mock_cloud_id mock_get_access_token = mock_get_access_token @@ -1702,19 +1705,66 @@ class TestJiraIntegration: # Mock failed response mock_response = MagicMock() mock_response.status_code = 400 - mock_response.json.return_value = {"errors": {"summary": "Required field"}} + mock_response.json.return_value = { + "errors": {"Team": "Team is required."}, + "errorMessages": ["Field 'Team' cannot be set."], + } mock_post.return_value = mock_response - result = self.jira_integration.send_finding( - check_id="test-check", - check_title="Test Finding", - severity="High", - status="FAIL", - project_key="TEST", - issue_type="Bug", - ) + with pytest.raises(JiraSendFindingsResponseError) as error: + self.jira_integration.send_finding( + check_id="test-check", + check_title="Test Finding", + severity="High", + status="FAIL", + project_key="TEST", + issue_type="Bug", + ) - assert result is False + assert "Failed to create Jira issue" in str(error.value) + assert "'Team': 'Team is required.'" in str(error.value) + assert "Field 'Team' cannot be set." in str(error.value) + mock_post.assert_called_once() + + @patch.object(Jira, "get_access_token", return_value="valid_access_token") + @patch.object( + Jira, "cloud_id", new_callable=PropertyMock, return_value="test_cloud_id" + ) + @patch.object(Jira, "get_projects", return_value={"TEST": {"name": "Test Project"}}) + @patch.object(Jira, "get_available_issue_types", return_value=["Bug"]) + @patch("prowler.lib.outputs.jira.jira.requests.post") + def test_send_finding_response_error_without_json_body( + self, + mock_post, + mock_get_issue_types, + mock_get_projects, + mock_cloud_id, + mock_get_access_token, + ): + """Test send_finding raises with status-code context for non-JSON errors.""" + # To disable vulture + mock_cloud_id = mock_cloud_id + mock_get_access_token = mock_get_access_token + mock_get_projects = mock_get_projects + mock_get_issue_types = mock_get_issue_types + + mock_response = MagicMock() + mock_response.status_code = 502 + mock_response.json.side_effect = ValueError("No JSON body") + mock_post.return_value = mock_response + + with pytest.raises(JiraSendFindingsResponseError) as error: + self.jira_integration.send_finding( + check_id="test-check", + check_title="Test Finding", + severity="High", + status="FAIL", + project_key="TEST", + issue_type="Bug", + ) + + assert "Failed to create Jira issue" in str(error.value) + assert "Jira returned status code 502" in str(error.value) mock_post.assert_called_once() @patch.object(Jira, "get_access_token", return_value="valid_access_token") @@ -1732,7 +1782,7 @@ class TestJiraIntegration: mock_cloud_id, mock_get_access_token, ): - """Test that send_finding returns False when custom fields cause an error.""" + """Test that send_finding raises when custom fields cause an error.""" # To disable vulture mock_cloud_id = mock_cloud_id mock_get_access_token = mock_get_access_token @@ -1750,18 +1800,89 @@ class TestJiraIntegration: } mock_post.return_value = mock_response - result = self.jira_integration.send_finding( - check_id="test-check", - check_title="Test Finding", - severity="High", - status="FAIL", - project_key="TEST", - issue_type="Bug", - ) + with pytest.raises(JiraRequiredCustomFieldsError) as error: + self.jira_integration.send_finding( + check_id="test-check", + check_title="Test Finding", + severity="High", + status="FAIL", + project_key="TEST", + issue_type="Bug", + ) - assert result is False + assert "Jira project requires custom fields" in str(error.value) + assert "customfield_10001" in str(error.value) mock_post.assert_called_once() + @patch.object( + Jira, + "get_access_token", + side_effect=JiraRefreshTokenError(message="Failed to refresh the access token"), + ) + def test_send_finding_reraises_refresh_token_error(self, mock_get_access_token): + """Test send_finding re-raises refresh token errors for API propagation.""" + # To disable vulture + mock_get_access_token = mock_get_access_token + + with pytest.raises(JiraRefreshTokenError) as error: + self.jira_integration.send_finding( + check_id="test-check", + check_title="Test Finding", + severity="High", + status="FAIL", + project_key="TEST", + issue_type="Bug", + ) + + assert error.value.message == "Failed to refresh the access token" + + @patch.object(Jira, "get_access_token", return_value=None) + def test_send_finding_reraises_no_token_error(self, mock_get_access_token): + """Test send_finding re-raises missing token errors for API propagation.""" + # To disable vulture + mock_get_access_token = mock_get_access_token + + with pytest.raises(JiraNoTokenError) as error: + self.jira_integration.send_finding( + check_id="test-check", + check_title="Test Finding", + severity="High", + status="FAIL", + project_key="TEST", + issue_type="Bug", + ) + + assert error.value.message == "No token was found" + + @patch.object( + Jira, + "get_access_token", + side_effect=JiraRefreshTokenResponseError( + message="Failed to refresh the access token, response code did not match 200" + ), + ) + def test_send_finding_reraises_refresh_token_response_error( + self, mock_get_access_token + ): + """Test send_finding re-raises refresh token response errors for API propagation.""" + # To disable vulture + mock_get_access_token = mock_get_access_token + + with pytest.raises(JiraRefreshTokenResponseError) as error: + self.jira_integration.send_finding( + check_id="test-check", + check_title="Test Finding", + severity="High", + status="FAIL", + project_key="TEST", + issue_type="Bug", + ) + + assert ( + error.value.message + == "Failed to refresh the access token, response code did not match 200" + ) + def test_get_headers_oauth_with_access_token(self): """Test get_headers returns correct OAuth headers with access token.""" self.jira_integration._using_basic_auth = False diff --git a/ui/actions/integrations/jira-dispatch.test.ts b/ui/actions/integrations/jira-dispatch.test.ts new file mode 100644 index 0000000000..05c3188e47 --- /dev/null +++ b/ui/actions/integrations/jira-dispatch.test.ts @@ -0,0 +1,152 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { pollTaskUntilSettledMock } = vi.hoisted(() => ({ + pollTaskUntilSettledMock: vi.fn(), +})); + +vi.mock("@/actions/task/poll", () => ({ + pollTaskUntilSettled: pollTaskUntilSettledMock, +})); + +vi.mock("@/lib", () => ({ + apiBaseUrl: "https://api.example.com/api/v1", + getAuthHeaders: vi.fn(), +})); + +vi.mock("@/lib/server-actions-helper", () => ({ + handleApiError: vi.fn(), +})); + +import { pollJiraDispatchTask } from "./jira-dispatch"; + +describe("pollJiraDispatchTask", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should return the backend error when a completed task has failed Jira dispatches", async () => { + // Given + pollTaskUntilSettledMock.mockResolvedValue({ + ok: true, + state: "completed", + result: { + created_count: 0, + failed_count: 1, + error: "Jira project requires custom fields: Team is required", + }, + }); + + // When + const result = await pollJiraDispatchTask("task-123"); + + // Then + expect(result).toEqual({ + success: false, + error: "Jira project requires custom fields: Team is required", + }); + }); + + it("should return a fallback error when a completed task has failures without an error", async () => { + // Given + pollTaskUntilSettledMock.mockResolvedValue({ + ok: true, + state: "completed", + result: { + created_count: 0, + failed_count: 1, + }, + }); + + // When + const result = await pollJiraDispatchTask("task-123"); + + // Then + expect(result).toEqual({ + success: false, + error: "Failed to create Jira issue.", + }); + }); + + it("should return a plural fallback error when a completed task has multiple failures without an error", async () => { + // Given + pollTaskUntilSettledMock.mockResolvedValue({ + ok: true, + state: "completed", + result: { + created_count: 0, + failed_count: 3, + }, + }); + + // When + const result = await pollJiraDispatchTask("task-123"); + + // Then + expect(result).toEqual({ + success: false, + error: "Failed to create 3 Jira issues.", + }); + }); + + it("should surface task failure result errors", async () => { + // Given + pollTaskUntilSettledMock.mockResolvedValue({ + ok: true, + state: "failed", + result: { + error: "Jira credentials are invalid.", + }, + }); + + // When + const result = await pollJiraDispatchTask("task-123"); + + // Then + expect(result).toEqual({ + success: false, + error: "Jira credentials are invalid.", + }); + }); + + it("should return success when a completed task has no failures", async () => { + // Given + pollTaskUntilSettledMock.mockResolvedValue({ + ok: true, + state: "completed", + result: { + created_count: 1, + failed_count: 0, + }, + }); + + // When + const result = await pollJiraDispatchTask("task-123"); + + // Then + expect(result).toEqual({ + success: true, + message: "Finding successfully sent to Jira!", + }); + }); + + it("should return a fallback error when no Jira issue was created", async () => { + // Given + pollTaskUntilSettledMock.mockResolvedValue({ + ok: true, + state: "completed", + result: { + created_count: 0, + failed_count: 0, + }, + }); + + // When + const result = await pollJiraDispatchTask("task-123"); + + // Then + expect(result).toEqual({ + success: false, + error: "Failed to create Jira issue.", + }); + }); +}); diff --git a/ui/actions/integrations/jira-dispatch.ts b/ui/actions/integrations/jira-dispatch.ts index 9e3b25679f..b6d3215e13 100644 --- a/ui/actions/integrations/jira-dispatch.ts +++ b/ui/actions/integrations/jira-dispatch.ts @@ -159,12 +159,18 @@ export const pollJiraDispatchTask = async ( const jiraResult = result as JiraTaskResult | undefined; if (state === "completed") { - if (!jiraResult?.error) { + const createdCount = jiraResult?.created_count ?? 0; + const failedCount = jiraResult?.failed_count ?? 0; + if (!jiraResult?.error && failedCount === 0 && createdCount > 0) { return { success: true, message: "Finding successfully sent to Jira!" }; } return { success: false, - error: jiraResult?.error || "Failed to create Jira issue.", + error: + jiraResult?.error || + (failedCount > 1 + ? `Failed to create ${failedCount} Jira issues.` + : "Failed to create Jira issue."), }; } diff --git a/ui/changelog.d/jira-dispatch-errors.fixed.md b/ui/changelog.d/jira-dispatch-errors.fixed.md new file mode 100644 index 0000000000..3d9afe9918 --- /dev/null +++ b/ui/changelog.d/jira-dispatch-errors.fixed.md @@ -0,0 +1 @@ +Jira dispatch polling now reports failed issue creation tasks instead of treating partial failures as successful diff --git a/ui/types/integrations.ts b/ui/types/integrations.ts index 63b1e37bff..3ff525964e 100644 --- a/ui/types/integrations.ts +++ b/ui/types/integrations.ts @@ -64,6 +64,8 @@ export interface JiraDispatchResponse { message?: string; issue_url?: string; issue_key?: string; + created_count?: number; + failed_count?: number; } | null; task_args: Record | null; metadata: Record | null; From 6f72785a28ae35892bdb399861e22b0d5c32f437 Mon Sep 17 00:00:00 2001 From: Hugo Pereira Brito <101209179+HugoPBrito@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:23:09 +0100 Subject: [PATCH 05/20] fix(azure): warn on optional function app permission failures (#11922) --- .../providers/azure/authentication.mdx | 2 + ...-function-app-permission-warnings.fixed.md | 1 + ...p_function_application_insights_enabled.py | 6 +-- .../app_function_latest_runtime_version.py | 6 +-- .../azure/services/app/app_service.py | 8 +-- ...pp_function_access_keys_configured_test.py | 4 +- ...ction_application_insights_enabled_test.py | 12 +++-- ..._function_ftps_deployment_disabled_test.py | 6 +-- ...pp_function_identity_is_configured_test.py | 4 +- ..._identity_without_admin_privileges_test.py | 6 +-- ...pp_function_latest_runtime_version_test.py | 4 +- ...p_function_not_publicly_accessible_test.py | 4 +- ..._function_vnet_integration_enabled_test.py | 4 +- .../azure/services/app/app_service_test.py | 52 ++++++++++++++++++- 14 files changed, 88 insertions(+), 31 deletions(-) create mode 100644 prowler/changelog.d/azure-function-app-permission-warnings.fixed.md diff --git a/docs/user-guide/providers/azure/authentication.mdx b/docs/user-guide/providers/azure/authentication.mdx index 852e79d115..08bc79d8ff 100644 --- a/docs/user-guide/providers/azure/authentication.mdx +++ b/docs/user-guide/providers/azure/authentication.mdx @@ -210,7 +210,9 @@ For more detailed guidance on subscription management and permissions: The following security checks require the `ProwlerRole` permissions for execution. Ensure the role is assigned to the identity assumed by Prowler before running these checks: - `app_function_access_keys_configured` +- `app_function_application_insights_enabled` - `app_function_ftps_deployment_disabled` +- `app_function_latest_runtime_version` --- diff --git a/prowler/changelog.d/azure-function-app-permission-warnings.fixed.md b/prowler/changelog.d/azure-function-app-permission-warnings.fixed.md new file mode 100644 index 0000000000..f455f1ee19 --- /dev/null +++ b/prowler/changelog.d/azure-function-app-permission-warnings.fixed.md @@ -0,0 +1 @@ +Azure Function App optional permission failures now log as warnings, and Function App environment variable fields use the correct spelling internally diff --git a/prowler/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled.py b/prowler/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled.py index 6fec5e7042..a903097beb 100644 --- a/prowler/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled.py +++ b/prowler/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled.py @@ -14,7 +14,7 @@ class app_function_application_insights_enabled(Check): subscription_id, subscription_id ) for function in functions.values(): - if function.enviroment_variables is not None: + if function.environment_variables is not None: report = Check_Report_Azure( metadata=self.metadata(), resource=function ) @@ -22,9 +22,9 @@ class app_function_application_insights_enabled(Check): report.status = "FAIL" report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is not using Application Insights." - if function.enviroment_variables.get( + if function.environment_variables.get( "APPINSIGHTS_INSTRUMENTATIONKEY", None - ) or function.enviroment_variables.get( + ) or function.environment_variables.get( "APPLICATIONINSIGHTS_CONNECTION_STRING", None ): report.status = "PASS" diff --git a/prowler/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version.py b/prowler/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version.py index 828362a8fe..694c5bbdc6 100644 --- a/prowler/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version.py +++ b/prowler/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version.py @@ -14,7 +14,7 @@ class app_function_latest_runtime_version(Check): subscription_id, subscription_id ) for function in functions.values(): - if function.enviroment_variables is not None: + if function.environment_variables is not None: report = Check_Report_Azure( metadata=self.metadata(), resource=function ) @@ -23,13 +23,13 @@ class app_function_latest_runtime_version(Check): report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is using the latest runtime." if ( - function.enviroment_variables.get( + function.environment_variables.get( "FUNCTIONS_EXTENSION_VERSION", "" ) != "~4" ): report.status = "FAIL" - report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is not using the latest runtime. The current runtime is '{function.enviroment_variables.get('FUNCTIONS_EXTENSION_VERSION', '')}' and should be '~4'." + report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is not using the latest runtime. The current runtime is '{function.environment_variables.get('FUNCTIONS_EXTENSION_VERSION', '')}' and should be '~4'." findings.append(report) diff --git a/prowler/providers/azure/services/app/app_service.py b/prowler/providers/azure/services/app/app_service.py index 83d23be516..5b090bf1f2 100644 --- a/prowler/providers/azure/services/app/app_service.py +++ b/prowler/providers/azure/services/app/app_service.py @@ -158,7 +158,7 @@ class App(AzureService): location=function.location, kind=function.kind, function_keys=function_keys, - enviroment_variables=getattr( + environment_variables=getattr( application_settings, "properties", None ), identity=getattr(function, "identity", None), @@ -225,7 +225,7 @@ class App(AzureService): name=name, ) except Exception as error: - logger.error( + logger.warning( f"Error getting host keys for {name} in {resource_group}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) return None @@ -249,7 +249,7 @@ class App(AzureService): name=name, ) except Exception as error: - logger.error( + logger.warning( f"Error getting application settings for {name} in {resource_group}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) return None @@ -296,7 +296,7 @@ class FunctionApp: location: str kind: str function_keys: Optional[Dict[str, str]] - enviroment_variables: Optional[Dict[str, str]] + environment_variables: Optional[Dict[str, str]] identity: ManagedServiceIdentity public_access: bool vnet_subnet_id: str diff --git a/tests/providers/azure/services/app/app_function_access_keys_configured/app_function_access_keys_configured_test.py b/tests/providers/azure/services/app/app_function_access_keys_configured/app_function_access_keys_configured_test.py index 770ca07b2b..803ba7bd75 100644 --- a/tests/providers/azure/services/app/app_function_access_keys_configured/app_function_access_keys_configured_test.py +++ b/tests/providers/azure/services/app/app_function_access_keys_configured/app_function_access_keys_configured_test.py @@ -87,7 +87,7 @@ class Test_app_function_access_keys_configured: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=False, vnet_subnet_id=None, @@ -143,7 +143,7 @@ class Test_app_function_access_keys_configured: "default": "key1", "key2": "key2", }, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=False, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled_test.py b/tests/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled_test.py index 4a55b1e108..b113d96e50 100644 --- a/tests/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled_test.py +++ b/tests/providers/azure/services/app/app_function_application_insights_enabled/app_function_application_insights_enabled_test.py @@ -87,7 +87,7 @@ class Test_app_function_application_insights_enabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=False, vnet_subnet_id=None, @@ -138,7 +138,9 @@ class Test_app_function_application_insights_enabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={"APPINSIGHTS_INSTRUMENTATIONKEY": "1234"}, + environment_variables={ + "APPINSIGHTS_INSTRUMENTATIONKEY": "1234" + }, identity=None, public_access=False, vnet_subnet_id=None, @@ -189,7 +191,9 @@ class Test_app_function_application_insights_enabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={"APPINSIGHTS_INSTRUMENTATIONKEY": "1234"}, + environment_variables={ + "APPINSIGHTS_INSTRUMENTATIONKEY": "1234" + }, identity=None, public_access=False, vnet_subnet_id=None, @@ -240,7 +244,7 @@ class Test_app_function_application_insights_enabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=False, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_ftps_deployment_disabled/app_function_ftps_deployment_disabled_test.py b/tests/providers/azure/services/app/app_function_ftps_deployment_disabled/app_function_ftps_deployment_disabled_test.py index b08c712da1..fb20317347 100644 --- a/tests/providers/azure/services/app/app_function_ftps_deployment_disabled/app_function_ftps_deployment_disabled_test.py +++ b/tests/providers/azure/services/app/app_function_ftps_deployment_disabled/app_function_ftps_deployment_disabled_test.py @@ -87,7 +87,7 @@ class Test_app_function_ftps_deployment_disabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(type="SystemAssigned"), public_access=False, vnet_subnet_id=None, @@ -138,7 +138,7 @@ class Test_app_function_ftps_deployment_disabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(type="SystemAssigned"), public_access=False, vnet_subnet_id=None, @@ -189,7 +189,7 @@ class Test_app_function_ftps_deployment_disabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(type="SystemAssigned"), public_access=False, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_identity_is_configured/app_function_identity_is_configured_test.py b/tests/providers/azure/services/app/app_function_identity_is_configured/app_function_identity_is_configured_test.py index 84dbece6d2..5a770a196c 100644 --- a/tests/providers/azure/services/app/app_function_identity_is_configured/app_function_identity_is_configured_test.py +++ b/tests/providers/azure/services/app/app_function_identity_is_configured/app_function_identity_is_configured_test.py @@ -87,7 +87,7 @@ class Test_app_function_identity_is_configured: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=False, vnet_subnet_id=None, @@ -138,7 +138,7 @@ class Test_app_function_identity_is_configured: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(type="SystemAssigned"), public_access=False, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_identity_without_admin_privileges/app_function_identity_without_admin_privileges_test.py b/tests/providers/azure/services/app/app_function_identity_without_admin_privileges/app_function_identity_without_admin_privileges_test.py index 9c7f074c3b..ef42098847 100644 --- a/tests/providers/azure/services/app/app_function_identity_without_admin_privileges/app_function_identity_without_admin_privileges_test.py +++ b/tests/providers/azure/services/app/app_function_identity_without_admin_privileges/app_function_identity_without_admin_privileges_test.py @@ -88,7 +88,7 @@ class Test_app_function_identity_without_admin_privileges: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=False, vnet_subnet_id=None, @@ -140,7 +140,7 @@ class Test_app_function_identity_without_admin_privileges: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(principal_id="123"), public_access=False, vnet_subnet_id=None, @@ -228,7 +228,7 @@ class Test_app_function_identity_without_admin_privileges: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(principal_id="123"), public_access=False, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version_test.py b/tests/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version_test.py index 89bfc642b0..597335bc39 100644 --- a/tests/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version_test.py +++ b/tests/providers/azure/services/app/app_function_latest_runtime_version/app_function_latest_runtime_version_test.py @@ -87,7 +87,7 @@ class Test_app_function_latest_runtime_version: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={"FUNCTIONS_EXTENSION_VERSION": "~4"}, + environment_variables={"FUNCTIONS_EXTENSION_VERSION": "~4"}, identity=None, public_access=False, vnet_subnet_id=None, @@ -137,7 +137,7 @@ class Test_app_function_latest_runtime_version: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={"FUNCTIONS_EXTENSION_VERSION": "2"}, + environment_variables={"FUNCTIONS_EXTENSION_VERSION": "2"}, identity=None, public_access=False, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_not_publicly_accessible/app_function_not_publicly_accessible_test.py b/tests/providers/azure/services/app/app_function_not_publicly_accessible/app_function_not_publicly_accessible_test.py index 3c60aebc20..f8f36af8f4 100644 --- a/tests/providers/azure/services/app/app_function_not_publicly_accessible/app_function_not_publicly_accessible_test.py +++ b/tests/providers/azure/services/app/app_function_not_publicly_accessible/app_function_not_publicly_accessible_test.py @@ -87,7 +87,7 @@ class Test_app_function_not_publicly_accessible: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(type="SystemAssigned"), public_access=False, vnet_subnet_id=None, @@ -138,7 +138,7 @@ class Test_app_function_not_publicly_accessible: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=mock.MagicMock(type="SystemAssigned"), public_access=True, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_function_vnet_integration_enabled/app_function_vnet_integration_enabled_test.py b/tests/providers/azure/services/app/app_function_vnet_integration_enabled/app_function_vnet_integration_enabled_test.py index f12422f1da..c12b14de7c 100644 --- a/tests/providers/azure/services/app/app_function_vnet_integration_enabled/app_function_vnet_integration_enabled_test.py +++ b/tests/providers/azure/services/app/app_function_vnet_integration_enabled/app_function_vnet_integration_enabled_test.py @@ -87,7 +87,7 @@ class Test_app_function_vnet_integration_enabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=True, vnet_subnet_id="vnet_subnet_id", @@ -138,7 +138,7 @@ class Test_app_function_vnet_integration_enabled: location="West Europe", kind="functionapp,linux", function_keys={}, - enviroment_variables={}, + environment_variables={}, identity=None, public_access=True, vnet_subnet_id=None, diff --git a/tests/providers/azure/services/app/app_service_test.py b/tests/providers/azure/services/app/app_service_test.py index 76f602431e..ccd396cc0d 100644 --- a/tests/providers/azure/services/app/app_service_test.py +++ b/tests/providers/azure/services/app/app_service_test.py @@ -222,7 +222,7 @@ class Test_App_Service: location="West Europe", kind="functionapp", function_keys=None, - enviroment_variables=None, + environment_variables=None, identity=ManagedServiceIdentity(type="SystemAssigned"), public_access=True, vnet_subnet_id="", @@ -247,6 +247,56 @@ class Test_App_Service: == "functionapp-1" ) + def test_get_function_host_keys_logs_warning_on_optional_failure(self): + from prowler.providers.azure.services.app.app_service import App + + mock_client = MagicMock() + mock_client.web_apps.list_host_keys.side_effect = Exception("Forbidden") + app = object.__new__(App) + app.clients = {AZURE_SUBSCRIPTION_ID: mock_client} + + with ( + patch( + "prowler.providers.azure.services.app.app_service.logger.warning" + ) as mock_warning, + patch( + "prowler.providers.azure.services.app.app_service.logger.error" + ) as mock_error, + ): + result = app._get_function_host_keys( + AZURE_SUBSCRIPTION_ID, RESOURCE_GROUP, "functionapp-1" + ) + + assert result is None + mock_warning.assert_called_once() + mock_error.assert_not_called() + + def test_list_application_settings_logs_warning_on_optional_failure(self): + from prowler.providers.azure.services.app.app_service import App + + mock_client = MagicMock() + mock_client.web_apps.list_application_settings.side_effect = Exception( + "Forbidden" + ) + app = object.__new__(App) + app.clients = {AZURE_SUBSCRIPTION_ID: mock_client} + + with ( + patch( + "prowler.providers.azure.services.app.app_service.logger.warning" + ) as mock_warning, + patch( + "prowler.providers.azure.services.app.app_service.logger.error" + ) as mock_error, + ): + result = app._list_application_settings( + AZURE_SUBSCRIPTION_ID, RESOURCE_GROUP, "functionapp-1" + ) + + assert result is None + mock_warning.assert_called_once() + mock_error.assert_not_called() + class Test_App_get_apps: def test_get_apps_no_resource_groups(self): From b44f8ebf5ffb308fe8519cfeafdbbf6f79874b21 Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Thu, 9 Jul 2026 17:47:03 +0200 Subject: [PATCH 06/20] fix(api): add query-level retry with primary fallback to `rls_transaction` via `execute_wrapper` (#10379) Co-authored-by: Pepe Fagoaga --- .../rls-transaction-primary-fallback.fixed.md | 1 + api/src/backend/api/db_utils.py | 323 ++++++++-- .../tests/integration/test_rls_transaction.py | 40 +- api/src/backend/api/tests/test_db_utils.py | 592 +++++++++++++++++- api/src/backend/conftest.py | 20 + 5 files changed, 905 insertions(+), 71 deletions(-) create mode 100644 api/changelog.d/rls-transaction-primary-fallback.fixed.md diff --git a/api/changelog.d/rls-transaction-primary-fallback.fixed.md b/api/changelog.d/rls-transaction-primary-fallback.fixed.md new file mode 100644 index 0000000000..4f1035aa9f --- /dev/null +++ b/api/changelog.d/rls-transaction-primary-fallback.fixed.md @@ -0,0 +1 @@ +`rls_transaction` now falls back directly to the primary DB for connection-level mid-query read replica failures via `execute_wrapper`, reducing non-streaming read crashes during replica recovery diff --git a/api/src/backend/api/db_utils.py b/api/src/backend/api/db_utils.py index 2c378f2ea8..b6d3fdada1 100644 --- a/api/src/backend/api/db_utils.py +++ b/api/src/backend/api/db_utils.py @@ -2,7 +2,7 @@ import re import secrets import time import uuid -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager, nullcontext from datetime import UTC, datetime, timedelta from api.db_router import ( @@ -48,6 +48,140 @@ REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3) REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5) SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);" +SET_TRANSACTION_READ_ONLY_QUERY = "SET TRANSACTION READ ONLY;" + +REPLICA_CONNECTION_SQLSTATE_PREFIXES = ("08",) +REPLICA_CONNECTION_SQLSTATES = {"57P01", "57P02", "57P03"} +REPLICA_NON_FAILOVER_SQLSTATES = {"57014", "40001", "40P01"} +REPLICA_CONNECTION_ERROR_MESSAGES = ( + "ssl syscall", + "eof detected", + "server closed the connection", + "connection already closed", + "connection not open", + "could not connect to server", + "connection refused", + "connection reset", + "connection timed out", + "lost synchronization", + "terminating connection", + "database system is starting up", + "database system is shutting down", + "database system is in recovery mode", +) +REPLICA_NON_FAILOVER_ERROR_MESSAGES = ( + "canceling statement due to user request", + "deadlock detected", + "could not serialize access", +) + + +def _iter_exception_chain(error: BaseException): + seen = set() + pending = [error] + while pending: + current = pending.pop(0) + if current is None or id(current) in seen: + continue + seen.add(id(current)) + yield current + + cause = getattr(current, "__cause__", None) + context = getattr(current, "__context__", None) + if cause is not None: + pending.append(cause) + if context is not None: + pending.append(context) + for arg in getattr(current, "args", ()): + if isinstance(arg, BaseException): + pending.append(arg) + + +def _get_exception_sqlstate(error: BaseException) -> str | None: + for attr in ("pgcode", "sqlstate"): + sqlstate = getattr(error, attr, None) + if sqlstate: + return sqlstate + + diag = getattr(error, "diag", None) + if diag is not None: + sqlstate = getattr(diag, "sqlstate", None) + if sqlstate: + return sqlstate + return None + + +def _is_replica_connection_failure(error: BaseException) -> bool: + """ + Return True only for replica failures where retrying on primary is safe. + + Query cancellations, serialization failures, and deadlocks should surface to + callers because replaying them can hide real query or concurrency problems. + """ + messages = [] + sqlstates = set() + + for chained_error in _iter_exception_chain(error): + sqlstate = _get_exception_sqlstate(chained_error) + if sqlstate: + sqlstates.add(sqlstate) + messages.append(str(chained_error).lower()) + + if sqlstates & REPLICA_NON_FAILOVER_SQLSTATES: + return False + if any( + sqlstate.startswith(REPLICA_CONNECTION_SQLSTATE_PREFIXES) + or sqlstate in REPLICA_CONNECTION_SQLSTATES + for sqlstate in sqlstates + ): + return True + + message = " ".join(messages) + if any(marker in message for marker in REPLICA_NON_FAILOVER_ERROR_MESSAGES): + return False + + return any(marker in message for marker in REPLICA_CONNECTION_ERROR_MESSAGES) + + +def _strip_leading_sql_comments(sql: str) -> str: + if not isinstance(sql, str): + return "" + + sql_text = sql.lstrip() + while True: + if sql_text.startswith("--"): + newline_index = sql_text.find("\n") + if newline_index == -1: + return "" + sql_text = sql_text[newline_index + 1 :].lstrip() + continue + + if sql_text.startswith("/*"): + comment_end_index = sql_text.find("*/", 2) + if comment_end_index == -1: + return "" + sql_text = sql_text[comment_end_index + 2 :].lstrip() + continue + + return sql_text + + +def _is_safe_primary_replay(sql: str, many: bool) -> bool: + if many: + return False + + sql_text = _strip_leading_sql_comments(sql) + if not re.match(r"(?is)^SELECT\b", sql_text): + return False + + return not any( + re.search(pattern, sql_text, re.IGNORECASE | re.DOTALL) + for pattern in ( + r"\bINTO\b", + r"\bFOR\s+(?:NO\s+KEY\s+)?UPDATE\b", + r"\bFOR\s+(?:KEY\s+)?SHARE\b", + ) + ) @contextmanager @@ -77,14 +211,36 @@ def rls_transaction( retry_on_replica: bool = True, ): """ - Creates a new database transaction setting the given configuration value for Postgres RLS. It validates the - if the value is a valid UUID. + Context manager that opens an RLS-scoped database transaction. + + Sets a Postgres configuration variable (``set_config``) so that Row-Level + Security policies can filter by tenant. When *using* points to a read + replica and *retry_on_replica* is True, replica failures are handled in two + places: + + 1. **Pre-yield** (connection-setup failures): the function retries + up to ``REPLICA_MAX_ATTEMPTS`` times on the replica, then falls + back to the primary DB. + 2. **Post-yield** (mid-query failures): an ``execute_wrapper`` + intercepts connection-level ``OperationalError`` during + ``cursor.execute()`` calls and falls back directly to the primary DB + for single ``SELECT`` statements. The primary fallback transaction is + read-only, and unsafe statements keep raising the original error. + The wrapper swaps the inner cursor so ``fetchall()`` / ``fetchone()`` + read from the new connection transparently. + + Limitation: server-side cursors (``.iterator()``) fetch rows via + ``fetchmany()``, which the wrapper does not intercept. Call sites + that iterate large result sets with ``.iterator()`` on the replica + should add their own retry logic. Args: - value (str): Database configuration parameter value. - parameter (str): Database configuration parameter name, by default is 'api.tenant_id'. - using (str | None): Optional database alias to run the transaction against. Defaults to the - active read alias (if any) or Django's default connection. + value: Database configuration parameter value (must be a valid UUID). + parameter: Database configuration parameter name. + using: Optional database alias. Defaults to the active read + alias or Django's default connection. + retry_on_replica: Whether replica setup failures can retry and + connection-level mid-query failures can fall back to primary. """ requested_alias = using or get_read_db_alias() db_alias = requested_alias or DEFAULT_DB_ALIAS @@ -92,54 +248,121 @@ def rls_transaction( db_alias = DEFAULT_DB_ALIAS alias = db_alias - is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS - max_attempts = REPLICA_MAX_ATTEMPTS if is_replica and retry_on_replica else 1 + is_replica = bool(READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS) + can_failover = is_replica and retry_on_replica + replica_alias = alias # captured before the loop mutates alias + max_attempts = (REPLICA_MAX_ATTEMPTS + 1) if can_failover else 1 - for attempt in range(1, max_attempts + 1): - router_token = None - yielded_cursor = False + # State shared between the generator and the _query_failover closure. + # The fallback transaction.atomic() is registered into fallback_stack + # via enter_context so its __exit__ runs when the outer with-ExitStack + # block exits, with the right exc_info. No manual __enter__/__exit__. + _fallback = {"succeeded": False, "token": None, "caller_exited_cleanly": False} - # On final attempt, fallback to primary - if attempt == max_attempts and is_replica: - logger.warning( - f"RLS transaction failed after {attempt - 1} attempts on replica, " - f"falling back to primary DB" - ) - alias = DEFAULT_DB_ALIAS + with ExitStack() as fallback_stack: - conn = connections[alias] - try: - if alias != DEFAULT_DB_ALIAS: - router_token = set_read_db_alias(alias) + def _query_failover(execute, sql, params, many, context): + """execute_wrapper: replay failed replica queries on the primary DB.""" + try: + return execute(sql, params, many, context) + except OperationalError as err: + if not _is_replica_connection_failure(err): + raise + if not _is_safe_primary_replay(sql, many): + raise - with transaction.atomic(using=alias): - with conn.cursor() as cursor: - try: - # just in case the value is a UUID object - uuid.UUID(str(value)) - except ValueError: - raise ValidationError("Must be a valid UUID") - cursor.execute(SET_CONFIG_QUERY, [parameter, value]) - yielded_cursor = True - yield cursor - return - except OperationalError as e: - if yielded_cursor: - raise - # If on primary or max attempts reached, raise - if not is_replica or attempt == max_attempts: - raise + try: + connections[replica_alias].close() + except Exception: + pass # Best-effort; connection may already be dead - # Retry with exponential backoff - delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1)) - logger.info( - f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), " - f"retrying in {delay}s. Error: {e}" - ) - time.sleep(delay) - finally: - if router_token is not None: - reset_read_db_alias(router_token) + logger.warning( + "Mid-query replica connection failure, falling back to primary DB" + ) + primary = connections[DEFAULT_DB_ALIAS] + primary.ensure_connection() + fallback_stack.enter_context(transaction.atomic(using=DEFAULT_DB_ALIAS)) + + fallback_cursor = primary.cursor() + fallback_stack.callback(fallback_cursor.close) + fallback_cursor.execute(SET_TRANSACTION_READ_ONLY_QUERY) + fallback_cursor.execute(SET_CONFIG_QUERY, [parameter, value]) + _fallback["token"] = set_read_db_alias(DEFAULT_DB_ALIAS) + + fallback_cursor.execute(sql, params) + + context["cursor"].db = primary + context["cursor"].cursor = fallback_cursor.cursor + _fallback["succeeded"] = True + return None + + for attempt in range(1, max_attempts + 1): + router_token = None + yielded_cursor = False + + # On final attempt, fall back to primary + if attempt == max_attempts and can_failover: + if attempt > 1: + logger.warning( + f"RLS transaction failed after {attempt - 1} attempts on replica, " + f"falling back to primary DB" + ) + alias = DEFAULT_DB_ALIAS + + conn = connections[alias] + try: + if alias != DEFAULT_DB_ALIAS: + router_token = set_read_db_alias(alias) + + with transaction.atomic(using=alias): + with conn.cursor() as cursor: + try: + uuid.UUID(str(value)) + except ValueError: + raise ValidationError("Must be a valid UUID") + cursor.execute(SET_CONFIG_QUERY, [parameter, value]) + + wrapper_cm = ( + conn.execute_wrapper(_query_failover) + if can_failover and alias == replica_alias + else nullcontext() + ) + with wrapper_cm: + yielded_cursor = True + yield cursor + _fallback["caller_exited_cleanly"] = True + return + except OperationalError as e: + if yielded_cursor: + if _fallback["succeeded"] and _fallback["caller_exited_cleanly"]: + # Caller's queries succeeded on primary via failover. + # This error is transaction.atomic() cleanup on the + # dead replica connection, suppress it. + return + raise + + if not can_failover or attempt == max_attempts: + raise + + try: + connections[alias].close() + except Exception: + pass # Best-effort; connection may already be dead + + # Retry with exponential backoff + delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1)) + logger.info( + f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), " + f"retrying in {delay}s. Error: {e}" + ) + time.sleep(delay) + finally: + if _fallback["token"] is not None: + reset_read_db_alias(_fallback["token"]) + _fallback["token"] = None + + if router_token is not None: + reset_read_db_alias(router_token) class CustomUserManager(BaseUserManager): diff --git a/api/src/backend/api/tests/integration/test_rls_transaction.py b/api/src/backend/api/tests/integration/test_rls_transaction.py index bd46871586..ce026d4f58 100644 --- a/api/src/backend/api/tests/integration/test_rls_transaction.py +++ b/api/src/backend/api/tests/integration/test_rls_transaction.py @@ -1,8 +1,12 @@ """Tests for rls_transaction retry and fallback logic.""" +from unittest.mock import patch + import pytest -from api.db_utils import rls_transaction -from django.db import DEFAULT_DB_ALIAS +from api.db_utils import POSTGRES_TENANT_VAR, rls_transaction +from conftest import TEST_REPLICA_ALIAS +from django.db import DEFAULT_DB_ALIAS, OperationalError, connections +from psycopg2 import OperationalError as Psycopg2OperationalError from rest_framework_json_api.serializers import ValidationError @@ -36,3 +40,35 @@ class TestRLSTransaction: cursor.execute("SELECT current_setting(%s, true)", [custom_param]) result = cursor.fetchone() assert result == (str(tenant.id),) + + @pytest.mark.requires_test_replica_alias + @pytest.mark.django_db( + transaction=True, databases=[DEFAULT_DB_ALIAS, TEST_REPLICA_ALIAS] + ) + def test_mid_query_replica_connection_loss_falls_back_to_primary(self, tenant): + """Real Django connection state: closed replica atomic falls back to primary.""" + replica = connections[TEST_REPLICA_ALIAS] + sql = "SELECT current_setting(%s, true), %s" + params = [POSTGRES_TENANT_VAR, 42] + failed_once = {"value": False} + + def close_replica_and_raise(execute, sql_arg, params_arg, many, context): + if not failed_once["value"] and sql_arg == sql: + failed_once["value"] = True + replica.close() + try: + raise Psycopg2OperationalError("SSL SYSCALL error: EOF detected") + except Psycopg2OperationalError as psycopg_error: + raise OperationalError( + "SSL SYSCALL error: EOF detected" + ) from psycopg_error + return execute(sql_arg, params_arg, many, context) + + with patch("api.db_utils.READ_REPLICA_ALIAS", TEST_REPLICA_ALIAS): + with rls_transaction(str(tenant.id), using=TEST_REPLICA_ALIAS) as cursor: + with replica.execute_wrapper(close_replica_and_raise): + cursor.execute(sql, params) + result = cursor.fetchone() + + assert failed_once["value"] + assert result == (str(tenant.id), 42) diff --git a/api/src/backend/api/tests/test_db_utils.py b/api/src/backend/api/tests/test_db_utils.py index 06b528b44a..347be2b64d 100644 --- a/api/src/backend/api/tests/test_db_utils.py +++ b/api/src/backend/api/tests/test_db_utils.py @@ -1,11 +1,16 @@ +from contextlib import contextmanager from datetime import UTC, datetime from enum import Enum -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest from api.db_utils import ( POSTGRES_TENANT_VAR, + SET_CONFIG_QUERY, + SET_TRANSACTION_READ_ONLY_QUERY, PostgresEnumMigration, + _is_replica_connection_failure, + _is_safe_primary_replay, _should_create_index_on_partition, batch_delete, create_objects_in_batches, @@ -392,10 +397,23 @@ class TestRlsTransaction: with patch("api.db_utils.get_read_db_alias", return_value=None): with patch("api.db_utils.connections") as mock_connections: - mock_conn = MagicMock() - mock_cursor = MagicMock() - mock_conn.cursor.return_value.__enter__.return_value = mock_cursor - mock_connections.__getitem__.return_value = mock_conn + mock_replica_conn = MagicMock() + mock_replica_cursor = MagicMock() + mock_replica_conn.cursor.return_value.__enter__.return_value = ( + mock_replica_cursor + ) + mock_primary_conn = MagicMock() + mock_primary_cursor = MagicMock() + mock_primary_conn.cursor.return_value.__enter__.return_value = ( + mock_primary_cursor + ) + + def connections_getitem(alias): + if alias == "replica": + return mock_replica_conn + return mock_primary_conn + + mock_connections.__getitem__.side_effect = connections_getitem mock_connections.__contains__.return_value = True with patch("api.db_utils.transaction.atomic"): @@ -525,7 +543,7 @@ class TestRlsTransaction: def atomic_side_effect(*args, **kwargs): nonlocal call_count call_count += 1 - if call_count < 3: + if call_count < 4: raise OperationalError("Connection error") return MagicMock( __enter__=MagicMock(return_value=None), @@ -544,10 +562,11 @@ class TestRlsTransaction: with rls_transaction(tenant_id): pass - assert mock_sleep.call_count == 2 + assert mock_sleep.call_count == 3 mock_sleep.assert_any_call(0.5) mock_sleep.assert_any_call(1.0) - assert mock_logger.info.call_count == 2 + mock_sleep.assert_any_call(2.0) + assert mock_logger.info.call_count == 3 def test_rls_transaction_operational_error_inside_context_no_retry( self, tenants_fixture, enable_read_replica @@ -578,11 +597,12 @@ class TestRlsTransaction: raise OperationalError("Conflict with recovery") mock_sleep.assert_not_called() + mock_conn.close.assert_not_called() - def test_rls_transaction_max_three_attempts_for_replica( + def test_rls_transaction_max_attempts_for_replica( self, tenants_fixture, enable_read_replica ): - """Test maximum 3 attempts for replica database.""" + """Test REPLICA_MAX_ATTEMPTS replica tries + 1 primary fallback.""" tenant = tenants_fixture[0] tenant_id = str(tenant.id) @@ -606,7 +626,11 @@ class TestRlsTransaction: with rls_transaction(tenant_id): pass - assert mock_atomic.call_count == 3 + assert mock_atomic.call_args_list[-1] == call( + using=DEFAULT_DB_ALIAS + ) + # 3 replica + 1 primary = 4 total + assert mock_atomic.call_count == 4 def test_rls_transaction_replica_no_retry_when_disabled( self, tenants_fixture, enable_read_replica @@ -617,10 +641,23 @@ class TestRlsTransaction: with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): with patch("api.db_utils.connections") as mock_connections: - mock_conn = MagicMock() - mock_cursor = MagicMock() - mock_conn.cursor.return_value.__enter__.return_value = mock_cursor - mock_connections.__getitem__.return_value = mock_conn + mock_replica_conn = MagicMock() + mock_replica_cursor = MagicMock() + mock_replica_conn.cursor.return_value.__enter__.return_value = ( + mock_replica_cursor + ) + mock_primary_conn = MagicMock() + mock_primary_cursor = MagicMock() + mock_primary_conn.cursor.return_value.__enter__.return_value = ( + mock_primary_cursor + ) + + def connections_getitem(alias): + if alias == "replica": + return mock_replica_conn + return mock_primary_conn + + mock_connections.__getitem__.side_effect = connections_getitem mock_connections.__contains__.return_value = True with patch("api.db_utils.transaction.atomic") as mock_atomic: @@ -682,7 +719,7 @@ class TestRlsTransaction: def atomic_side_effect(*args, **kwargs): nonlocal call_count call_count += 1 - if call_count < 3: + if call_count < 4: raise OperationalError("Replica error") return MagicMock( __enter__=MagicMock(return_value=None), @@ -691,7 +728,7 @@ class TestRlsTransaction: with patch( "api.db_utils.transaction.atomic", side_effect=atomic_side_effect - ): + ) as mock_atomic: with patch("api.db_utils.time.sleep"): with patch( "api.db_utils.set_read_db_alias", return_value="token" @@ -701,6 +738,9 @@ class TestRlsTransaction: with rls_transaction(tenant_id): pass + assert mock_atomic.call_args_list[-1] == call( + using=DEFAULT_DB_ALIAS + ) mock_logger.warning.assert_called_once() warning_msg = mock_logger.warning.call_args[0][0] assert "falling back to primary DB" in warning_msg @@ -725,7 +765,7 @@ class TestRlsTransaction: def atomic_side_effect(*args, **kwargs): nonlocal call_count call_count += 1 - if call_count < 3: + if call_count < 4: raise OperationalError("Replica error") return MagicMock( __enter__=MagicMock(return_value=None), @@ -744,7 +784,7 @@ class TestRlsTransaction: with rls_transaction(tenant_id): pass - assert mock_logger.info.call_count == 2 + assert mock_logger.info.call_count == 3 assert mock_logger.warning.call_count == 1 def test_rls_transaction_operational_error_raised_immediately_on_primary( @@ -910,6 +950,520 @@ class TestRlsTransaction: result = cursor.fetchone() assert result[0] == 1 + # --- Mid-query failover tests --- + + class _FakeDatabaseError(Exception): + def __init__(self, message, pgcode=None): + super().__init__(message) + self.pgcode = pgcode + + def _install_execute_wrapper(self, connection): + connection.execute_wrappers = [] + + @contextmanager + def _execute_wrapper(fn): + connection.execute_wrappers.append(fn) + try: + yield + finally: + connection.execute_wrappers.remove(fn) + + connection.execute_wrapper = _execute_wrapper + + def _mock_replica_and_primary_connections(self, mock_connections): + mock_replica_conn = MagicMock() + self._install_execute_wrapper(mock_replica_conn) + mock_replica_cursor = MagicMock() + mock_replica_conn.cursor.return_value.__enter__.return_value = ( + mock_replica_cursor + ) + + mock_primary_conn = MagicMock() + mock_primary_cursor = MagicMock() + mock_primary_raw_cursor = MagicMock() + mock_primary_cursor.cursor = mock_primary_raw_cursor + mock_primary_conn.cursor.return_value = mock_primary_cursor + + def connections_getitem(alias): + if alias == "replica": + return mock_replica_conn + return mock_primary_conn + + mock_connections.__getitem__.side_effect = connections_getitem + mock_connections.__contains__.return_value = True + + return mock_replica_conn, mock_primary_conn, mock_primary_cursor + + @pytest.mark.parametrize( + "error", + [ + _FakeDatabaseError("connection lost", pgcode="08006"), + _FakeDatabaseError("terminating connection", pgcode="57P01"), + OperationalError("SSL SYSCALL error: EOF detected"), + OperationalError("server closed the connection unexpectedly"), + OperationalError("database system is starting up"), + ], + ) + def test_replica_connection_failure_detection_allows_failover(self, error): + assert _is_replica_connection_failure(error) + + @pytest.mark.parametrize( + "error", + [ + _FakeDatabaseError("canceling statement", pgcode="57014"), + _FakeDatabaseError("could not serialize access", pgcode="40001"), + _FakeDatabaseError("deadlock detected", pgcode="40P01"), + OperationalError("deadlock detected"), + ], + ) + def test_replica_connection_failure_detection_rejects_query_errors(self, error): + assert not _is_replica_connection_failure(error) + + @pytest.mark.parametrize( + ("sql", "many", "expected"), + [ + ("SELECT 1", False, True), + (" -- leading comment\nSELECT 1", False, True), + ("/* leading comment */ SELECT 1", False, True), + ("SELECT 1", True, False), + ("SELECTING 1", False, False), + ("INSERT INTO fake_table (name) VALUES (%s)", False, False), + ("WITH rows AS (SELECT 1) SELECT * FROM rows", False, False), + ("SELECT * INTO fake_table_copy FROM fake_table", False, False), + ("SELECT * FROM fake_table FOR UPDATE", False, False), + ("SELECT * FROM fake_table FOR SHARE", False, False), + ], + ) + def test_primary_replay_safety_detection(self, sql, many, expected): + assert _is_safe_primary_replay(sql, many) is expected + + def test_mid_query_failure_falls_directly_back_to_primary( + self, tenants_fixture, enable_read_replica + ): + """Mid-query replica connection loss is replayed once on primary.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + ( + mock_replica_conn, + mock_primary_conn, + mock_primary_cursor, + ) = self._mock_replica_and_primary_connections(mock_connections) + + outer_atomic = MagicMock() + outer_atomic.__enter__ = MagicMock(return_value=None) + outer_atomic.__exit__ = MagicMock(return_value=False) + fallback_atomic = MagicMock() + fallback_atomic.__enter__ = MagicMock(return_value=None) + fallback_atomic.__exit__ = MagicMock(return_value=False) + + with patch( + "api.db_utils.transaction.atomic", + side_effect=[outer_atomic, fallback_atomic], + ) as mock_atomic: + with patch("api.db_utils.time.sleep") as mock_sleep: + with patch( + "api.db_utils.set_read_db_alias", + side_effect=["replica-token", "primary-token"], + ) as mock_set_alias: + with patch( + "api.db_utils.reset_read_db_alias" + ) as mock_reset_alias: + with rls_transaction(tenant_id): + wrapper = mock_replica_conn.execute_wrappers[0] + context_cursor = MagicMock() + mock_execute = MagicMock( + side_effect=OperationalError( + "SSL SYSCALL error: EOF detected" + ) + ) + + wrapper( + mock_execute, + "SELECT %s", + ["value"], + False, + {"cursor": context_cursor}, + ) + + mock_sleep.assert_not_called() + ( + mock_replica_conn.ensure_connection.assert_not_called() + ) + mock_replica_conn.close.assert_called_once() + ( + mock_primary_conn.ensure_connection.assert_called_once() + ) + mock_primary_conn.cursor.assert_called_once_with() + mock_primary_cursor.execute.assert_has_calls( + [ + call(SET_TRANSACTION_READ_ONLY_QUERY), + call( + SET_CONFIG_QUERY, + [POSTGRES_TENANT_VAR, tenant_id], + ), + call("SELECT %s", ["value"]), + ] + ) + assert context_cursor.db == mock_primary_conn + assert ( + context_cursor.cursor + == mock_primary_cursor.cursor + ) + + mock_set_alias.assert_has_calls( + [ + call(enable_read_replica), + call(DEFAULT_DB_ALIAS), + ] + ) + mock_reset_alias.assert_has_calls( + [call("primary-token"), call("replica-token")] + ) + assert mock_atomic.call_args_list == [ + call(using=enable_read_replica), + call(using=DEFAULT_DB_ALIAS), + ] + assert mock_replica_conn.execute_wrappers == [] + + @pytest.mark.parametrize( + ("sql", "params", "many"), + [ + ("INSERT INTO fake_table (name) VALUES (%s)", [("one",), ("two",)], True), + ("INSERT INTO fake_table (name) VALUES (%s)", ["one"], False), + ("UPDATE fake_table SET name = %s", ["one"], False), + ("DELETE FROM fake_table WHERE id = %s", [1], False), + ( + "WITH deleted AS (DELETE FROM fake_table RETURNING *) " + "SELECT * FROM deleted", + None, + False, + ), + ("SELECT * INTO fake_table_copy FROM fake_table", None, False), + ], + ) + def test_mid_query_fallback_rejects_unsafe_replay( + self, tenants_fixture, enable_read_replica, sql, params, many + ): + """Only single SELECT statements are replayed on primary.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + ( + mock_replica_conn, + mock_primary_conn, + mock_primary_cursor, + ) = self._mock_replica_and_primary_connections(mock_connections) + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.return_value.__enter__ = MagicMock(return_value=None) + mock_atomic.return_value.__exit__ = MagicMock(return_value=False) + with patch( + "api.db_utils.set_read_db_alias", + side_effect=["replica-token", "primary-token"], + ): + with patch("api.db_utils.reset_read_db_alias"): + with rls_transaction(tenant_id): + wrapper = mock_replica_conn.execute_wrappers[0] + mock_execute = MagicMock( + side_effect=OperationalError( + "server closed the connection" + ) + ) + + with pytest.raises(OperationalError): + wrapper( + mock_execute, + sql, + params, + many, + {"cursor": MagicMock()}, + ) + + mock_primary_conn.ensure_connection.assert_not_called() + mock_primary_conn.cursor.assert_not_called() + mock_primary_cursor.execute.assert_not_called() + mock_primary_cursor.executemany.assert_not_called() + + def test_mid_query_non_connection_error_does_not_fall_back( + self, tenants_fixture, enable_read_replica + ): + """Query/concurrency errors are not replayed on primary.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + ( + mock_replica_conn, + mock_primary_conn, + _mock_primary_cursor, + ) = self._mock_replica_and_primary_connections(mock_connections) + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.return_value.__enter__ = MagicMock(return_value=None) + mock_atomic.return_value.__exit__ = MagicMock(return_value=False) + with patch( + "api.db_utils.set_read_db_alias", return_value="replica-token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with rls_transaction(tenant_id): + wrapper = mock_replica_conn.execute_wrappers[0] + mock_execute = MagicMock( + side_effect=OperationalError("deadlock detected") + ) + + with pytest.raises(OperationalError): + wrapper( + mock_execute, + "SELECT 1", + None, + False, + {"cursor": MagicMock()}, + ) + + mock_replica_conn.close.assert_not_called() + ( + mock_primary_conn.ensure_connection.assert_not_called() + ) + + def test_mid_query_primary_replay_failure_propagates( + self, tenants_fixture, enable_read_replica + ): + """Primary fallback errors propagate as Django OperationalError.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + ( + mock_replica_conn, + _mock_primary_conn, + mock_primary_cursor, + ) = self._mock_replica_and_primary_connections(mock_connections) + mock_primary_cursor.execute.side_effect = [ + None, + None, + OperationalError("primary down"), + ] + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.return_value.__enter__ = MagicMock(return_value=None) + mock_atomic.return_value.__exit__ = MagicMock(return_value=False) + with patch( + "api.db_utils.set_read_db_alias", + side_effect=["replica-token", "primary-token"], + ): + with patch("api.db_utils.reset_read_db_alias"): + with pytest.raises(OperationalError, match="primary down"): + with rls_transaction(tenant_id): + wrapper = mock_replica_conn.execute_wrappers[0] + mock_execute = MagicMock( + side_effect=OperationalError( + "server closed the connection" + ) + ) + wrapper( + mock_execute, + "SELECT 1", + None, + False, + {"cursor": MagicMock()}, + ) + + mock_primary_cursor.close.assert_called_once() + + def test_mid_query_fallback_suppresses_cleanup_error( + self, tenants_fixture, enable_read_replica + ): + """After successful primary fallback, replica cleanup error is suppressed.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + ( + mock_replica_conn, + _mock_primary_conn, + mock_primary_cursor, + ) = self._mock_replica_and_primary_connections(mock_connections) + + # Replica's atomic.__exit__ raises on dead replica cleanup; + # primary's atomic.__exit__ returns False (healthy commit). + mock_outer_atomic = MagicMock() + mock_outer_atomic.__enter__ = MagicMock(return_value=None) + mock_outer_atomic.__exit__ = MagicMock( + side_effect=OperationalError("cleanup failed on dead replica") + ) + + mock_fallback_atomic = MagicMock() + mock_fallback_atomic.__enter__ = MagicMock(return_value=None) + mock_fallback_atomic.__exit__ = MagicMock(return_value=False) + + atomic_call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal atomic_call_count + atomic_call_count += 1 + if atomic_call_count == 1: + return mock_outer_atomic + return mock_fallback_atomic + + with patch( + "api.db_utils.transaction.atomic", + side_effect=atomic_side_effect, + ): + with patch( + "api.db_utils.set_read_db_alias", + side_effect=["replica-token", "primary-token"], + ): + with patch("api.db_utils.reset_read_db_alias"): + with rls_transaction(tenant_id): + wrapper = mock_replica_conn.execute_wrappers[0] + mock_execute = MagicMock( + side_effect=OperationalError( + "server closed the connection" + ) + ) + mock_context = {"cursor": MagicMock()} + wrapper( + mock_execute, + "SELECT 1", + None, + False, + mock_context, + ) + + mock_primary_cursor.execute.assert_has_calls( + [ + call(SET_TRANSACTION_READ_ONLY_QUERY), + call( + SET_CONFIG_QUERY, + [POSTGRES_TENANT_VAR, tenant_id], + ), + call("SELECT 1", None), + ] + ) + + def test_wrapper_not_installed_on_primary(self, tenants_fixture): + """execute_wrapper is not installed when targeting primary DB.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_conn.execute_wrappers = [] + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.return_value.__enter__ = MagicMock(return_value=None) + mock_atomic.return_value.__exit__ = MagicMock(return_value=False) + + with rls_transaction(tenant_id): + # No wrapper installed on primary + assert len(mock_conn.execute_wrappers) == 0 + + def test_stale_connection_closed_on_pre_yield_retry( + self, tenants_fixture, enable_read_replica + ): + """Stale connection is closed before each pre-yield retry.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_conn.execute_wrappers = [] + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Connection error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with rls_transaction(tenant_id): + pass + + # close() called for each failed pre-yield attempt + assert mock_conn.close.call_count == 2 + + def test_caller_error_propagates_after_successful_failover( + self, tenants_fixture, enable_read_replica + ): + """OperationalError raised by caller after failover is NOT suppressed.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + ( + mock_replica_conn, + _mock_primary_conn, + _mock_primary_cursor, + ) = self._mock_replica_and_primary_connections(mock_connections) + + # Transaction cleanup succeeds so the caller error should surface. + mock_atomic_cm = MagicMock() + mock_atomic_cm.__enter__ = MagicMock(return_value=None) + mock_atomic_cm.__exit__ = MagicMock(return_value=False) + + with patch( + "api.db_utils.transaction.atomic", return_value=mock_atomic_cm + ): + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with pytest.raises( + OperationalError, match="caller error" + ): + with rls_transaction(tenant_id): + # Trigger failover (succeeds on primary) + wrapper = mock_replica_conn.execute_wrappers[0] + mock_execute = MagicMock( + side_effect=OperationalError( + "server closed the connection" + ) + ) + mock_context = {"cursor": MagicMock()} + wrapper( + mock_execute, + "SELECT 1", + None, + False, + mock_context, + ) + # Caller errors after successful failover + # should still propagate. + raise OperationalError("caller error") + class TestPostgresEnumMigration: """ diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index f4e398d76a..daa0616971 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -70,6 +70,7 @@ API_JSON_CONTENT_TYPE = "application/vnd.api+json" NO_TENANT_HTTP_STATUS = status.HTTP_401_UNAUTHORIZED TEST_USER = "dev@prowler.com" TEST_PASSWORD = "testing_psswd" +TEST_REPLICA_ALIAS = "test_replica" def _install_compliance_catalog_test_cache() -> None: @@ -2542,8 +2543,27 @@ def pytest_collection_modifyitems(items): """Ensure test_rbac.py is executed first.""" items.sort(key=lambda item: 0 if "test_rbac.py" in item.nodeid else 1) + if any(item.get_closest_marker("requires_test_replica_alias") for item in items): + default_database = settings.DATABASES["default"] + if TEST_REPLICA_ALIAS not in settings.DATABASES: + settings.DATABASES[TEST_REPLICA_ALIAS] = { + **default_database, + "TEST": { + **default_database.get("TEST", {}), + "MIRROR": "default", + }, + } + django_connections.databases[TEST_REPLICA_ALIAS] = settings.DATABASES[ + TEST_REPLICA_ALIAS + ] + def pytest_configure(config): + config.addinivalue_line( + "markers", + "requires_test_replica_alias: creates a test-only replica alias mirrored " + "to default", + ) # Apply the mock before the test session starts. This is necessary to avoid admin error when running the # 0004_rbac_missing_admin_roles migration patch("api.db_router.MainRouter.admin_db", new="default").start() From 1ee4de18be50ce41ca9d441f7d3d2238fdc231e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jand=C3=A9rik=20Marins?= <65249843+janderik@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:19:55 -0300 Subject: [PATCH 07/20] chore: add trailing newlines to 7 files for POSIX compliance (#11765) Co-authored-by: Janderik Marins Co-authored-by: Daniel Barranquero --- contrib/k8s/helm/prowler-app/templates/api/configmap.yaml | 2 +- contrib/k8s/helm/prowler-app/templates/api/role.yaml | 2 +- prowler/changelog.d/trailing-newlines.changed.md | 1 + prowler/compliance/aws/cis_7.0_aws.json | 2 +- prowler/compliance/m365/cis_7.0_m365.json | 2 +- prowler/providers/aws/aws_regions_by_service.json | 2 +- tests/providers/aws/aws_regions_by_service.json | 2 +- tests/providers/nhn/lib/mutelist/fixtures/nhn_mutelist.yaml | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 prowler/changelog.d/trailing-newlines.changed.md diff --git a/contrib/k8s/helm/prowler-app/templates/api/configmap.yaml b/contrib/k8s/helm/prowler-app/templates/api/configmap.yaml index 8e219a9271..a76982d403 100644 --- a/contrib/k8s/helm/prowler-app/templates/api/configmap.yaml +++ b/contrib/k8s/helm/prowler-app/templates/api/configmap.yaml @@ -7,4 +7,4 @@ metadata: data: {{- range $key, $value := .Values.api.djangoConfig }} {{ $key }}: {{ $value | quote }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/contrib/k8s/helm/prowler-app/templates/api/role.yaml b/contrib/k8s/helm/prowler-app/templates/api/role.yaml index 172b035076..f55d3c7dc9 100644 --- a/contrib/k8s/helm/prowler-app/templates/api/role.yaml +++ b/contrib/k8s/helm/prowler-app/templates/api/role.yaml @@ -26,4 +26,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "prowler.api.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} diff --git a/prowler/changelog.d/trailing-newlines.changed.md b/prowler/changelog.d/trailing-newlines.changed.md new file mode 100644 index 0000000000..4236f816b1 --- /dev/null +++ b/prowler/changelog.d/trailing-newlines.changed.md @@ -0,0 +1 @@ +Add missing trailing newlines to compliance, region, and fixture data files for POSIX compliance diff --git a/prowler/compliance/aws/cis_7.0_aws.json b/prowler/compliance/aws/cis_7.0_aws.json index f4f7fedff8..a927c375bb 100644 --- a/prowler/compliance/aws/cis_7.0_aws.json +++ b/prowler/compliance/aws/cis_7.0_aws.json @@ -1607,4 +1607,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/prowler/compliance/m365/cis_7.0_m365.json b/prowler/compliance/m365/cis_7.0_m365.json index a913f339be..941a33b187 100644 --- a/prowler/compliance/m365/cis_7.0_m365.json +++ b/prowler/compliance/m365/cis_7.0_m365.json @@ -3611,4 +3611,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/prowler/providers/aws/aws_regions_by_service.json b/prowler/providers/aws/aws_regions_by_service.json index 68e164b13e..1fa83ab593 100644 --- a/prowler/providers/aws/aws_regions_by_service.json +++ b/prowler/providers/aws/aws_regions_by_service.json @@ -13607,4 +13607,4 @@ } } } -} \ No newline at end of file +} diff --git a/tests/providers/aws/aws_regions_by_service.json b/tests/providers/aws/aws_regions_by_service.json index e7eb2e28f6..6d53a88d8c 100644 --- a/tests/providers/aws/aws_regions_by_service.json +++ b/tests/providers/aws/aws_regions_by_service.json @@ -5916,4 +5916,4 @@ } } } -} \ No newline at end of file +} diff --git a/tests/providers/nhn/lib/mutelist/fixtures/nhn_mutelist.yaml b/tests/providers/nhn/lib/mutelist/fixtures/nhn_mutelist.yaml index 6a4be42b4b..7fc22acedb 100644 --- a/tests/providers/nhn/lib/mutelist/fixtures/nhn_mutelist.yaml +++ b/tests/providers/nhn/lib/mutelist/fixtures/nhn_mutelist.yaml @@ -13,4 +13,4 @@ Mutelist: - "*" Resources: - "resource_1" - - "resource_2" \ No newline at end of file + - "resource_2" From 2a077cae5ef8e3d3329eb94f6bf7cdb8f3569738 Mon Sep 17 00:00:00 2001 From: "stepsecurity-app[bot]" <188008098+stepsecurity-app[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:00:40 +0200 Subject: [PATCH 08/20] feat(security): security best practices from StepSecurity (#11937) Signed-off-by: StepSecurity Bot Co-authored-by: stepsecurity-app[bot] <188008098+stepsecurity-app[bot]@users.noreply.github.com> Co-authored-by: Pepe Fagoaga --- .github/aw/actions-lock.json | 5 +++ .github/workflows/api-code-quality.yml | 2 +- .github/workflows/api-codeql.yml | 2 +- .../workflows/api-container-build-push.yml | 10 ++--- .github/workflows/api-container-checks.yml | 4 +- .github/workflows/api-security.yml | 2 +- .github/workflows/api-tests.yml | 2 +- .github/workflows/backport.yml | 2 +- .github/workflows/bump-version.yml | 8 ++-- .github/workflows/ci-zizmor.yml | 2 +- .github/workflows/comment-label-update.yml | 2 +- .github/workflows/compile-changelogs.yml | 2 +- .github/workflows/conventional-commit.yml | 2 +- .github/workflows/create-backport-label.yml | 2 +- .github/workflows/find-secrets.yml | 2 +- .github/workflows/helm-chart-checks.yml | 2 +- .github/workflows/helm-chart-release.yml | 2 +- .github/workflows/issue-lock-on-close.yml | 2 +- .github/workflows/issue-triage.lock.yml | 40 ++++++++++++++++++- .github/workflows/issue-triage.md | 6 +++ .github/workflows/labeler.yml | 4 +- .github/workflows/markdown-lint.yml | 2 +- .../workflows/mcp-container-build-push.yml | 10 ++--- .github/workflows/mcp-container-checks.yml | 4 +- .github/workflows/mcp-pypi-release.yml | 4 +- .github/workflows/mcp-security.yml | 2 +- .../nightly-arm64-container-builds.yml | 4 +- .github/workflows/pr-check-changelog.yml | 4 +- .../workflows/pr-check-compliance-mapping.yml | 2 +- .github/workflows/pr-conflict-checker.yml | 2 +- .github/workflows/pr-merged.yml | 2 +- .github/workflows/prepare-release.yml | 2 +- .../workflows/renovate-config-validate.yml | 2 +- .../sdk-check-duplicate-test-names.yml | 2 +- .github/workflows/sdk-code-quality.yml | 2 +- .github/workflows/sdk-codeql.yml | 2 +- .../workflows/sdk-container-build-push.yml | 10 ++--- .github/workflows/sdk-container-checks.yml | 4 +- .github/workflows/sdk-pypi-release.yml | 6 +-- .../sdk-refresh-aws-services-regions.yml | 2 +- .github/workflows/sdk-refresh-oci-regions.yml | 2 +- .github/workflows/sdk-security.yml | 2 +- .github/workflows/sdk-tests.yml | 2 +- .github/workflows/test-impact-analysis.yml | 2 +- .github/workflows/ui-codeql.yml | 2 +- .github/workflows/ui-container-build-push.yml | 10 ++--- .github/workflows/ui-container-checks.yml | 4 +- .github/workflows/ui-e2e-tests-v2.yml | 4 +- .github/workflows/ui-security.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- 50 files changed, 126 insertions(+), 79 deletions(-) diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 26ff95964c..cb7f0bb05d 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -14,6 +14,11 @@ "repo": "github/gh-aw/actions/setup", "version": "v0.43.23", "sha": "9382be3ca9ac18917e111a99d4e6bbff58d0dccc" + }, + "step-security/harden-runner@v2.20.0": { + "repo": "step-security/harden-runner", + "version": "v2.20.0", + "sha": "bf7454d06d71f1098171f2acdf0cd4708d7b5920" } } } diff --git a/.github/workflows/api-code-quality.yml b/.github/workflows/api-code-quality.yml index 93e8d1006d..7cb86206d2 100644 --- a/.github/workflows/api-code-quality.yml +++ b/.github/workflows/api-code-quality.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/api-codeql.yml b/.github/workflows/api-codeql.yml index f464cfcf96..18e001b269 100644 --- a/.github/workflows/api-codeql.yml +++ b/.github/workflows/api-codeql.yml @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/api-container-build-push.yml b/.github/workflows/api-container-build-push.yml index 196a6dfc50..79217455c1 100644 --- a/.github/workflows/api-container-build-push.yml +++ b/.github/workflows/api-container-build-push.yml @@ -46,7 +46,7 @@ jobs: contents: read steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block @@ -65,7 +65,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -108,7 +108,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -175,7 +175,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -236,7 +236,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/api-container-checks.yml b/.github/workflows/api-container-checks.yml index b6881f93f2..4798fd9df7 100644 --- a/.github/workflows/api-container-checks.yml +++ b/.github/workflows/api-container-checks.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -69,7 +69,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/api-security.yml b/.github/workflows/api-security.yml index d687ff0d30..aaa714531c 100644 --- a/.github/workflows/api-security.yml +++ b/.github/workflows/api-security.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 5473139414..1475e5344a 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -78,7 +78,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b1ea9ec7a2..8563f43038 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 079e3134b7..dfeeedfbd9 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -29,7 +29,7 @@ jobs: patch_version: ${{ steps.detect.outputs.patch_version }} steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -75,7 +75,7 @@ jobs: pull-requests: write steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -202,7 +202,7 @@ jobs: pull-requests: write steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -307,7 +307,7 @@ jobs: pull-requests: write steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/ci-zizmor.yml b/.github/workflows/ci-zizmor.yml index c0c68d21b9..b341326d50 100644 --- a/.github/workflows/ci-zizmor.yml +++ b/.github/workflows/ci-zizmor.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/comment-label-update.yml b/.github/workflows/comment-label-update.yml index 0af688b412..d8360c1e4c 100644 --- a/.github/workflows/comment-label-update.yml +++ b/.github/workflows/comment-label-update.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/compile-changelogs.yml b/.github/workflows/compile-changelogs.yml index b7e1b4fb93..2634f65a91 100644 --- a/.github/workflows/compile-changelogs.yml +++ b/.github/workflows/compile-changelogs.yml @@ -54,7 +54,7 @@ jobs: pull-requests: write steps: - name: Harden the runner (Block outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/conventional-commit.yml b/.github/workflows/conventional-commit.yml index 502881bc73..681f938f12 100644 --- a/.github/workflows/conventional-commit.yml +++ b/.github/workflows/conventional-commit.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/create-backport-label.yml b/.github/workflows/create-backport-label.yml index 4af9fefd5f..9ee73db542 100644 --- a/.github/workflows/create-backport-label.yml +++ b/.github/workflows/create-backport-label.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index ac3efaaa69..38cbfb6253 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: # We can't block as Trufflehog needs to verify secrets against vendors egress-policy: audit diff --git a/.github/workflows/helm-chart-checks.yml b/.github/workflows/helm-chart-checks.yml index 1691f21d35..f0c02a1494 100644 --- a/.github/workflows/helm-chart-checks.yml +++ b/.github/workflows/helm-chart-checks.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index ca179adeef..e51125d222 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/issue-lock-on-close.yml b/.github/workflows/issue-lock-on-close.yml index 3778c77d05..c5f53953ea 100644 --- a/.github/workflows/issue-lock-on-close.yml +++ b/.github/workflows/issue-lock-on-close.yml @@ -22,7 +22,7 @@ jobs: issues: write steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index 5b7d3aec64..d08e11c031 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"f3fa849cd645d1a648474caf20ea8291f35fb1b6801b8df2846b21487c789038","body_hash":"5a253c083c0c90f122591d8f2014dec00be2c10a75eff404e2a03bf1d7d60e36","compiler_version":"v0.81.6","agent_id":"copilot","engine_versions":{"copilot":"1.0.65"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"ba6380cc6e5be5d21677bebe04d52fb48e3abec7","version":"v0.81.6"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11","digest":"sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11","digest":"sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11","digest":"sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.30","digest":"sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.30@sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8237a612a8f5a8a197c2aab679f1177f5128eed8cb7cefbfdc0d635b57b91fa6","body_hash":"5a253c083c0c90f122591d8f2014dec00be2c10a75eff404e2a03bf1d7d60e36","compiler_version":"v0.81.6","agent_id":"copilot","engine_versions":{"copilot":"1.0.65"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"ba6380cc6e5be5d21677bebe04d52fb48e3abec7","version":"v0.81.6"},{"repo":"step-security/harden-runner","sha":"bf7454d06d71f1098171f2acdf0cd4708d7b5920","version":"v2.20.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11","digest":"sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11","digest":"sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11","digest":"sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.30","digest":"sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.30@sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]} # This file was automatically generated by gh-aw (v0.81.6). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -46,6 +46,7 @@ # - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 # - github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 +# - step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 # # Container images used: # - ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7 @@ -103,6 +104,11 @@ jobs: text: ${{ steps.sanitized.outputs.text }} title: ${{ steps.sanitized.outputs.title }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Setup Scripts id: setup uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 @@ -462,6 +468,11 @@ jobs: setup-trace-id: ${{ steps.setup.outputs.trace-id }} unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Setup Scripts id: setup uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 @@ -484,6 +495,11 @@ jobs: echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" } >> "$GITHUB_OUTPUT" + - name: Harden the runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -1129,6 +1145,11 @@ jobs: tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Setup Scripts id: setup uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 @@ -1369,6 +1390,11 @@ jobs: detection_reason: ${{ steps.detection_conclusion.outputs.reason }} detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Setup Scripts id: setup uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 @@ -1616,6 +1642,11 @@ jobs: setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Setup Scripts id: setup uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 @@ -1695,6 +1726,11 @@ jobs: process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Setup Scripts id: setup uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6 diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index d252d5493c..00b60e36aa 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -30,6 +30,12 @@ permissions: engine: copilot strict: false +pre-steps: + - name: Harden the runner + uses: step-security/harden-runner@v2.20.0 + with: + egress-policy: audit + imports: - ../agents/issue-triage.md diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 5d519b20d1..2281cb3b09 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 1a01e97762..7918c701e5 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/mcp-container-build-push.yml b/.github/workflows/mcp-container-build-push.yml index 6eeb5734cc..59994f8319 100644 --- a/.github/workflows/mcp-container-build-push.yml +++ b/.github/workflows/mcp-container-build-push.yml @@ -45,7 +45,7 @@ jobs: contents: read steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block @@ -64,7 +64,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -106,7 +106,7 @@ jobs: packages: write steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -165,7 +165,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -227,7 +227,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/mcp-container-checks.yml b/.github/workflows/mcp-container-checks.yml index 1f47c31189..94cb740457 100644 --- a/.github/workflows/mcp-container-checks.yml +++ b/.github/workflows/mcp-container-checks.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -68,7 +68,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/mcp-pypi-release.yml b/.github/workflows/mcp-pypi-release.yml index 124f67eb19..a6dae75017 100644 --- a/.github/workflows/mcp-pypi-release.yml +++ b/.github/workflows/mcp-pypi-release.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -67,7 +67,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/mcp-security.yml b/.github/workflows/mcp-security.yml index 4deb6a478d..271167dcee 100644 --- a/.github/workflows/mcp-security.yml +++ b/.github/workflows/mcp-security.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/nightly-arm64-container-builds.yml b/.github/workflows/nightly-arm64-container-builds.yml index 061ec61ff2..ba515d5898 100644 --- a/.github/workflows/nightly-arm64-container-builds.yml +++ b/.github/workflows/nightly-arm64-container-builds.yml @@ -48,7 +48,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -83,7 +83,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/pr-check-changelog.yml b/.github/workflows/pr-check-changelog.yml index 82f090d4be..4c54666956 100644 --- a/.github/workflows/pr-check-changelog.yml +++ b/.github/workflows/pr-check-changelog.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -85,7 +85,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/pr-check-compliance-mapping.yml b/.github/workflows/pr-check-compliance-mapping.yml index 4df61f49b0..1e42a5270a 100644 --- a/.github/workflows/pr-check-compliance-mapping.yml +++ b/.github/workflows/pr-check-compliance-mapping.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/pr-conflict-checker.yml b/.github/workflows/pr-conflict-checker.yml index a4fc2a4751..fafddf9840 100644 --- a/.github/workflows/pr-conflict-checker.yml +++ b/.github/workflows/pr-conflict-checker.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index fc88a69e08..e35bf558c8 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -26,7 +26,7 @@ jobs: contents: read steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index ca324aa006..35d5e95dee 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -29,7 +29,7 @@ jobs: pull-requests: write steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/renovate-config-validate.yml b/.github/workflows/renovate-config-validate.yml index af32eac074..8426efa6b4 100644 --- a/.github/workflows/renovate-config-validate.yml +++ b/.github/workflows/renovate-config-validate.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/sdk-check-duplicate-test-names.yml b/.github/workflows/sdk-check-duplicate-test-names.yml index 7c81e3ae3f..2bd673d660 100644 --- a/.github/workflows/sdk-check-duplicate-test-names.yml +++ b/.github/workflows/sdk-check-duplicate-test-names.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/sdk-code-quality.yml b/.github/workflows/sdk-code-quality.yml index 289508ce7d..e26133b640 100644 --- a/.github/workflows/sdk-code-quality.yml +++ b/.github/workflows/sdk-code-quality.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/sdk-codeql.yml b/.github/workflows/sdk-codeql.yml index a86d6dcc53..ab560b870b 100644 --- a/.github/workflows/sdk-codeql.yml +++ b/.github/workflows/sdk-codeql.yml @@ -53,7 +53,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/sdk-container-build-push.yml b/.github/workflows/sdk-container-build-push.yml index 282e78b1db..bbe85ab2c5 100644 --- a/.github/workflows/sdk-container-build-push.yml +++ b/.github/workflows/sdk-container-build-push.yml @@ -60,7 +60,7 @@ jobs: contents: read steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -98,7 +98,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -142,7 +142,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -215,7 +215,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -330,7 +330,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/sdk-container-checks.yml b/.github/workflows/sdk-container-checks.yml index b4821a2fe7..709a785974 100644 --- a/.github/workflows/sdk-container-checks.yml +++ b/.github/workflows/sdk-container-checks.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -71,7 +71,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/sdk-pypi-release.yml b/.github/workflows/sdk-pypi-release.yml index 06e3908be0..a61c9157e8 100644 --- a/.github/workflows/sdk-pypi-release.yml +++ b/.github/workflows/sdk-pypi-release.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -66,7 +66,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -102,7 +102,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/sdk-refresh-aws-services-regions.yml b/.github/workflows/sdk-refresh-aws-services-regions.yml index 41ec249a53..5a38858247 100644 --- a/.github/workflows/sdk-refresh-aws-services-regions.yml +++ b/.github/workflows/sdk-refresh-aws-services-regions.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/sdk-refresh-oci-regions.yml b/.github/workflows/sdk-refresh-oci-regions.yml index 65a36c7714..17b4205620 100644 --- a/.github/workflows/sdk-refresh-oci-regions.yml +++ b/.github/workflows/sdk-refresh-oci-regions.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/sdk-security.yml b/.github/workflows/sdk-security.yml index 192ef6279a..5f95d47af0 100644 --- a/.github/workflows/sdk-security.yml +++ b/.github/workflows/sdk-security.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index 3f312e8f0b..afbbcd72ba 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/test-impact-analysis.yml b/.github/workflows/test-impact-analysis.yml index ead669ae29..07bb920f66 100644 --- a/.github/workflows/test-impact-analysis.yml +++ b/.github/workflows/test-impact-analysis.yml @@ -52,7 +52,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/ui-codeql.yml b/.github/workflows/ui-codeql.yml index 41b1810591..d03786cc0f 100644 --- a/.github/workflows/ui-codeql.yml +++ b/.github/workflows/ui-codeql.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/ui-container-build-push.yml b/.github/workflows/ui-container-build-push.yml index 95214a233a..124db7b78b 100644 --- a/.github/workflows/ui-container-build-push.yml +++ b/.github/workflows/ui-container-build-push.yml @@ -45,7 +45,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -64,7 +64,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -107,7 +107,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -160,7 +160,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -222,7 +222,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/ui-container-checks.yml b/.github/workflows/ui-container-checks.yml index 96b3315bc0..14988ef203 100644 --- a/.github/workflows/ui-container-checks.yml +++ b/.github/workflows/ui-container-checks.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > @@ -69,7 +69,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/ui-e2e-tests-v2.yml b/.github/workflows/ui-e2e-tests-v2.yml index c4de8a9f88..98654699e0 100644 --- a/.github/workflows/ui-e2e-tests-v2.yml +++ b/.github/workflows/ui-e2e-tests-v2.yml @@ -96,7 +96,7 @@ jobs: steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -316,7 +316,7 @@ jobs: contents: read steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/ui-security.yml b/.github/workflows/ui-security.yml index 2dc444654f..cb7d64ee98 100644 --- a/.github/workflows/ui-security.yml +++ b/.github/workflows/ui-security.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 5e7e7517bb..18df9c4fa4 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: block allowed-endpoints: > From 892cc2bc075e93d91f186c7abb718cb4214077f9 Mon Sep 17 00:00:00 2001 From: "Pablo Fernandez Guerra (PFE)" <148432447+pfe-nazaries@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:26:33 +0200 Subject: [PATCH 09/20] refactor(ui): rename reserved billing flag to CLOUD_BILLING_ENABLED (#11920) Co-authored-by: Pablo F.G --- ui/lib/env.test.ts | 27 +++++++++++++++++++ ui/lib/env.ts | 15 ++++++++++- ui/lib/get-runtime-config.client.test.ts | 6 ++--- ui/lib/get-runtime-config.client.ts | 2 +- ui/lib/runtime-config.shared.ts | 4 +-- ui/lib/runtime-config.ts | 8 ++++-- .../runtime-config/runtime-config-page.ts | 2 +- ui/types/env.d.ts | 2 +- 8 files changed, 55 insertions(+), 11 deletions(-) diff --git a/ui/lib/env.test.ts b/ui/lib/env.test.ts index 3c696a8baf..637ace023d 100644 --- a/ui/lib/env.test.ts +++ b/ui/lib/env.test.ts @@ -62,6 +62,7 @@ describe("lib/env gated integration validation", () => { "POSTHOG_KEY", "UI_POSTHOG_HOST", "POSTHOG_HOST", + "CLOUD_BILLING_ENABLED", ] as const; beforeEach(() => { @@ -141,4 +142,30 @@ describe("lib/env gated integration validation", () => { await expect(import("@/lib/env")).rejects.toThrow("POSTHOG_HOST"); }); + + it("throws when CLOUD_BILLING_ENABLED is metronome but PostHog is not enabled", async () => { + // metronome billing routes per tenant via the BILLING_SYSTEM_METRONOME + // PostHog flag, so PostHog must be enabled. + vi.stubEnv("CLOUD_BILLING_ENABLED", "metronome"); + + await expect(import("@/lib/env")).rejects.toThrow( + "PostHog is required for per-tenant billing routing", + ); + }); + + it("resolves when CLOUD_BILLING_ENABLED is metronome and PostHog is enabled", async () => { + vi.stubEnv("CLOUD_BILLING_ENABLED", "metronome"); + vi.stubEnv("UI_POSTHOG_ENABLE", "true"); + vi.stubEnv("UI_POSTHOG_KEY", "phc_key"); + vi.stubEnv("UI_POSTHOG_HOST", "https://eu.i.posthog.com"); + + await expect(import("@/lib/env")).resolves.toBeDefined(); + }); + + it("resolves when CLOUD_BILLING_ENABLED is legacy without PostHog", async () => { + // legacy billing forces V1 and never touches PostHog. + vi.stubEnv("CLOUD_BILLING_ENABLED", "legacy"); + + await expect(import("@/lib/env")).resolves.toBeDefined(); + }); }); diff --git a/ui/lib/env.ts b/ui/lib/env.ts index e32828ebf2..1dbc6234ef 100644 --- a/ui/lib/env.ts +++ b/ui/lib/env.ts @@ -2,7 +2,7 @@ import { assertGatedIntegrations, warnGatedIntegrationsMisconfig, } from "@/lib/integrations"; -import { readEnv } from "@/lib/runtime-env"; +import { readBoolEnv, readEnv } from "@/lib/runtime-env"; // Boot-time required-env assertion so a misconfigured container fails fast // with a clear message. A key with a deprecated legacy name is satisfied by @@ -23,6 +23,19 @@ for (const { key, legacy } of REQUIRED) { } assertGatedIntegrations(); + +// `metronome` billing evaluates the BILLING_SYSTEM_METRONOME PostHog flag per +// tenant, so PostHog must be enabled or every tenant would be misrouted to the +// wrong billing system. Fail fast instead of degrading silently. +if ( + readEnv("CLOUD_BILLING_ENABLED") === "metronome" && + !readBoolEnv("UI_POSTHOG_ENABLE") +) { + throw new Error( + 'CLOUD_BILLING_ENABLED is "metronome" but UI_POSTHOG_ENABLE is not "true"; PostHog is required for per-tenant billing routing.', + ); +} + warnGatedIntegrationsMisconfig(); export {}; diff --git a/ui/lib/get-runtime-config.client.test.ts b/ui/lib/get-runtime-config.client.test.ts index d4c91cf7e3..d46b633d38 100644 --- a/ui/lib/get-runtime-config.client.test.ts +++ b/ui/lib/get-runtime-config.client.test.ts @@ -98,7 +98,7 @@ describe("getRuntimeConfigClient", () => { [ "apiBaseUrl", "apiDocsUrl", - "billingCloudEnable", + "cloudBillingEnabled", "googleTagManagerId", "posthogHost", "posthogKey", @@ -108,9 +108,9 @@ describe("getRuntimeConfigClient", () => { ].sort(), ); expect(config.apiBaseUrl).toBe("https://api.example.com/api/v1"); - // billingCloudEnable is a boolean flag, so it defaults to false (not null) + // cloudBillingEnabled is a boolean flag, so it defaults to false (not null) // when absent from the island. - expect(config.billingCloudEnable).toBe(false); + expect(config.cloudBillingEnabled).toBe(false); expect( (config as unknown as Record).notAllowlisted, ).toBeUndefined(); diff --git a/ui/lib/get-runtime-config.client.ts b/ui/lib/get-runtime-config.client.ts index 0ae7d94de8..9aa4ea15c8 100644 --- a/ui/lib/get-runtime-config.client.ts +++ b/ui/lib/get-runtime-config.client.ts @@ -20,7 +20,7 @@ const pickConfig = ( posthogKey: parsed.posthogKey ?? null, posthogHost: parsed.posthogHost ?? null, reoDevClientId: parsed.reoDevClientId ?? null, - billingCloudEnable: parsed.billingCloudEnable ?? false, + cloudBillingEnabled: parsed.cloudBillingEnabled ?? false, }); // Reads the island once (memoized); all-null during SSR or if it's diff --git a/ui/lib/runtime-config.shared.ts b/ui/lib/runtime-config.shared.ts index 22f53002a7..e4fbbeceed 100644 --- a/ui/lib/runtime-config.shared.ts +++ b/ui/lib/runtime-config.shared.ts @@ -9,7 +9,7 @@ export interface RuntimePublicConfig { posthogKey: string | null; // reserved posthogHost: string | null; // reserved reoDevClientId: string | null; // reserved - billingCloudEnable: boolean; + cloudBillingEnabled: boolean; } export const RUNTIME_CONFIG_SCRIPT_ID = "__PROWLER_RUNTIME_CONFIG__"; @@ -24,5 +24,5 @@ export const EMPTY_RUNTIME_PUBLIC_CONFIG: RuntimePublicConfig = { posthogKey: null, posthogHost: null, reoDevClientId: null, - billingCloudEnable: false, + cloudBillingEnabled: false, }; diff --git a/ui/lib/runtime-config.ts b/ui/lib/runtime-config.ts index 9c149355e5..eda71bc114 100644 --- a/ui/lib/runtime-config.ts +++ b/ui/lib/runtime-config.ts @@ -4,7 +4,7 @@ import { connection } from "next/server"; import { readGatedEnv } from "@/lib/integrations"; import type { RuntimePublicConfig } from "@/lib/runtime-config.shared"; -import { readBoolEnv, readEnv } from "@/lib/runtime-env"; +import { readEnv } from "@/lib/runtime-env"; // `connection()` forces a per-request runtime read (never build-snapshotted); // only this allowlist reaches the client. Each migrated key falls back to its @@ -44,6 +44,10 @@ export async function getRuntimePublicConfig(): Promise { "POSTHOG_HOST", ), reoDevClientId: readEnv("REO_DEV_CLIENT_ID"), - billingCloudEnable: readBoolEnv("BILLING_CLOUD_ENABLE"), + // Install-level selector "legacy" | "metronome" | "false"; the client only + // needs on/off, so expose a derived boolean (the raw selector is read + // server-side for V1/V2 routing). Default (unset) is off. + cloudBillingEnabled: + (readEnv("CLOUD_BILLING_ENABLED") ?? "false") !== "false", }; } diff --git a/ui/tests/runtime-config/runtime-config-page.ts b/ui/tests/runtime-config/runtime-config-page.ts index 41a415ed9a..dcdf3e546c 100644 --- a/ui/tests/runtime-config/runtime-config-page.ts +++ b/ui/tests/runtime-config/runtime-config-page.ts @@ -19,7 +19,7 @@ export const RUNTIME_CONFIG_KEYS = [ "posthogKey", "posthogHost", "reoDevClientId", - "billingCloudEnable", + "cloudBillingEnabled", ] as const satisfies ReadonlyArray; /** diff --git a/ui/types/env.d.ts b/ui/types/env.d.ts index 301a59802d..b07ed75286 100644 --- a/ui/types/env.d.ts +++ b/ui/types/env.d.ts @@ -28,7 +28,7 @@ declare global { NEXT_PUBLIC_SENTRY_ENVIRONMENT?: string; UI_SENTRY_ENVIRONMENT?: string; - BILLING_CLOUD_ENABLE?: "true" | "false"; + CLOUD_BILLING_ENABLED?: "legacy" | "metronome" | "false"; // Build-time public config NEXT_PUBLIC_IS_CLOUD_ENV?: "true" | "false"; From 08ee83c572423605f7896642ea200956a9e3c790 Mon Sep 17 00:00:00 2001 From: lydiavilchez <114735608+lydiavilchez@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:27:34 +0200 Subject: [PATCH 10/20] fix(docs): use python3 and add CI check for provider cards hook (#11930) --- .../workflows/docs-check-provider-cards.yml | 50 +++++++++++++++++++ .pre-commit-config.yaml | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs-check-provider-cards.yml diff --git a/.github/workflows/docs-check-provider-cards.yml b/.github/workflows/docs-check-provider-cards.yml new file mode 100644 index 0000000000..dcd0ff048f --- /dev/null +++ b/.github/workflows/docs-check-provider-cards.yml @@ -0,0 +1,50 @@ +name: 'Docs: Check Provider Cards Snippet' + +on: + pull_request: + branches: + - 'master' + - 'v5.*' + paths: + - 'docs/user-guide/providers/**/getting-started-*.mdx' + - 'docs/scripts/generate_provider_cards.py' + - 'docs/snippets/provider-cards.mdx' + - 'api/src/backend/api/models.py' + - '.github/workflows/docs-check-provider-cards.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + check-provider-cards: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Harden Runner + uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Verify provider cards snippet is up to date + run: | + if ! python3 docs/scripts/generate_provider_cards.py; then + echo "::error::docs/snippets/provider-cards.mdx is out of sync with the provider getting-started pages or the API ProviderChoices enum." + echo "Run 'python3 docs/scripts/generate_provider_cards.py' locally and commit the regenerated snippet." + echo "--- diff ---" + git diff docs/snippets/provider-cards.mdx + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f4caec382..31931b3e19 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -144,7 +144,7 @@ repos: - id: generate-provider-cards name: "Docs - regenerate provider cards snippet" - entry: python docs/scripts/generate_provider_cards.py + entry: python3 docs/scripts/generate_provider_cards.py language: system files: { glob: ["docs/user-guide/providers/**/getting-started-*.mdx", "docs/scripts/generate_provider_cards.py", "docs/snippets/provider-cards.mdx", "api/src/backend/api/models.py"] } pass_filenames: false From 847997672dee901ed8f8338cca75731d4f9389bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Arroba?= <19954079+cesararroba@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:02:34 +0200 Subject: [PATCH 11/20] fix(ui): reference renamed UI_POSTHOG_ENABLED flag in metronome billing guard (#11938) --- ui/changelog.d/metronome-posthog-flag.fixed.md | 1 + ui/lib/env.test.ts | 2 +- ui/lib/env.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 ui/changelog.d/metronome-posthog-flag.fixed.md diff --git a/ui/changelog.d/metronome-posthog-flag.fixed.md b/ui/changelog.d/metronome-posthog-flag.fixed.md new file mode 100644 index 0000000000..108cdd87d7 --- /dev/null +++ b/ui/changelog.d/metronome-posthog-flag.fixed.md @@ -0,0 +1 @@ +Fixed metronome billing failing to start when PostHog was enabled, caused by a stale reference to the renamed UI_POSTHOG_ENABLED flag diff --git a/ui/lib/env.test.ts b/ui/lib/env.test.ts index 637ace023d..eaeab7e92d 100644 --- a/ui/lib/env.test.ts +++ b/ui/lib/env.test.ts @@ -155,7 +155,7 @@ describe("lib/env gated integration validation", () => { it("resolves when CLOUD_BILLING_ENABLED is metronome and PostHog is enabled", async () => { vi.stubEnv("CLOUD_BILLING_ENABLED", "metronome"); - vi.stubEnv("UI_POSTHOG_ENABLE", "true"); + vi.stubEnv("UI_POSTHOG_ENABLED", "true"); vi.stubEnv("UI_POSTHOG_KEY", "phc_key"); vi.stubEnv("UI_POSTHOG_HOST", "https://eu.i.posthog.com"); diff --git a/ui/lib/env.ts b/ui/lib/env.ts index 1dbc6234ef..46b019d7df 100644 --- a/ui/lib/env.ts +++ b/ui/lib/env.ts @@ -29,10 +29,10 @@ assertGatedIntegrations(); // wrong billing system. Fail fast instead of degrading silently. if ( readEnv("CLOUD_BILLING_ENABLED") === "metronome" && - !readBoolEnv("UI_POSTHOG_ENABLE") + !readBoolEnv("UI_POSTHOG_ENABLED") ) { throw new Error( - 'CLOUD_BILLING_ENABLED is "metronome" but UI_POSTHOG_ENABLE is not "true"; PostHog is required for per-tenant billing routing.', + 'CLOUD_BILLING_ENABLED is "metronome" but UI_POSTHOG_ENABLED is not "true"; PostHog is required for per-tenant billing routing.', ); } From c93ea035fcc640677ef699b9b56ad12c454fecff Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 10 Jul 2026 11:19:08 +0200 Subject: [PATCH 12/20] fix(api): make AWS Attack Paths query aggregation Neo4j compatible (#11931) --- ...ws-attack-paths-neo4j-aggregation.fixed.md | 1 + .../backend/api/attack_paths/queries/aws.py | 39 ++++++++++++------- skills/prowler-attack-paths-query/SKILL.md | 4 +- 3 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 api/changelog.d/aws-attack-paths-neo4j-aggregation.fixed.md diff --git a/api/changelog.d/aws-attack-paths-neo4j-aggregation.fixed.md b/api/changelog.d/aws-attack-paths-neo4j-aggregation.fixed.md new file mode 100644 index 0000000000..17d53096da --- /dev/null +++ b/api/changelog.d/aws-attack-paths-neo4j-aggregation.fixed.md @@ -0,0 +1 @@ +AWS Attack Paths privilege escalation queries no longer fail on Neo4j with `Aggregation column contains implicit grouping expressions` diff --git a/api/src/backend/api/attack_paths/queries/aws.py b/api/src/backend/api/attack_paths/queries/aws.py index fa42854156..60660a1632 100644 --- a/api/src/backend/api/attack_paths/queries/aws.py +++ b/api/src/backend/api/attack_paths/queries/aws.py @@ -418,7 +418,8 @@ AWS_APPRUNNER_PRIVESC_UPDATE_SERVICE = AttackPathsQueryDefinition( // Find existing App Runner services with roles attached (potential targets) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'tasks.apprunner.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -523,7 +524,8 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition( // Find roles that trust the Bedrock AgentCore service (already attached to existing code interpreters) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -607,7 +609,8 @@ AWS_CLOUDFORMATION_PRIVESC_UPDATE_STACK = AttackPathsQueryDefinition( // Find roles that trust CloudFormation service (already attached to existing stacks) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -753,7 +756,8 @@ AWS_CLOUDFORMATION_PRIVESC_CHANGESET = AttackPathsQueryDefinition( // Find roles that trust CloudFormation service (already attached to existing stacks) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -844,7 +848,8 @@ AWS_CODEBUILD_PRIVESC_START_BUILD = AttackPathsQueryDefinition( // Find roles that trust CodeBuild service (already attached to existing projects) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -880,7 +885,8 @@ AWS_CODEBUILD_PRIVESC_START_BUILD_BATCH = AttackPathsQueryDefinition( // Find roles that trust CodeBuild service (already attached to existing projects) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -1096,7 +1102,8 @@ AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE = AttackPathsQueryDefinition( // Find EC2 instances with instance profiles (potential targets) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -1187,7 +1194,8 @@ AWS_EC2_PRIVESC_LAUNCH_TEMPLATE = AttackPathsQueryDefinition( // Find launch templates in the account (potential targets) MATCH path_target = (aws)--(template:LaunchTemplate) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -1223,7 +1231,8 @@ AWS_EC2INSTANCECONNECT_PRIVESC_SEND_SSH_PUBLIC_KEY = AttackPathsQueryDefinition( // Find EC2 instances with attached roles (targets for credential theft via IMDS) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -1539,7 +1548,8 @@ AWS_ECS_PRIVESC_EXECUTE_COMMAND = AttackPathsQueryDefinition( // Target: roles already attached to running tasks (trust ECS tasks service) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -1622,7 +1632,8 @@ AWS_GLUE_PRIVESC_UPDATE_DEV_ENDPOINT = AttackPathsQueryDefinition( // Find roles that trust Glue service (already attached to existing dev endpoints) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'glue.amazonaws.com'}}) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -3337,7 +3348,8 @@ AWS_SSM_PRIVESC_START_SESSION = AttackPathsQueryDefinition( // Find EC2 instances with attached roles (targets for credential theft via IMDS) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n @@ -3373,7 +3385,8 @@ AWS_SSM_PRIVESC_SEND_COMMAND = AttackPathsQueryDefinition( // Find EC2 instances with attached roles (targets for credential theft via IMDS) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - WITH principal_paths + collect(DISTINCT path_target) AS paths + WITH principal_paths, collect(DISTINCT path_target) AS target_paths + WITH principal_paths + target_paths AS paths UNWIND paths AS p UNWIND nodes(p) AS n diff --git a/skills/prowler-attack-paths-query/SKILL.md b/skills/prowler-attack-paths-query/SKILL.md index 9fedff4472..a70734aa6f 100644 --- a/skills/prowler-attack-paths-query/SKILL.md +++ b/skills/prowler-attack-paths-query/SKILL.md @@ -195,7 +195,8 @@ When all matching principals can target the same independent resource set, colle ```cypher WITH aws, collect(DISTINCT path_principal) AS principal_paths MATCH path_target = (aws)--(target) -WITH principal_paths + collect(DISTINCT path_target) AS paths +WITH principal_paths, collect(DISTINCT path_target) AS target_paths +WITH principal_paths + target_paths AS paths ``` Statements that constrain a target are still checked via `HAS_RESOURCE` traversals (`res`, `res2`). See IAM-015 or EC2-001 in `aws.py`. @@ -419,6 +420,7 @@ Queries must run on both Neo4j and Amazon Neptune. Avoid these constructs: | `FOREACH` | `WITH` + `UNWIND` + `SET` | | Regex `=~` | `toLower()` + exact match, or `STARTS WITH` / `CONTAINS` | | `CALL () { UNION }` | Multi-label `OR` in `WHERE` (see pattern above) | +| Carried value plus aggregate expression | Project the aggregate first: `WITH principal_paths, collect(...) AS target_paths`, then combine lists in the next `WITH` | | `any(x IN list ...)` | `size([x IN list WHERE pred]) > 0` | | `all(x IN list ...)` | `size([x IN list WHERE pred]) = size(list)` | | `none(x IN list ...)` | `size([x IN list WHERE pred]) = 0` | From 3f2e5929d9ff3c1a5d2fae49f8edd419ee99a8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Pe=C3=B1a?= Date: Fri, 10 Jul 2026 11:24:40 +0200 Subject: [PATCH 13/20] fix(api): harden Lighthouse provider base URLs (#11928) Co-authored-by: Josema Camacho --- .../lighthouse-provider-base-url.security.md | 1 + api/src/backend/api/tests/test_validators.py | 150 ++++++++++ api/src/backend/api/tests/test_views.py | 70 +++++ api/src/backend/api/v1/serializers.py | 76 +++-- api/src/backend/api/validators.py | 130 +++++++++ .../tasks/jobs/lighthouse_providers.py | 105 ++++++- api/src/backend/tasks/tests/test_tasks.py | 268 +++++++++++++++++- 7 files changed, 748 insertions(+), 52 deletions(-) create mode 100644 api/changelog.d/lighthouse-provider-base-url.security.md create mode 100644 api/src/backend/api/tests/test_validators.py diff --git a/api/changelog.d/lighthouse-provider-base-url.security.md b/api/changelog.d/lighthouse-provider-base-url.security.md new file mode 100644 index 0000000000..6e70e9fbdd --- /dev/null +++ b/api/changelog.d/lighthouse-provider-base-url.security.md @@ -0,0 +1 @@ +OpenAI-compatible Lighthouse provider base URLs are restricted before connection checks diff --git a/api/src/backend/api/tests/test_validators.py b/api/src/backend/api/tests/test_validators.py new file mode 100644 index 0000000000..2b6c384cf0 --- /dev/null +++ b/api/src/backend/api/tests/test_validators.py @@ -0,0 +1,150 @@ +import socket + +import pytest +from api.validators import validate_lighthouse_openai_compatible_base_url +from django.core.exceptions import ValidationError + + +def test_lighthouse_base_url_rejects_http_scheme(): + with pytest.raises(ValidationError, match="HTTPS"): + validate_lighthouse_openai_compatible_base_url( + "http://openrouter.ai/api/v1", + resolve_dns=False, + ) + + +@pytest.mark.parametrize( + "base_url", + [ + "https://openrouter.ai:0/api/v1", + "https://openrouter.ai:-1/api/v1", + "https://openrouter.ai:65536/api/v1", + "https://openrouter.ai:invalid/api/v1", + ], +) +def test_lighthouse_base_url_rejects_invalid_port(base_url): + with pytest.raises(ValidationError, match="port is invalid"): + validate_lighthouse_openai_compatible_base_url( + base_url, + resolve_dns=False, + ) + + +@pytest.mark.parametrize("port", [1, 65535]) +def test_lighthouse_base_url_accepts_valid_port_boundaries(port): + assert ( + validate_lighthouse_openai_compatible_base_url( + f"https://openrouter.ai:{port}/api/v1", + resolve_dns=False, + ) + is None + ) + + +def test_lighthouse_base_url_rejects_localhost(): + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + "https://localhost/v1", + resolve_dns=False, + ) + + +@pytest.mark.parametrize("ip_address", ["10.0.0.1", "172.16.0.1", "192.168.1.1"]) +def test_lighthouse_base_url_rejects_private_ip_literal(ip_address): + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + f"https://{ip_address}/v1", + resolve_dns=False, + ) + + +def test_lighthouse_base_url_rejects_metadata_ip_literal(): + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + "https://169.254.169.254/latest/meta-data", + resolve_dns=False, + ) + + +@pytest.mark.parametrize( + "base_url", + [ + "https://[::ffff:169.254.169.254]/v1", + "https://[64:ff9b::a9fe:a9fe]/v1", + "https://[2002:a9fe:a9fe::]/v1", + ], +) +def test_lighthouse_base_url_rejects_embedded_non_global_ip(base_url): + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + base_url, + resolve_dns=False, + ) + + +@pytest.mark.parametrize( + "base_url", + [ + "https://[::ffff:93.184.216.34]/v1", + "https://[64:ff9b::5db8:d822]/v1", + "https://[2002:5db8:d822::]/v1", + ], +) +def test_lighthouse_base_url_accepts_embedded_public_ip(base_url): + assert ( + validate_lighthouse_openai_compatible_base_url( + base_url, + resolve_dns=False, + ) + is None + ) + + +def test_lighthouse_base_url_accepts_hostname_without_dns_resolution(): + assert ( + validate_lighthouse_openai_compatible_base_url( + "https://openrouter.ai/api/v1", + resolve_dns=False, + ) + is None + ) + + +def test_lighthouse_base_url_rejects_post_dns_internal_address(monkeypatch): + def resolve_to_metadata(*_args, **_kwargs): + return [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + 6, + "", + ("169.254.169.254", 443), + ) + ] + + monkeypatch.setattr("api.validators.socket.getaddrinfo", resolve_to_metadata) + + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + "https://metadata.example.test/v1" + ) + + +def test_lighthouse_base_url_accepts_public_resolved_address(monkeypatch): + def resolve_to_public(*_args, **_kwargs): + return [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + 6, + "", + ("93.184.216.34", 443), + ) + ] + + monkeypatch.setattr("api.validators.socket.getaddrinfo", resolve_to_public) + + assert ( + validate_lighthouse_openai_compatible_base_url("https://openrouter.ai/api/v1") + is None + ) diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 4fdb023955..be52393dfa 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -17308,6 +17308,76 @@ class TestLighthouseProviderConfigViewSet: error_detail = str(resp.json()).lower() assert "base_url" in error_detail + @pytest.mark.parametrize( + "base_url", + [ + "https://127.0.0.1/v1", + "https://169.254.169.254/latest/meta-data", + ], + ) + def test_openai_compatible_rejects_internal_base_url_on_create( + self, authenticated_client, base_url + ): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": base_url, + "credentials": {"api_key": "compat-key"}, + }, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert "base_url" in str(resp.json()).lower() + + def test_openai_compatible_rejects_internal_base_url_on_update( + self, authenticated_client + ): + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://openrouter.ai/api/v1", + "credentials": {"api_key": "compat-key-123"}, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "base_url": "https://169.254.169.254/latest/meta-data", + }, + } + } + + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + + assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST + assert "base_url" in str(patch_resp.json()).lower() + def test_openai_compatible_invalid_credentials(self, authenticated_client): payload = { "data": { diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index f650a1368a..0f08c8f4ba 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -57,6 +57,7 @@ from api.v1.serializer_utils.lighthouse import ( ) from api.v1.serializer_utils.processors import ProcessorConfigField from api.v1.serializer_utils.providers import ProviderSecretField +from api.validators import validate_lighthouse_openai_compatible_base_url from django.conf import settings from django.contrib.auth import authenticate from django.contrib.auth.models import update_last_login @@ -80,6 +81,21 @@ from rest_framework_simplejwt.utils import get_md5_hash_password # Base +def _validate_lighthouse_base_url_without_dns(base_url: str) -> None: + try: + validate_lighthouse_openai_compatible_base_url(base_url, resolve_dns=False) + except DjangoValidationError as error: + raise ValidationError({"base_url": error.messages[0]}) from error + + +def _reraise_lighthouse_credentials_errors(error: ValidationError) -> None: + details = error.detail.copy() + for key, value in details.items(): + error.detail[f"credentials/{key}"] = value + del error.detail[key] + raise error + + class BaseModelSerializerV1(serializers.ModelSerializer): def get_root_meta(self, _resource, _many): return {"version": "v1"} @@ -3645,11 +3661,7 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize raise_exception=True ) except ValidationError as e: - details = e.detail.copy() - for key, value in details.items(): - e.detail[f"credentials/{key}"] = value - del e.detail[key] - raise e + _reraise_lighthouse_credentials_errors(e) elif ( provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK ): @@ -3658,27 +3670,20 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize raise_exception=True ) except ValidationError as e: - details = e.detail.copy() - for key, value in details.items(): - e.detail[f"credentials/{key}"] = value - del e.detail[key] - raise e + _reraise_lighthouse_credentials_errors(e) elif ( provider_type == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE ): if not base_url: raise ValidationError({"base_url": "Base URL is required."}) + _validate_lighthouse_base_url_without_dns(base_url) try: OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( raise_exception=True ) except ValidationError as e: - details = e.detail.copy() - for key, value in details.items(): - e.detail[f"credentials/{key}"] = value - del e.detail[key] - raise e + _reraise_lighthouse_credentials_errors(e) return super().validate(attrs) @@ -3741,11 +3746,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): raise_exception=True ) except ValidationError as e: - details = e.detail.copy() - for key, value in details.items(): - e.detail[f"credentials/{key}"] = value - del e.detail[key] - raise e + _reraise_lighthouse_credentials_errors(e) elif ( credentials is not None and provider_type @@ -3769,11 +3770,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): raise_exception=True ) except ValidationError as e: - details = e.detail.copy() - for key, value in details.items(): - e.detail[f"credentials/{key}"] = value - del e.detail[key] - raise e + _reraise_lighthouse_credentials_errors(e) # Then enforce invariants about not changing the auth method # If the existing config uses an API key, forbid introducing access keys. @@ -3800,24 +3797,23 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): } ) elif ( - credentials is not None - and provider_type + provider_type == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE ): - if base_url is None: - pass - elif not base_url: + effective_base_url = ( + base_url if "base_url" in attrs else getattr(self.instance, "base_url") + ) + if not effective_base_url: raise ValidationError({"base_url": "Base URL cannot be empty."}) - try: - OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( - raise_exception=True - ) - except ValidationError as e: - details = e.detail.copy() - for key, value in details.items(): - e.detail[f"credentials/{key}"] = value - del e.detail[key] - raise e + if "base_url" in attrs: + _validate_lighthouse_base_url_without_dns(effective_base_url) + if credentials is not None: + try: + OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + _reraise_lighthouse_credentials_errors(e) return super().validate(attrs) diff --git a/api/src/backend/api/validators.py b/api/src/backend/api/validators.py index 543406f202..0bc45fb48b 100644 --- a/api/src/backend/api/validators.py +++ b/api/src/backend/api/validators.py @@ -1,14 +1,140 @@ +import ipaddress +import socket import string +from urllib.parse import urlparse from django.core.exceptions import ValidationError from django.utils.translation import gettext as _ +LIGHTHOUSE_OPENAI_COMPATIBLE_ALLOWED_SCHEMES = frozenset({"https"}) +LIGHTHOUSE_NAT64_WELL_KNOWN_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") +LIGHTHOUSE_BLOCKED_METADATA_HOSTS = frozenset( + { + "169.254.169.254", + "169.254.170.2", + "fd00:ec2::254", + "localhost", + "metadata.google.internal", + } +) + + +def _normalize_hostname(hostname: str) -> str: + return hostname.rstrip(".").lower() + + +def _validate_lighthouse_public_ip(address: str) -> None: + ip_address = ipaddress.ip_address(address) + if isinstance(ip_address, ipaddress.IPv6Address): + # Classify transition addresses by their effective IPv4 destination. + embedded_ip_address = ip_address.ipv4_mapped or ip_address.sixtofour + if ( + embedded_ip_address is None + and ip_address in LIGHTHOUSE_NAT64_WELL_KNOWN_PREFIX + ): + embedded_ip_address = ipaddress.IPv4Address(int(ip_address) & 0xFFFFFFFF) + if embedded_ip_address is not None: + ip_address = embedded_ip_address + if not ip_address.is_global: + raise ValidationError( + _("Base URL must use an external public endpoint."), + code="lighthouse_base_url_not_public", + ) + + +def resolve_lighthouse_openai_compatible_host( + hostname: str, + port: int, + *, + resolve_dns: bool = True, +) -> tuple[str, ...]: + """Return public IP addresses that are safe for Lighthouse outbound use.""" + hostname = _normalize_hostname(hostname) + if hostname in LIGHTHOUSE_BLOCKED_METADATA_HOSTS or hostname.endswith(".localhost"): + raise ValidationError( + _("Base URL must use an external public endpoint."), + code="lighthouse_base_url_blocked_host", + ) + + try: + _validate_lighthouse_public_ip(hostname) + except ValueError: + if not resolve_dns: + return () + else: + return (hostname,) + + try: + resolved_addresses = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror as error: + raise ValidationError( + _("Base URL host could not be resolved."), + code="lighthouse_base_url_resolution_failed", + ) from error + + if not resolved_addresses: + raise ValidationError( + _("Base URL host could not be resolved."), + code="lighthouse_base_url_resolution_failed", + ) + + public_addresses: list[str] = [] + for resolved_address in resolved_addresses: + socket_address = resolved_address[4] + resolved_ip_address = socket_address[0] + _validate_lighthouse_public_ip(resolved_ip_address) + if resolved_ip_address not in public_addresses: + public_addresses.append(resolved_ip_address) + + return tuple(public_addresses) + + +def validate_lighthouse_openai_compatible_base_url( + base_url: str, + *, + resolve_dns: bool = True, +) -> None: + """Validate an OpenAI-compatible Lighthouse base URL before outbound use.""" + parsed = urlparse(str(base_url)) + if parsed.scheme.lower() not in LIGHTHOUSE_OPENAI_COMPATIBLE_ALLOWED_SCHEMES: + raise ValidationError( + _("Base URL must use HTTPS."), + code="lighthouse_base_url_invalid_scheme", + ) + + if not parsed.hostname: + raise ValidationError( + _("Base URL must include a host."), + code="lighthouse_base_url_missing_host", + ) + + try: + port = parsed.port + except ValueError as error: + raise ValidationError( + _("Base URL port is invalid."), + code="lighthouse_base_url_invalid_port", + ) from error + + if port is not None and not 1 <= port <= 65535: + raise ValidationError( + _("Base URL port is invalid."), + code="lighthouse_base_url_invalid_port", + ) + + resolve_lighthouse_openai_compatible_host( + parsed.hostname, + port or 443, + resolve_dns=resolve_dns, + ) + class MaximumLengthValidator: def __init__(self, max_length=72): self.max_length = max_length def validate(self, password, user=None): + del user if len(password) > self.max_length: raise ValidationError( _( @@ -31,6 +157,7 @@ class SpecialCharactersValidator: self.min_special_characters = min_special_characters def validate(self, password, user=None): + del user if ( sum(1 for char in password if char in self.special_characters) < self.min_special_characters @@ -55,6 +182,7 @@ class UppercaseValidator: self.min_uppercase = min_uppercase def validate(self, password, user=None): + del user if sum(1 for char in password if char.isupper()) < self.min_uppercase: raise ValidationError( _( @@ -75,6 +203,7 @@ class LowercaseValidator: self.min_lowercase = min_lowercase def validate(self, password, user=None): + del user if sum(1 for char in password if char.islower()) < self.min_lowercase: raise ValidationError( _( @@ -95,6 +224,7 @@ class NumericValidator: self.min_numeric = min_numeric def validate(self, password, user=None): + del user if sum(1 for char in password if char.isdigit()) < self.min_numeric: raise ValidationError( _( diff --git a/api/src/backend/tasks/jobs/lighthouse_providers.py b/api/src/backend/tasks/jobs/lighthouse_providers.py index 0f28725e01..1eec1d3c6a 100644 --- a/api/src/backend/tasks/jobs/lighthouse_providers.py +++ b/api/src/backend/tasks/jobs/lighthouse_providers.py @@ -1,6 +1,15 @@ +import ssl +from collections.abc import Iterable + import boto3 +import httpcore +import httpx import openai from api.models import LighthouseProviderConfiguration, LighthouseProviderModels +from api.validators import ( + resolve_lighthouse_openai_compatible_host, + validate_lighthouse_openai_compatible_base_url, +) from botocore import UNSIGNED from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError @@ -43,6 +52,90 @@ EXCLUDED_OPENAI_MODEL_SUBSTRINGS = ( "-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.) ) +OPENAI_COMPATIBLE_AUTHENTICATION_ERROR = "API key is invalid or missing" +OPENAI_COMPATIBLE_CONNECTION_ERROR = "Provider connection failed" + + +class _OpenAICompatibleProviderError(Exception): + """Sanitized OpenAI-compatible provider error safe for task results.""" + + +def _sanitize_openai_compatible_error(error: Exception) -> str: + status_code = getattr(error, "status_code", None) + if status_code is None: + response = getattr(error, "response", None) + status_code = getattr(response, "status_code", None) + + if status_code == 401: + return OPENAI_COMPATIBLE_AUTHENTICATION_ERROR + return OPENAI_COMPATIBLE_CONNECTION_ERROR + + +class _LighthouseOpenAICompatibleNetworkBackend(httpcore.SyncBackend): + """Validate and pin DNS results immediately before TCP connections.""" + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None, + ) -> httpcore.NetworkStream: + resolved_addresses = resolve_lighthouse_openai_compatible_host(host, port) + last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None + + for address in resolved_addresses: + try: + return super().connect_tcp( + address, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as error: + last_error = error + + if last_error: + raise last_error + raise httpcore.ConnectError("No resolved addresses are available") + + +class _LighthouseOpenAICompatibleHTTPTransport(httpx.HTTPTransport): + """HTTP transport that connects only to validated public IP addresses.""" + + def __init__(self) -> None: + self._pool = httpcore.ConnectionPool( + ssl_context=ssl.create_default_context(), + network_backend=_LighthouseOpenAICompatibleNetworkBackend(), + ) + + +def _create_openai_compatible_http_client() -> httpx.Client: + """Create the restricted HTTP client used for OpenAI-compatible providers.""" + return httpx.Client( + follow_redirects=False, + trust_env=False, + transport=_LighthouseOpenAICompatibleHTTPTransport(), + ) + + +def _list_openai_compatible_models(base_url: str, api_key: str): + validate_lighthouse_openai_compatible_base_url(base_url) + try: + with _create_openai_compatible_http_client() as http_client: + client = openai.OpenAI( + api_key=api_key, + base_url=base_url, + http_client=http_client, + ) + return client.models.list() + except Exception as error: + raise _OpenAICompatibleProviderError( + _sanitize_openai_compatible_error(error) + ) from error + def _extract_error_message(e: Exception) -> str: """ @@ -114,6 +207,7 @@ def _extract_openai_compatible_params( return None if not isinstance(base_url, str) or not base_url: return None + validate_lighthouse_openai_compatible_base_url(base_url, resolve_dns=False) return {"base_url": base_url, "api_key": api_key} @@ -285,13 +379,7 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> dict: "error": "Base URL or API key is invalid or missing", } - # Test connection using OpenAI SDK with custom base_url - # Note: base_url should include version (e.g., https://openrouter.ai/api/v1) - client = openai.OpenAI( - api_key=params["api_key"], - base_url=params["base_url"], - ) - _ = client.models.list() + _ = _list_openai_compatible_models(params["base_url"], params["api_key"]) else: return {"connected": False, "error": "Unsupported provider type"} @@ -361,8 +449,7 @@ def _fetch_openai_compatible_models(base_url: str, api_key: str) -> dict[str, st Note: base_url should include version (e.g., https://openrouter.ai/api/v1) """ - client = openai.OpenAI(api_key=api_key, base_url=base_url) - models = client.models.list() + models = _list_openai_compatible_models(base_url, api_key) available_models: dict[str, str] = {} for model in models.data: diff --git a/api/src/backend/tasks/tests/test_tasks.py b/api/src/backend/tasks/tests/test_tasks.py index 631b91bf6b..6622ab10a5 100644 --- a/api/src/backend/tasks/tests/test_tasks.py +++ b/api/src/backend/tasks/tests/test_tasks.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from datetime import UTC, datetime, timedelta from unittest.mock import MagicMock, patch +import httpx import openai import pytest from api.models import ( @@ -20,6 +21,7 @@ from django_celery_results.models import TaskResult from tasks.jobs.lighthouse_providers import ( _create_bedrock_client, _extract_bedrock_credentials, + _LighthouseOpenAICompatibleNetworkBackend, ) from tasks.tasks import ( DJANGO_TMP_OUTPUT_DIRECTORY, @@ -1566,7 +1568,7 @@ class TestCheckLighthouseProviderConnectionTask: ( LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, {"api_key": "sk-test123"}, - "https://openrouter.ai/api/v1", + "https://93.184.216.34/api/v1", {"connected": True, "error": None}, ), ( @@ -1641,7 +1643,7 @@ class TestCheckLighthouseProviderConnectionTask: ( LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, {"api_key": "sk-invalid"}, - "https://openrouter.ai/api/v1", + "https://93.184.216.34/api/v1", openai.APIConnectionError(request=MagicMock()), ), ( @@ -1755,6 +1757,166 @@ class TestCheckLighthouseProviderConnectionTask: provider_cfg.refresh_from_db() assert provider_cfg.is_active is False + def test_openai_compatible_connection_rejects_metadata_base_url_without_request( + self, tenants_fixture + ): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://169.254.169.254/latest/meta-data", + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + eager_result = check_lighthouse_provider_connection_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result["connected"] is False + assert "base url" in result["error"].lower() + mock_openai.assert_not_called() + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_openai_compatible_connection_disables_redirects(self, tenants_fixture): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://93.184.216.34/api/v1", + is_active=False, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.return_value = MagicMock() + mock_openai.return_value = mock_client + + eager_result = check_lighthouse_provider_connection_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result == {"connected": True, "error": None} + http_client = mock_openai.call_args.kwargs["http_client"] + assert http_client.follow_redirects is False + assert http_client.trust_env is False + + def test_openai_compatible_connection_masks_remote_http_error( + self, tenants_fixture + ): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://93.184.216.34/api/v1", + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + remote_body = "

remote 404 body

" + response = httpx.Response( + 404, + request=httpx.Request("GET", "https://provider.example/v1/models"), + ) + + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.side_effect = openai.NotFoundError( + remote_body, + response=response, + body=remote_body, + ) + mock_openai.return_value = mock_client + + eager_result = check_lighthouse_provider_connection_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result == {"connected": False, "error": "Provider connection failed"} + assert remote_body not in result["error"] + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_openai_compatible_connection_masks_remote_auth_error( + self, tenants_fixture + ): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://93.184.216.34/api/v1", + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + remote_body = {"error": {"message": "remote auth detail"}} + response = httpx.Response( + 401, + request=httpx.Request("GET", "https://provider.example/v1/models"), + ) + + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.side_effect = openai.AuthenticationError( + "Unauthorized", + response=response, + body=remote_body, + ) + mock_openai.return_value = mock_client + + eager_result = check_lighthouse_provider_connection_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result == {"connected": False, "error": "API key is invalid or missing"} + assert "remote auth detail" not in result["error"] + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_openai_compatible_network_backend_uses_validated_ip(self, monkeypatch): + backend = _LighthouseOpenAICompatibleNetworkBackend() + stream = MagicMock() + + def resolve_to_public_ip(host, port): + del host, port + return ("93.184.216.34",) + + monkeypatch.setattr( + "tasks.jobs.lighthouse_providers.resolve_lighthouse_openai_compatible_host", + resolve_to_public_ip, + ) + + with patch( + "tasks.jobs.lighthouse_providers.httpcore.SyncBackend.connect_tcp", + return_value=stream, + ) as mock_connect_tcp: + result = backend.connect_tcp("provider.example", 443, timeout=1.0) + + assert result is stream + assert mock_connect_tcp.call_args.args[:2] == ("93.184.216.34", 443) + assert mock_connect_tcp.call_args.kwargs["timeout"] == 1.0 + def test_check_connection_provider_does_not_exist(self, tenants_fixture): """Test that checking non-existent provider raises DoesNotExist.""" non_existent_id = str(uuid.uuid4()) @@ -1784,7 +1946,7 @@ class TestRefreshLighthouseProviderModelsTask: ( LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, {"api_key": "sk-test123"}, - "https://openrouter.ai/api/v1", + "https://93.184.216.34/api/v1", {"model-1": "Model One", "model-2": "Model Two"}, 2, ), @@ -1864,6 +2026,106 @@ class TestRefreshLighthouseProviderModelsTask: == expected_count ) + def test_refresh_models_rejects_metadata_base_url_without_request( + self, tenants_fixture + ): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://169.254.169.254/latest/meta-data", + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + + with patch( + "tasks.jobs.lighthouse_providers._fetch_openai_compatible_models" + ) as mock_fetch: + eager_result = refresh_lighthouse_provider_models_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result["created"] == 0 + assert result["updated"] == 0 + assert result["deleted"] == 0 + assert "base url" in result["error"].lower() + mock_fetch.assert_not_called() + + def test_refresh_models_disables_redirects(self, tenants_fixture): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://93.184.216.34/api/v1", + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.return_value = MagicMock(data=[]) + mock_openai.return_value = mock_client + + eager_result = refresh_lighthouse_provider_models_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result["created"] == 0 + assert result["updated"] == 0 + assert result["deleted"] == 0 + http_client = mock_openai.call_args.kwargs["http_client"] + assert http_client.follow_redirects is False + assert http_client.trust_env is False + + def test_refresh_models_masks_remote_http_error(self, tenants_fixture): + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + base_url="https://93.184.216.34/api/v1", + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "compatible-key"} + provider_cfg.save() + remote_body = "

remote 404 body

" + response = httpx.Response( + 404, + request=httpx.Request("GET", "https://provider.example/v1/models"), + ) + + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.side_effect = openai.NotFoundError( + remote_body, + response=response, + body=remote_body, + ) + mock_openai.return_value = mock_client + + eager_result = refresh_lighthouse_provider_models_task.apply( + kwargs={ + "provider_config_id": str(provider_cfg.id), + "tenant_id": str(tenants_fixture[0].id), + } + ) + + assert eager_result.successful() + result = eager_result.result + assert result["created"] == 0 + assert result["updated"] == 0 + assert result["deleted"] == 0 + assert result["error"] == "Provider connection failed" + assert remote_body not in result["error"] + def test_refresh_models_mixed_operations(self, tenants_fixture): """Test mixed create, update, and delete operations.""" # Create provider configuration From 106026614d6b1ad97b132834d7095b7d36fc19f0 Mon Sep 17 00:00:00 2001 From: Pepe Fagoaga Date: Fri, 10 Jul 2026 12:55:16 +0200 Subject: [PATCH 14/20] chore(security): allow internal endpoints for Lighthouse AI OpenAI compatible (#11941) --- ...penai-compatible-allowed-hosts.security.md | 1 + api/src/backend/api/tests/test_validators.py | 98 ++++++++++++++++++- api/src/backend/api/validators.py | 15 +++ api/src/backend/config/django/base.py | 9 ++ .../prowler-app-lighthouse-multi-llm.mdx | 19 ++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 api/changelog.d/lighthouse-openai-compatible-allowed-hosts.security.md diff --git a/api/changelog.d/lighthouse-openai-compatible-allowed-hosts.security.md b/api/changelog.d/lighthouse-openai-compatible-allowed-hosts.security.md new file mode 100644 index 0000000000..a015cd090b --- /dev/null +++ b/api/changelog.d/lighthouse-openai-compatible-allowed-hosts.security.md @@ -0,0 +1 @@ +`LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS` environment variable to allow internal hosts as OpenAI-compatible Lighthouse AI base URLs diff --git a/api/src/backend/api/tests/test_validators.py b/api/src/backend/api/tests/test_validators.py index 2b6c384cf0..a431a659c7 100644 --- a/api/src/backend/api/tests/test_validators.py +++ b/api/src/backend/api/tests/test_validators.py @@ -1,8 +1,12 @@ import socket import pytest -from api.validators import validate_lighthouse_openai_compatible_base_url +from api.validators import ( + resolve_lighthouse_openai_compatible_host, + validate_lighthouse_openai_compatible_base_url, +) from django.core.exceptions import ValidationError +from django.test import override_settings def test_lighthouse_base_url_rejects_http_scheme(): @@ -148,3 +152,95 @@ def test_lighthouse_base_url_accepts_public_resolved_address(monkeypatch): validate_lighthouse_openai_compatible_base_url("https://openrouter.ai/api/v1") is None ) + + +@override_settings( + LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"] +) +def test_lighthouse_base_url_accepts_allowlisted_host_without_resolution(monkeypatch): + def fail_resolution(*_args, **_kwargs): + raise AssertionError("allowlisted hosts must not be resolved") + + monkeypatch.setattr("api.validators.socket.getaddrinfo", fail_resolution) + + assert ( + validate_lighthouse_openai_compatible_base_url( + "https://custom-openai.internal/v1" + ) + is None + ) + + +@override_settings( + LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"] +) +def test_lighthouse_resolve_returns_allowlisted_hostname_unpinned(): + assert resolve_lighthouse_openai_compatible_host( + "Custom-OpenAI.internal.", 443 + ) == ("custom-openai.internal",) + + +@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["localhost"]) +def test_lighthouse_base_url_accepts_allowlisted_blocked_host(): + assert ( + validate_lighthouse_openai_compatible_base_url( + "https://localhost/v1", + resolve_dns=False, + ) + is None + ) + + +@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["10.0.0.1"]) +def test_lighthouse_base_url_accepts_allowlisted_private_ip_literal(): + assert ( + validate_lighthouse_openai_compatible_base_url( + "https://10.0.0.1/v1", + resolve_dns=False, + ) + is None + ) + + +@override_settings( + LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=[" Custom-OpenAI.Internal. "] +) +def test_lighthouse_allowlist_entries_are_normalized(): + assert ( + validate_lighthouse_openai_compatible_base_url( + "https://custom-openai.internal/v1", + resolve_dns=False, + ) + is None + ) + + +@override_settings( + LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"] +) +def test_lighthouse_base_url_rejects_host_not_in_allowlist(): + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + "https://localhost/v1", + resolve_dns=False, + ) + + +@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=[""]) +def test_lighthouse_allowlist_ignores_empty_entries(): + with pytest.raises(ValidationError, match="external public endpoint"): + validate_lighthouse_openai_compatible_base_url( + "https://localhost/v1", + resolve_dns=False, + ) + + +@override_settings( + LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"] +) +def test_lighthouse_base_url_allowlisted_host_still_requires_https(): + with pytest.raises(ValidationError, match="HTTPS"): + validate_lighthouse_openai_compatible_base_url( + "http://custom-openai.internal/v1", + resolve_dns=False, + ) diff --git a/api/src/backend/api/validators.py b/api/src/backend/api/validators.py index 0bc45fb48b..6ad7ccfe0f 100644 --- a/api/src/backend/api/validators.py +++ b/api/src/backend/api/validators.py @@ -3,6 +3,7 @@ import socket import string from urllib.parse import urlparse +from django.conf import settings from django.core.exceptions import ValidationError from django.utils.translation import gettext as _ @@ -23,6 +24,14 @@ def _normalize_hostname(hostname: str) -> str: return hostname.rstrip(".").lower() +def _lighthouse_openai_compatible_allowed_hosts() -> frozenset[str]: + return frozenset( + _normalize_hostname(allowed_host.strip()) + for allowed_host in settings.LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS + if allowed_host and allowed_host.strip() + ) + + def _validate_lighthouse_public_ip(address: str) -> None: ip_address = ipaddress.ip_address(address) if isinstance(ip_address, ipaddress.IPv6Address): @@ -50,6 +59,12 @@ def resolve_lighthouse_openai_compatible_host( ) -> tuple[str, ...]: """Return public IP addresses that are safe for Lighthouse outbound use.""" hostname = _normalize_hostname(hostname) + if hostname in _lighthouse_openai_compatible_allowed_hosts(): + # Operator-allowlisted hosts skip the public-endpoint checks; returning + # the hostname makes the network backend connect through regular DNS + # resolution instead of pinned addresses. + return (hostname,) + if hostname in LIGHTHOUSE_BLOCKED_METADATA_HOSTS or hostname.endswith(".localhost"): raise ValidationError( _("Base URL must use an external public endpoint."), diff --git a/api/src/backend/config/django/base.py b/api/src/backend/config/django/base.py index 63dab22c2c..23af0cc6d3 100644 --- a/api/src/backend/config/django/base.py +++ b/api/src/backend/config/django/base.py @@ -317,6 +317,15 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int( # Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted). ATTACK_PATHS_SINK_DATABASE = env.str("ATTACK_PATHS_SINK_DATABASE", default="neo4j") +# Lighthouse AI +# Comma-separated hostnames (or IP literals) that bypass the SSRF validation +# applied to OpenAI-compatible provider base URLs, so self-hosted deployments +# can point Lighthouse AI at internal endpoints. Empty by default: every base +# URL must resolve to a public endpoint. +LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS = env.list( + "LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS", default=[] +) + # Orphan task recovery feature flags. The master switch is OFF by default, so task # recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group # toggles default to enabled, so once the master is on every group recovers unless a diff --git a/docs/user-guide/tutorials/prowler-app-lighthouse-multi-llm.mdx b/docs/user-guide/tutorials/prowler-app-lighthouse-multi-llm.mdx index cdc6e84b9a..b1cb918de7 100644 --- a/docs/user-guide/tutorials/prowler-app-lighthouse-multi-llm.mdx +++ b/docs/user-guide/tutorials/prowler-app-lighthouse-multi-llm.mdx @@ -128,6 +128,25 @@ To connect a provider: 3. Configure in Lighthouse AI: - **API Key**: OpenRouter API key - **Base URL**: `https://openrouter.ai/api/v1` + + ### Base URL Validation + + To prevent server-side request forgery (SSRF), Prowler API validates the base URL before connecting to it: + + - The URL must use HTTPS. + - The host must resolve to a public IP address. Private, loopback, link-local, and cloud metadata addresses are rejected. + + + This validation can break configurations that point to internal endpoints, such as a self-hosted Ollama server. This is intentional: it fixes a security issue where the Prowler API could be directed to internal services. Internal endpoints must now be allowed explicitly through `LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS`. + + + To allow internal endpoints, set a comma-separated list of hostnames or IP addresses in the Prowler API environment (for Docker Compose deployments, the shared `.env` file): + + ```bash + LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=custom-openai.internal,10.0.0.20 + ``` + + Hosts in this list skip the public-endpoint validation. HTTPS is still required, so the endpoint needs a certificate the Prowler API trusts. From 3369e4826055b31b1bc6fdece592834101c92513 Mon Sep 17 00:00:00 2001 From: GOUTHAM HARIGOVIND Date: Fri, 10 Jul 2026 16:40:15 +0530 Subject: [PATCH 15/20] feat(ec2): Implement AMI block public access check (#11794) (#11828) Co-authored-by: Daniel Barranquero --- ...account-block-public-access-check.added.md | 1 + .../__init__.py | 0 ..._account_block_public_access.metadata.json | 40 ++++++ .../ec2_ami_account_block_public_access.py | 29 ++++ .../providers/aws/services/ec2/ec2_service.py | 22 +++ ...c2_ami_account_block_public_access_test.py | 133 ++++++++++++++++++ 6 files changed, 225 insertions(+) create mode 100644 prowler/changelog.d/ami-account-block-public-access-check.added.md create mode 100644 prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/__init__.py create mode 100644 prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.metadata.json create mode 100644 prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.py create mode 100644 tests/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access_test.py diff --git a/prowler/changelog.d/ami-account-block-public-access-check.added.md b/prowler/changelog.d/ami-account-block-public-access-check.added.md new file mode 100644 index 0000000000..32d830bd60 --- /dev/null +++ b/prowler/changelog.d/ami-account-block-public-access-check.added.md @@ -0,0 +1 @@ +`ec2_ami_account_block_public_access` check for AWS provider, verifying AMI block public access is enabled at the account level in each Region so AMIs cannot be shared publicly diff --git a/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/__init__.py b/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.metadata.json b/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.metadata.json new file mode 100644 index 0000000000..c8638927a3 --- /dev/null +++ b/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.metadata.json @@ -0,0 +1,40 @@ +{ + "Provider": "aws", + "CheckID": "ec2_ami_account_block_public_access", + "CheckTitle": "AMI block public access is enabled at the account level", + "CheckType": [ + "Software and Configuration Checks/AWS Security Best Practices", + "Effects/Data Exposure" + ], + "ServiceName": "ec2", + "SubServiceName": "", + "ResourceIdTemplate": "", + "Severity": "medium", + "ResourceType": "Other", + "ResourceGroup": "compute", + "Description": "AMI block public access configuration is assessed to see whether public sharing of AMIs is blocked in the account and Region. When enabled (`block-new-sharing`), no AMI in the Region can be made public regardless of individual image permissions.", + "Risk": "Without blocking public access, AMIs could be accidentally or maliciously shared publicly, exposing baked-in secrets, source code, and infrastructure details to unauthorized actors.", + "RelatedUrl": "", + "AdditionalURLs": [ + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/sharingamis-intro.html", + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/image-block-public-access.html" + ], + "Remediation": { + "Code": { + "CLI": "aws ec2 enable-image-block-public-access --image-block-public-access-state block-new-sharing", + "NativeIaC": "", + "Other": "1. In the AWS console, select the target Region in the top-right.\n2. Go to EC2 > AMIs.\n3. In the AMIs page, choose Block public access for AMIs (or EC2 Dashboard > Account attributes > Data protection and security).\n4. Choose Manage and enable Block public access.\n5. Save changes.", + "Terraform": "```hcl\nresource \"aws_ec2_image_block_public_access\" \"\" {\n state = \"block-new-sharing\" # Blocks new public sharing of AMIs in the configured Region\n}\n```" + }, + "Recommendation": { + "Text": "Enable AMI block public access (`block-new-sharing`) in every active Region. Apply guardrails (SCPs) to prevent it from being disabled, and review any AMIs that are currently shared publicly.", + "Url": "https://hub.prowler.com/check/ec2_ami_account_block_public_access" + } + }, + "Categories": [ + "internet-exposed" + ], + "DependsOn": [], + "RelatedTo": [], + "Notes": "" +} diff --git a/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.py b/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.py new file mode 100644 index 0000000000..08a892a674 --- /dev/null +++ b/prowler/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access.py @@ -0,0 +1,29 @@ +from prowler.lib.check.models import Check, Check_Report_AWS +from prowler.providers.aws.services.ec2.ec2_client import ec2_client + + +class ec2_ami_account_block_public_access(Check): + def execute(self): + findings = [] + for state in ec2_client.ami_block_public_access_states: + report = Check_Report_AWS( + metadata=self.metadata(), + resource=state, + ) + report.resource_id = ec2_client.audited_account + report.resource_arn = ec2_client.account_arn_template + + if state.status == "block-new-sharing": + report.status = "PASS" + report.status_extended = ( + f"AMI Block Public Access is enabled in {state.region}." + ) + else: + report.status = "FAIL" + report.status_extended = ( + f"AMI Block Public Access is disabled in {state.region}." + ) + + findings.append(report) + + return findings diff --git a/prowler/providers/aws/services/ec2/ec2_service.py b/prowler/providers/aws/services/ec2/ec2_service.py index 45a45e10d5..e630d9a1c7 100644 --- a/prowler/providers/aws/services/ec2/ec2_service.py +++ b/prowler/providers/aws/services/ec2/ec2_service.py @@ -52,6 +52,8 @@ class EC2(AWSService): self.__threading_call__(self._describe_ec2_addresses) self.ebs_block_public_access_snapshots_states = [] self.__threading_call__(self._get_snapshot_block_public_access_state) + self.ami_block_public_access_states = [] + self.__threading_call__(self._get_ami_block_public_access_state) self.instance_metadata_defaults = [] self.__threading_call__(self._get_instance_metadata_defaults) self.launch_templates = [] @@ -498,6 +500,21 @@ class EC2(AWSService): f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) + def _get_ami_block_public_access_state(self, regional_client): + try: + self.ami_block_public_access_states.append( + AmiBlockPublicAccess( + status=regional_client.get_image_block_public_access_state()[ + "ImageBlockPublicAccessState" + ], + region=regional_client.region, + ) + ) + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + def _get_instance_metadata_defaults(self, regional_client): try: instances_in_region = self.attributes_for_regions.get( @@ -798,6 +815,11 @@ class EbsSnapshotBlockPublicAccess(BaseModel): region: str +class AmiBlockPublicAccess(BaseModel): + status: str + region: str + + class InstanceMetadataDefaults(BaseModel): http_tokens: Optional[str] instances: bool diff --git a/tests/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access_test.py b/tests/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access_test.py new file mode 100644 index 0000000000..225a12348a --- /dev/null +++ b/tests/providers/aws/services/ec2/ec2_ami_account_block_public_access/ec2_ami_account_block_public_access_test.py @@ -0,0 +1,133 @@ +from unittest import mock + +from moto import mock_aws + +from tests.providers.aws.utils import ( + AWS_ACCOUNT_NUMBER, + AWS_REGION_US_EAST_1, + set_mocked_aws_provider, +) + + +class Test_ec2_ami_account_block_public_access: + @mock_aws + def test_ec2_ami_block_public_access_state_unblocked(self): + from prowler.providers.aws.services.ec2.ec2_service import AmiBlockPublicAccess + + ec2_client = mock.MagicMock() + ec2_client.ami_block_public_access_states = [ + AmiBlockPublicAccess(status="unblocked", region=AWS_REGION_US_EAST_1) + ] + ec2_client.audited_account = AWS_ACCOUNT_NUMBER + ec2_client.region = AWS_REGION_US_EAST_1 + ec2_client.account_arn_template = ( + f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account" + ) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_aws_provider(), + ), + mock.patch( + "prowler.providers.aws.services.ec2.ec2_ami_account_block_public_access.ec2_ami_account_block_public_access.ec2_client", + new=ec2_client, + ), + ): + # Test Check + from prowler.providers.aws.services.ec2.ec2_ami_account_block_public_access.ec2_ami_account_block_public_access import ( + ec2_ami_account_block_public_access, + ) + + check = ec2_ami_account_block_public_access() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"AMI Block Public Access is disabled in {AWS_REGION_US_EAST_1}." + ) + assert ( + result[0].resource_arn + == f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account" + ) + assert result[0].resource_id == AWS_ACCOUNT_NUMBER + assert result[0].region == AWS_REGION_US_EAST_1 + + @mock_aws + def test_ec2_ami_block_public_access_state_block_new_sharing(self): + from prowler.providers.aws.services.ec2.ec2_service import AmiBlockPublicAccess + + ec2_client = mock.MagicMock() + ec2_client.ami_block_public_access_states = [ + AmiBlockPublicAccess( + status="block-new-sharing", region=AWS_REGION_US_EAST_1 + ) + ] + ec2_client.audited_account = AWS_ACCOUNT_NUMBER + ec2_client.region = AWS_REGION_US_EAST_1 + ec2_client.account_arn_template = ( + f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account" + ) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_aws_provider(), + ), + mock.patch( + "prowler.providers.aws.services.ec2.ec2_ami_account_block_public_access.ec2_ami_account_block_public_access.ec2_client", + new=ec2_client, + ), + ): + # Test Check + from prowler.providers.aws.services.ec2.ec2_ami_account_block_public_access.ec2_ami_account_block_public_access import ( + ec2_ami_account_block_public_access, + ) + + check = ec2_ami_account_block_public_access() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"AMI Block Public Access is enabled in {AWS_REGION_US_EAST_1}." + ) + assert ( + result[0].resource_arn + == f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account" + ) + assert result[0].resource_id == AWS_ACCOUNT_NUMBER + assert result[0].region == AWS_REGION_US_EAST_1 + + @mock_aws + def test_ec2_ami_block_public_access_no_resources(self): + ec2_client = mock.MagicMock() + ec2_client.ami_block_public_access_states = [] + ec2_client.audited_account = AWS_ACCOUNT_NUMBER + ec2_client.region = AWS_REGION_US_EAST_1 + ec2_client.account_arn_template = ( + f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account" + ) + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_aws_provider(), + ), + mock.patch( + "prowler.providers.aws.services.ec2.ec2_ami_account_block_public_access.ec2_ami_account_block_public_access.ec2_client", + new=ec2_client, + ), + ): + # Test Check + from prowler.providers.aws.services.ec2.ec2_ami_account_block_public_access.ec2_ami_account_block_public_access import ( + ec2_ami_account_block_public_access, + ) + + check = ec2_ami_account_block_public_access() + result = check.execute() + + assert len(result) == 0 From 58fd4170b55ea57e956792f9e0b96b65bf50e5f0 Mon Sep 17 00:00:00 2001 From: "Pablo Fernandez Guerra (PFE)" <148432447+pfe-nazaries@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:22:41 +0200 Subject: [PATCH 16/20] feat(ui): support dynamic providers (#11869) Co-authored-by: Pablo F.G --- .../finding-groups/finding-groups.adapter.ts | 2 +- ui/actions/scans/scans.ts | 9 +- .../_components/accounts-selector.test.tsx | 4 +- .../provider-type-selector.test.tsx | 5 +- .../_components/provider-type-selector.tsx | 70 ++++++----- .../__tests__/alert-form-modal.test.tsx | 2 + .../providers/providers-page.utils.test.ts | 2 + .../_components/scan-configuration-editor.tsx | 27 ++--- .../scan-configuration-providers.test.ts | 91 ++++++++++++++ .../scan-configuration-providers.ts | 46 +++++++ ui/changelog.d/dynamic-providers.added.md | 1 + .../provider-account-selectors.test.tsx | 1 + .../table/provider-icon-cell.test.tsx | 8 +- .../findings/table/provider-icon-cell.tsx | 15 +-- .../generic-provider-badge.tsx | 22 ++++ .../provider-type-icon.test.tsx | 21 ++-- .../providers-badge/provider-type-icon.tsx | 26 ++-- .../providers-accounts-table.test.tsx | 1 + .../providers-accounts-view.test.tsx | 1 + .../providers/table/column-providers.test.tsx | 1 + .../providers/table/column-providers.tsx | 5 + .../table/data-table-row-actions.test.tsx | 107 +++++++++++++++++ .../table/data-table-row-actions.tsx | 74 +++++++----- .../workflow/forms/connect-account-form.tsx | 9 +- .../scans/launch-scan-modal.test.tsx | 4 +- ui/components/scans/scans-page-shell.test.tsx | 4 +- ui/components/scans/scans.utils.test.ts | 1 + .../shadcn/entities/get-provider-logo.tsx | 6 +- .../provider-credentials/build-credentials.ts | 14 ++- ui/lib/provider-filters.test.ts | 84 +++++++++++++ ui/lib/provider-filters.ts | 112 ++---------------- ui/lib/provider-helpers.ts | 4 +- ui/types/providers.test.ts | 68 +++++++++++ ui/types/providers.ts | 40 ++++++- 34 files changed, 640 insertions(+), 247 deletions(-) create mode 100644 ui/app/(prowler)/scans/config/_components/scan-configuration-providers.test.ts create mode 100644 ui/app/(prowler)/scans/config/_components/scan-configuration-providers.ts create mode 100644 ui/changelog.d/dynamic-providers.added.md create mode 100644 ui/components/icons/providers-badge/generic-provider-badge.tsx create mode 100644 ui/lib/provider-filters.test.ts create mode 100644 ui/types/providers.test.ts diff --git a/ui/actions/finding-groups/finding-groups.adapter.ts b/ui/actions/finding-groups/finding-groups.adapter.ts index e8d42beaa5..c9ee7167c6 100644 --- a/ui/actions/finding-groups/finding-groups.adapter.ts +++ b/ui/actions/finding-groups/finding-groups.adapter.ts @@ -210,7 +210,7 @@ export function adaptFindingGroupResourcesResponse( rowType: FINDINGS_ROW_TYPE.RESOURCE, findingId: item.attributes.finding_id || item.id, checkId, - providerType: (item.attributes.provider?.type || "aws") as ProviderType, + providerType: (item.attributes.provider?.type ?? "") as ProviderType, providerAlias: item.attributes.provider?.alias || "", providerUid: item.attributes.provider?.uid || "", resourceName: item.attributes.resource?.name || "-", diff --git a/ui/actions/scans/scans.ts b/ui/actions/scans/scans.ts index 5a6ddb7a29..e192bb5b49 100644 --- a/ui/actions/scans/scans.ts +++ b/ui/actions/scans/scans.ts @@ -14,10 +14,7 @@ import { type ComplianceReportType, } from "@/lib/compliance/compliance-report-types"; import { runWithConcurrencyLimit } from "@/lib/concurrency"; -import { - appendSanitizedProviderTypeFilters, - sanitizeProviderTypesCsv, -} from "@/lib/provider-filters"; +import { appendSanitizedProviderTypeFilters } from "@/lib/provider-filters"; import { addScanOperation } from "@/lib/sentry-breadcrumbs"; import { handleApiError, handleApiResponse } from "@/lib/server-actions-helper"; import { SCAN_STATES } from "@/types/attack-paths"; @@ -66,10 +63,6 @@ export const getScansByState = async () => { const url = new URL(`${apiBaseUrl}/scans`); // Request only the necessary fields to optimize the response url.searchParams.append("fields[scans]", "state"); - url.searchParams.append( - "filter[provider_type__in]", - sanitizeProviderTypesCsv(), - ); // Only need to know whether at least one completed scan exists; filter server-side // and cap to a single row so the answer is correct regardless of total scan count. url.searchParams.append("filter[state]", SCAN_STATES.COMPLETED); diff --git a/ui/app/(prowler)/_overview/_components/accounts-selector.test.tsx b/ui/app/(prowler)/_overview/_components/accounts-selector.test.tsx index e69d0427ee..0b6e90a697 100644 --- a/ui/app/(prowler)/_overview/_components/accounts-selector.test.tsx +++ b/ui/app/(prowler)/_overview/_components/accounts-selector.test.tsx @@ -2,6 +2,7 @@ import { render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; +import { ProviderProps } from "@/types"; import { FILTER_FIELD } from "@/types/filters"; import { AccountsSelector } from "./accounts-selector"; @@ -99,12 +100,13 @@ vi.mock("@/components/shadcn/select/multiselect", () => ({ ), })); -const providers = [ +const providers: ProviderProps[] = [ { id: "provider-1", type: "providers" as const, attributes: { provider: "aws" as const, + is_dynamic: false, uid: "123456789012", alias: "Production AWS", status: "completed" as const, diff --git a/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx b/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx index b2c05b336d..23c7d4ace3 100644 --- a/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx +++ b/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx @@ -1,6 +1,8 @@ import { render, screen, within } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; +import { ProviderProps } from "@/types"; + import { ProviderTypeSelector } from "./provider-type-selector"; const multiSelectContentSpy = vi.fn(); @@ -69,12 +71,13 @@ vi.mock("@/components/shadcn/select/multiselect", () => ({ ), })); -const providers = [ +const providers: ProviderProps[] = [ { id: "provider-1", type: "providers" as const, attributes: { provider: "aws" as const, + is_dynamic: false, uid: "123456789012", alias: "Production AWS", status: "completed" as const, diff --git a/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx b/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx index e78412eeeb..fdda0f8e49 100644 --- a/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx +++ b/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx @@ -16,7 +16,22 @@ import { MultiSelectValue, } from "@/components/shadcn/select/multiselect"; import { useUrlFilters } from "@/hooks/use-url-filters"; -import { type ProviderProps, ProviderType } from "@/types/providers"; +import { + humanizeProviderId, + isKnownProviderType, + type ProviderProps, + ProviderType, +} from "@/types/providers"; + +/** + * Label for a provider type: the rich configured label for known types, a + * humanized id for anything the API returns outside the known set (dynamic + * plug-ins). Icons fall back to the generic glyph via `ProviderTypeIcon`. + */ +const providerTypeLabel = (type: ProviderType): string => + isKnownProviderType(type) + ? PROVIDER_TYPE_DATA[type].label + : humanizeProviderId(type); /** Common props shared by both batch and instant modes. */ interface ProviderTypeSelectorBaseProps { @@ -88,16 +103,8 @@ export const ProviderTypeSelector = ({ }; const availableTypes = Array.from( - new Set( - providers - // .filter((p) => p.attributes.connection?.connected) - .map((p) => p.attributes.provider), - ), - ) - .filter((type): type is ProviderType => type in PROVIDER_TYPE_DATA) - .sort((a, b) => - PROVIDER_TYPE_DATA[a].label.localeCompare(PROVIDER_TYPE_DATA[b].label), - ); + new Set(providers.map((p) => p.attributes.provider)), + ).sort((a, b) => providerTypeLabel(a).localeCompare(providerTypeLabel(b))); const selectedLabel = () => { if (selectedTypes.length === 0) return null; @@ -108,9 +115,7 @@ export const ProviderTypeSelector = ({ - - {PROVIDER_TYPE_DATA[providerType].label} - + {providerTypeLabel(providerType)} ); } @@ -120,7 +125,7 @@ export const ProviderTypeSelector = ({ items={(selectedTypes as ProviderType[]).map((type) => ({ key: type, type, - tooltip: PROVIDER_TYPE_DATA[type].label, + tooltip: providerTypeLabel(type), }))} /> @@ -176,23 +181,24 @@ export const ProviderTypeSelector = ({ > {selectedTypes.length === 0 ? "All selected" : "Select All"} - {availableTypes.map((providerType) => ( - - - {PROVIDER_TYPE_DATA[providerType].label} - - ))} + {availableTypes.map((providerType) => { + const label = providerTypeLabel(providerType); + + return ( + + + {label} + + ); + })} ) : (
diff --git a/ui/app/(prowler)/alerts/_components/__tests__/alert-form-modal.test.tsx b/ui/app/(prowler)/alerts/_components/__tests__/alert-form-modal.test.tsx index 6d5ebc945d..28518fd576 100644 --- a/ui/app/(prowler)/alerts/_components/__tests__/alert-form-modal.test.tsx +++ b/ui/app/(prowler)/alerts/_components/__tests__/alert-form-modal.test.tsx @@ -104,6 +104,7 @@ const mockProviders: ProviderProps[] = [ type: "providers", attributes: { provider: "aws", + is_dynamic: false, uid: "123456789012", alias: "Production AWS", status: "completed", @@ -131,6 +132,7 @@ const mockProviders: ProviderProps[] = [ type: "providers", attributes: { provider: "gcp", + is_dynamic: false, uid: "prowler-prod-project", alias: "Production GCP", status: "completed", diff --git a/ui/app/(prowler)/providers/providers-page.utils.test.ts b/ui/app/(prowler)/providers/providers-page.utils.test.ts index 4c712ff081..fff1cc863c 100644 --- a/ui/app/(prowler)/providers/providers-page.utils.test.ts +++ b/ui/app/(prowler)/providers/providers-page.utils.test.ts @@ -62,6 +62,7 @@ const providersResponse: ProvidersApiResponse = { type: "providers", attributes: { provider: "aws", + is_dynamic: false, uid: "111111111111", alias: "AWS App Account", status: "completed", @@ -107,6 +108,7 @@ const providersResponse: ProvidersApiResponse = { type: "providers", attributes: { provider: "aws", + is_dynamic: false, uid: "222222222222", alias: "Standalone Account", status: "completed", diff --git a/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx b/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx index 1583ccc527..e9086cd366 100644 --- a/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx +++ b/ui/app/(prowler)/scans/config/_components/scan-configuration-editor.tsx @@ -31,6 +31,8 @@ import { scanConfigurationFormSchema } from "@/types/formSchemas"; import { ProviderProps } from "@/types/providers"; import { ScanConfigurationData } from "@/types/scan-configurations"; +import { getSelectableProviders } from "./scan-configuration-providers"; + interface ScanConfigurationEditorProps { open: boolean; onClose: (saved: boolean) => void; @@ -90,21 +92,12 @@ function ScanConfigurationForm({ ? validateYaml(configText) : { isValid: true as const }; - // A provider can only be attached to one config at a time. We exclude - // providers that are owned by *other* configs from the selector so the user - // can't double-attach them. (AccountsSelector doesn't expose a per-option - // disabled state, so filtering out is the cleanest contract here.) - const ownerByProvider = new Map(); - for (const c of existingConfigs) { - if (config && c.id === config.id) continue; - for (const pid of c.attributes.providers || []) { - ownerByProvider.set(pid, c.attributes.name); - } - } - const selectableProviders = richProviders.filter( - (p) => !ownerByProvider.has(p.id), - ); - const lockedCount = richProviders.length - selectableProviders.length; + // Dynamic providers can't use a Scan Configuration, and a provider can only be + // attached to one config at a time, so both are excluded from the selector. + // (AccountsSelector doesn't expose a per-option disabled state, so filtering + // out is the cleanest contract here.) + const { selectableProviders, configurableCount, lockedCount } = + getSelectableProviders(richProviders, existingConfigs, config?.id ?? null); const onSubmit = form.handleSubmit(async (values) => { // Block on a YAML syntax error (the inline message already explains it); the @@ -252,7 +245,9 @@ function ScanConfigurationForm({

{richProviders.length === 0 ? "No providers available in this tenant." - : "All providers are already attached to other Scan Configurations."} + : configurableCount === 0 + ? "Dynamic providers can't use custom Scan Configurations, so there are none to attach." + : "All providers are already attached to other Scan Configurations."}

) : ( + ({ + id, + type: "providers", + attributes: { + provider: isDynamic ? "template" : "aws", + is_dynamic: isDynamic, + }, + }) as unknown as ProviderProps; + +const config = (id: string, providers: string[]): ScanConfigurationData => ({ + type: "scan-configurations", + id, + attributes: { + inserted_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + name: id, + configuration: {}, + providers, + }, +}); + +describe("getSelectableProviders", () => { + it("excludes dynamic providers (they have no config baseline to override)", () => { + const result = getSelectableProviders( + [provider("aws-1"), provider("dyn-1", true)], + [], + null, + ); + + expect(result.selectableProviders.map((p) => p.id)).toEqual(["aws-1"]); + expect(result.configurableCount).toBe(1); + expect(result.lockedCount).toBe(0); + }); + + it("excludes providers already attached to another config", () => { + const result = getSelectableProviders( + [provider("aws-1"), provider("aws-2")], + [config("other", ["aws-2"])], + null, + ); + + expect(result.selectableProviders.map((p) => p.id)).toEqual(["aws-1"]); + expect(result.lockedCount).toBe(1); + }); + + it("keeps providers attached to the config being edited selectable", () => { + const result = getSelectableProviders( + [provider("aws-1"), provider("aws-2")], + [config("current", ["aws-1"])], + "current", + ); + + expect(result.selectableProviders.map((p) => p.id)).toEqual([ + "aws-1", + "aws-2", + ]); + expect(result.lockedCount).toBe(0); + }); + + it("lockedCount counts only configurable providers attached elsewhere, not dynamic ones", () => { + const result = getSelectableProviders( + [provider("aws-1"), provider("aws-2"), provider("dyn-1", true)], + [config("other", ["aws-2"])], + null, + ); + + expect(result.selectableProviders.map((p) => p.id)).toEqual(["aws-1"]); + expect(result.configurableCount).toBe(2); + expect(result.lockedCount).toBe(1); + }); + + it("reports zero configurable providers when every provider is dynamic", () => { + const result = getSelectableProviders( + [provider("dyn-1", true), provider("dyn-2", true)], + [], + null, + ); + + expect(result.selectableProviders).toEqual([]); + expect(result.configurableCount).toBe(0); + expect(result.lockedCount).toBe(0); + }); +}); diff --git a/ui/app/(prowler)/scans/config/_components/scan-configuration-providers.ts b/ui/app/(prowler)/scans/config/_components/scan-configuration-providers.ts new file mode 100644 index 0000000000..4e392e249e --- /dev/null +++ b/ui/app/(prowler)/scans/config/_components/scan-configuration-providers.ts @@ -0,0 +1,46 @@ +import { ProviderProps } from "@/types/providers"; +import { ScanConfigurationData } from "@/types/scan-configurations"; + +export interface SelectableProvidersResult { + selectableProviders: ProviderProps[]; + /** Providers eligible for a Scan Configuration at all (i.e. non-dynamic). */ + configurableCount: number; + /** Configurable providers hidden because another config already owns them. */ + lockedCount: number; +} + +/** + * Providers that can be attached to a Scan Configuration. + * + * Dynamic providers are SDK-defined and have no `config.yaml` baseline to + * override, so a Scan Configuration can't apply to them — they are never + * selectable. A provider can also belong to only one config at a time, so those + * already attached to *another* config are excluded (the one being edited is + * kept selectable). + */ +export function getSelectableProviders( + richProviders: ProviderProps[], + existingConfigs: ScanConfigurationData[], + currentConfigId: string | null, +): SelectableProvidersResult { + const attachedElsewhere = new Set(); + for (const c of existingConfigs) { + if (currentConfigId && c.id === currentConfigId) continue; + for (const pid of c.attributes.providers || []) { + attachedElsewhere.add(pid); + } + } + + const configurableProviders = richProviders.filter( + (p) => !p.attributes.is_dynamic, + ); + const selectableProviders = configurableProviders.filter( + (p) => !attachedElsewhere.has(p.id), + ); + + return { + selectableProviders, + configurableCount: configurableProviders.length, + lockedCount: configurableProviders.length - selectableProviders.length, + }; +} diff --git a/ui/changelog.d/dynamic-providers.added.md b/ui/changelog.d/dynamic-providers.added.md new file mode 100644 index 0000000000..e2d213dca7 --- /dev/null +++ b/ui/changelog.d/dynamic-providers.added.md @@ -0,0 +1 @@ +Dynamically registered providers are now listed, filtered, and rendered across the UI, using a generic icon and humanized label when no bespoke assets exist, a "Custom" badge, and read-only handling for non-configurable providers diff --git a/ui/components/filters/provider-account-selectors.test.tsx b/ui/components/filters/provider-account-selectors.test.tsx index 01868d0e8c..c30397eed4 100644 --- a/ui/components/filters/provider-account-selectors.test.tsx +++ b/ui/components/filters/provider-account-selectors.test.tsx @@ -77,6 +77,7 @@ const makeProvider = ({ type: "providers", attributes: { provider, + is_dynamic: false, uid, alias, status: "completed", diff --git a/ui/components/findings/table/provider-icon-cell.test.tsx b/ui/components/findings/table/provider-icon-cell.test.tsx index 593122d33a..2d7082b120 100644 --- a/ui/components/findings/table/provider-icon-cell.test.tsx +++ b/ui/components/findings/table/provider-icon-cell.test.tsx @@ -33,13 +33,15 @@ describe("ProviderIconCell", () => { expect(await screen.findByText("AWS")).toBeInTheDocument(); }); - it("renders a '?' placeholder for a provider type missing from the map", () => { - render( + it("renders the neutral generic glyph for a provider missing from the map", () => { + // Dynamic/unknown providers render the shared generic glyph, not a "?". + const { container } = render( , ); - expect(screen.getByText("?")).toBeInTheDocument(); + expect(screen.queryByText("?")).not.toBeInTheDocument(); + expect(container.querySelector("svg")).toBeInTheDocument(); }); }); diff --git a/ui/components/findings/table/provider-icon-cell.tsx b/ui/components/findings/table/provider-icon-cell.tsx index 350e730e78..5a996415e6 100644 --- a/ui/components/findings/table/provider-icon-cell.tsx +++ b/ui/components/findings/table/provider-icon-cell.tsx @@ -1,7 +1,4 @@ -import { - PROVIDER_TYPE_DATA, - ProviderTypeIcon, -} from "@/components/icons/providers-badge/provider-type-icon"; +import { ProviderTypeIcon } from "@/components/icons/providers-badge/provider-type-icon"; import { cn } from "@/lib/utils"; import { ProviderType } from "@/types"; @@ -16,16 +13,6 @@ export const ProviderIconCell = ({ size = 26, className = "size-8 rounded-md bg-white", }: ProviderIconCellProps) => { - // Unknown provider types (present in the data but missing from the shared - // PROVIDER_TYPE_DATA map) render an explicit "?" rather than an empty icon. - if (!(provider in PROVIDER_TYPE_DATA)) { - return ( -
- ? -
- ); - } - return (
= ({ + size, + width, + height, + ...props +}) => ( +
+ } + > + + + + ); + } + const regionFilter = getSingleSearchParam( + resolvedSearchParams["filter[region__in]"], + ); + const cisProfileFilter = getSingleSearchParam( + resolvedSearchParams["filter[cis_profile_level]"], + ); const logoPath = getComplianceIcon(compliancetitle); - // Create a key that excludes pagination parameters to preserve accordion state avoiding reloads with pagination - const paramsForKey = Object.fromEntries( - Object.entries(resolvedSearchParams).filter( - ([key]) => key !== "page" && key !== "pageSize", - ), - ); - const searchParamsKey = JSON.stringify(paramsForKey); + const searchParamsKey = buildSearchParamsKey(resolvedSearchParams); const formattedTitle = compliancetitle.split("-").join(" "); const pageTitle = version @@ -176,33 +228,45 @@ export default async function ComplianceDetail({ return ( -
-
- -
- {selectedScanId && ( -
- +
+
+
- )} -
+ {selectedScanId && ( +
+ + Report + + + } + reportType={getReportTypeForCompliance( + attributesData?.data?.[0]?.attributes?.framework, + complianceId, + latestCisIds.has(complianceId), + )} + /> +
+ )} +
+ */}
-
({ + fetchMock: vi.fn(), + getAuthHeadersMock: vi.fn(), + handleApiResponseMock: vi.fn(), + captureExceptionMock: vi.fn(), +})); + +vi.mock("@/lib", () => ({ + apiBaseUrl: "https://api.test/api/v1", + getAuthHeaders: getAuthHeadersMock, + getErrorMessage: (error: unknown) => + error instanceof Error ? error.message : String(error), + GENERIC_SERVER_ERROR_MESSAGE: "Generic server error.", +})); + +vi.mock("@/lib/server-actions-helper", () => ({ + handleApiResponse: handleApiResponseMock, +})); + +vi.mock("@sentry/nextjs", () => ({ + captureException: captureExceptionMock, +})); + +import { + generateCrossProviderPdf, + getCrossProviderComplianceOverview, + getCrossProviderPdfBinary, + getLatestCrossProviderPdf, +} from "./cross-provider"; + +const lastFetchUrl = (): URL => { + const call = fetchMock.mock.calls.at(-1); + if (!call) throw new Error("fetch was not called"); + return new URL(String(call[0])); +}; + +const jsonResponse = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/vnd.api+json" }, + }); + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", fetchMock); + fetchMock.mockResolvedValue(jsonResponse({ data: null })); + getAuthHeadersMock.mockResolvedValue({ + Accept: "application/vnd.api+json", + Authorization: "Bearer test-token", + }); + handleApiResponseMock.mockResolvedValue({ data: null }); +}); + +describe("getCrossProviderComplianceOverview", () => { + it("requests the overview with the required compliance_id filter", async () => { + await getCrossProviderComplianceOverview({ complianceId: "csa_ccm_4.0" }); + + const url = lastFetchUrl(); + expect(url.pathname).toBe("/api/v1/cross-provider-compliance-overviews"); + expect(url.searchParams.get("filter[compliance_id]")).toBe("csa_ccm_4.0"); + expect(Array.from(url.searchParams.keys())).toEqual([ + "filter[compliance_id]", + ]); + }); + + it("serializes optional filters and joins scan ids with commas", async () => { + await getCrossProviderComplianceOverview({ + complianceId: "dora_2022_2554", + filters: { + scanIds: ["scan-1", "scan-2"], + providerTypes: "aws,gcp", + providerIds: "prov-1", + providerGroups: "group-1", + }, + }); + + const url = lastFetchUrl(); + expect(url.searchParams.get("filter[scan__in]")).toBe("scan-1,scan-2"); + expect(url.searchParams.get("filter[provider_type__in]")).toBe("aws,gcp"); + expect(url.searchParams.get("filter[provider_id__in]")).toBe("prov-1"); + expect(url.searchParams.get("filter[provider_groups__in]")).toBe("group-1"); + expect(url.searchParams.has("filter[region__in]")).toBe(false); + }); + + it("wraps successful overview responses", async () => { + const payload = { data: { id: "csa_ccm_4.0" } }; + handleApiResponseMock.mockResolvedValue(payload); + + const result = await getCrossProviderComplianceOverview({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toEqual({ status: "success", response: payload }); + }); + + it("returns the shared action error shape for payment-required responses", async () => { + fetchMock.mockResolvedValue(jsonResponse({ errors: [] }, 402)); + handleApiResponseMock.mockResolvedValue({ + error: "Payment required.", + status: 402, + }); + + const result = await getCrossProviderComplianceOverview({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toEqual({ + status: "action-error", + result: { error: "Payment required.", status: 402 }, + }); + }); + + it("returns a load error when the network call throws", async () => { + fetchMock.mockRejectedValue(new Error("boom")); + + const result = await getCrossProviderComplianceOverview({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toEqual({ + status: "load-error", + message: + "Could not load cross-provider compliance data. Try again later.", + }); + }); +}); + +describe("generateCrossProviderPdf", () => { + it("POSTs pdf with filters and an optional report name", async () => { + fetchMock.mockResolvedValue( + jsonResponse({ data: { type: "tasks", id: "task-1" } }, 202), + ); + + const result = await generateCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + filters: { scanIds: ["scan-1"], providerTypes: "aws" }, + reportName: "quarterly.pdf", + }); + + expect(result).toEqual({ taskId: "task-1" }); + const call = fetchMock.mock.calls.at(-1); + expect((call?.[1] as RequestInit).method).toBe("POST"); + const url = lastFetchUrl(); + expect(url.pathname).toBe( + "/api/v1/cross-provider-compliance-overviews/pdf", + ); + expect(url.searchParams.get("filter[compliance_id]")).toBe("csa_ccm_4.0"); + expect(url.searchParams.get("filter[scan__in]")).toBe("scan-1"); + expect(url.searchParams.get("filter[provider_type__in]")).toBe("aws"); + expect(url.searchParams.get("report_name")).toBe("quarterly.pdf"); + expect(url.searchParams.has("only_failed")).toBe(false); + expect(url.searchParams.has("include_manual")).toBe(false); + }); + + it("omits the optional report name when not provided", async () => { + fetchMock.mockResolvedValue( + jsonResponse({ data: { type: "tasks", id: "task-1" } }, 202), + ); + + await generateCrossProviderPdf({ complianceId: "csa_ccm_4.0" }); + + const url = lastFetchUrl(); + expect(url.searchParams.has("report_name")).toBe(false); + }); + + it("returns the API error detail when generation fails", async () => { + fetchMock.mockResolvedValue( + jsonResponse({ errors: [{ detail: "No compatible scans." }] }, 422), + ); + + const result = await generateCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toEqual({ error: "No compatible scans." }); + }); + + it("reports 5xx failures to Sentry with the static route template only", async () => { + fetchMock.mockResolvedValue(jsonResponse({}, 500)); + + const result = await generateCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + reportName: "secret-name.pdf", + }); + + expect(result).toEqual({ error: "Generic server error." }); + expect(captureExceptionMock).toHaveBeenCalledTimes(1); + const serialized = JSON.stringify(captureExceptionMock.mock.calls[0]); + expect(serialized).toContain("cross-provider-compliance-overviews/pdf"); + expect(serialized).not.toContain("secret-name"); + }); + + it("falls back to the static message for text/html error pages", async () => { + // A proxy/gateway error page must not be parsed as JSON. + fetchMock.mockResolvedValue( + new Response("Bad Gateway", { + status: 422, + headers: { "Content-Type": "text/html" }, + }), + ); + + const result = await generateCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toEqual({ + error: + "Unable to start PDF generation. Contact support if the issue continues.", + }); + }); +}); + +describe("getCrossProviderPdfBinary", () => { + it("returns pending state on 202", async () => { + fetchMock.mockResolvedValue( + jsonResponse( + { data: { id: "task-1", attributes: { state: "executing" } } }, + 202, + ), + ); + + const result = await getCrossProviderPdfBinary("task-1"); + + expect(result).toEqual({ + pending: true, + state: "executing", + taskId: "task-1", + }); + expect(lastFetchUrl().pathname).toBe( + "/api/v1/cross-provider-compliance-overviews/pdf/task-1", + ); + }); + + it("returns the binary as base64 with the content-disposition filename", async () => { + fetchMock.mockResolvedValue( + new Response(Buffer.from("pdf-bytes"), { + status: 200, + headers: { + "Content-Type": "application/pdf", + "Content-Disposition": 'attachment; filename="csa-report.pdf"', + }, + }), + ); + + const result = await getCrossProviderPdfBinary("task-1"); + + expect(result).toEqual({ + success: true, + data: Buffer.from("pdf-bytes").toString("base64"), + filename: "csa-report.pdf", + }); + }); + + it("rejects task ids that could smuggle path segments, without fetching", async () => { + const result = await getCrossProviderPdfBinary("../../../etc/passwd"); + + expect(result).toEqual({ error: "Invalid task identifier." }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("returns the API error detail on failure", async () => { + fetchMock.mockResolvedValue( + jsonResponse({ errors: [{ detail: "Report expired." }] }, 410), + ); + + const result = await getCrossProviderPdfBinary("task-1"); + + expect(result).toEqual({ error: "Report expired." }); + }); + + it("reports 5xx failures to Sentry", async () => { + fetchMock.mockResolvedValue(jsonResponse({}, 500)); + + const result = await getCrossProviderPdfBinary("task-1"); + + expect(result).toEqual({ error: "Generic server error." }); + expect(captureExceptionMock).toHaveBeenCalledTimes(1); + const serialized = JSON.stringify(captureExceptionMock.mock.calls[0]); + expect(serialized).toContain("pdf/{taskId}"); + }); +}); + +describe("getLatestCrossProviderPdf", () => { + it("returns the report descriptor when a matching report exists", async () => { + fetchMock.mockResolvedValue( + jsonResponse({ + data: { + type: "tasks", + id: "task-9", + attributes: { + completed_at: "2026-07-01T10:00:00Z", + result: { filename: "csa-latest.pdf" }, + }, + }, + }), + ); + + const result = await getLatestCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + filters: { scanIds: ["scan-1"] }, + }); + + expect(result).toEqual({ + taskId: "task-9", + filename: "csa-latest.pdf", + completedAt: "2026-07-01T10:00:00Z", + }); + const url = lastFetchUrl(); + expect(url.pathname).toBe( + "/api/v1/cross-provider-compliance-overviews/pdf/latest", + ); + expect(url.searchParams.get("filter[scan__in]")).toBe("scan-1"); + }); + + it("returns null when no report has been generated yet (404)", async () => { + fetchMock.mockResolvedValue(jsonResponse({ errors: [] }, 404)); + + const result = await getLatestCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toBeNull(); + }); + + it("degrades to null on 5xx but still reports to Sentry", async () => { + const consoleErrorSpy = vi + .spyOn(console, "error") + .mockImplementation(() => {}); + fetchMock.mockResolvedValue(jsonResponse({}, 500)); + + const result = await getLatestCrossProviderPdf({ + complianceId: "csa_ccm_4.0", + }); + + expect(result).toBeNull(); + expect(captureExceptionMock).toHaveBeenCalledTimes(1); + const serialized = JSON.stringify(captureExceptionMock.mock.calls[0]); + expect(serialized).toContain("pdf/latest"); + + consoleErrorSpy.mockRestore(); + }); +}); diff --git a/ui/app/(prowler)/compliance/_actions/cross-provider.ts b/ui/app/(prowler)/compliance/_actions/cross-provider.ts new file mode 100644 index 0000000000..eacdc29fde --- /dev/null +++ b/ui/app/(prowler)/compliance/_actions/cross-provider.ts @@ -0,0 +1,315 @@ +"use server"; + +import * as Sentry from "@sentry/nextjs"; + +import type { ScanBinaryResult } from "@/actions/scans/scans"; +import { + apiBaseUrl, + GENERIC_SERVER_ERROR_MESSAGE, + getAuthHeaders, + getErrorMessage, +} from "@/lib"; +import { hasActionError } from "@/lib/action-errors"; +import { handleApiResponse } from "@/lib/server-actions-helper"; +import { SentryErrorSource, SentryErrorType } from "@/sentry"; + +import type { + CrossProviderApiFilters, + CrossProviderOverviewResponse, + CrossProviderOverviewResult, + LatestCrossProviderPdf, +} from "../_types"; +import { + CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE, + CROSS_PROVIDER_OVERVIEW_RESULT_STATUS, +} from "../_types"; + +const CROSS_PROVIDER_API_PATH = "/cross-provider-compliance-overviews"; + +/** Error payload shapes the PDF endpoints emit (JSON:API or plain). */ +interface PdfEndpointErrorBody { + errors?: Array<{ detail?: string }>; + error?: string; + message?: string; +} + +/** + * Extracts a user-safe message from a failed PDF endpoint response and, for + * unexpected failures, reports it to Sentry. `operation` must be a STATIC + * route template (e.g. `GET .../pdf/{taskId}`) — never the + * resolved URL, which would carry the task id or a user-typed report name. + */ +const getPdfEndpointErrorMessage = async ( + response: Response, + fallbackMessage: string, + operation: string, +): Promise => { + const contentType = response.headers.get("content-type")?.toLowerCase() || ""; + const errorData: PdfEndpointErrorBody | null = contentType.includes( + "text/html", + ) + ? null + : await response.json().catch(() => null); + + // These endpoints bypass handleApiResponse (binary/task protocol), so + // server failures would otherwise go unmonitored. + if (response.status >= 500) { + Sentry.captureException( + new Error( + `Cross-provider PDF request failed (${response.status}) at ${operation}`, + ), + { + tags: { + api_error: true, + status_code: response.status.toString(), + error_type: SentryErrorType.SERVER_ERROR, + error_source: SentryErrorSource.SERVER_ACTION, + }, + level: "error", + contexts: { + api_response: { + status: response.status, + statusText: response.statusText, + operation, + }, + }, + }, + ); + return GENERIC_SERVER_ERROR_MESSAGE; + } + + return ( + errorData?.errors?.[0]?.detail || + errorData?.error || + errorData?.message || + fallbackMessage + ); +}; + +/** Appends the shared cross-provider filter params to a request URL. */ +const applyFilters = (url: URL, filters?: CrossProviderApiFilters) => { + if (!filters) return; + + if (filters.scanIds && filters.scanIds.length > 0) { + url.searchParams.set("filter[scan__in]", filters.scanIds.join(",")); + } + + const paramMap = { + "filter[provider_type__in]": filters.providerTypes, + "filter[provider_id__in]": filters.providerIds, + "filter[provider_groups__in]": filters.providerGroups, + }; + for (const [key, value] of Object.entries(paramMap)) { + if (value && value.trim().length > 0) { + url.searchParams.set(key, value); + } + } +}; + +/** + * Aggregate a universal compliance framework across one scan per compatible + * provider (Prowler Cloud only — the OSS API has no such endpoint). + * + * When `filters.scanIds` is omitted the API auto-selects the latest COMPLETED + * scan per compatible provider. Non-2xx responses are returned as structured + * action errors so callers can reuse the app-wide 402/403 handlers. + */ +export const getCrossProviderComplianceOverview = async ({ + complianceId, + filters, +}: { + complianceId: string; + filters?: CrossProviderApiFilters; +}): Promise => { + const headers = await getAuthHeaders({ contentType: false }); + const url = new URL(`${apiBaseUrl}${CROSS_PROVIDER_API_PATH}`); + url.searchParams.set("filter[compliance_id]", complianceId); + applyFilters(url, filters); + + try { + const response = await fetch(url.toString(), { headers }); + const responseData = await handleApiResponse(response); + + if (hasActionError(responseData)) { + return { + status: CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.ACTION_ERROR, + result: responseData, + }; + } + + return { + status: CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.SUCCESS, + response: responseData as CrossProviderOverviewResponse, + }; + } catch (error) { + console.error("Error fetching cross-provider compliance overview:", error); + return { + status: CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.LOAD_ERROR, + message: CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE, + }; + } +}; + +/** + * Trigger ad-hoc generation of the combined cross-provider compliance PDF. + * + * The PDF is built asynchronously by a backend task: this returns the task id + * so the caller can poll it and then download via + * {@link getCrossProviderPdfBinary}. Pass the exact `scanIds` currently on + * screen (`attributes.scan_ids`) so the report matches the displayed data + * instead of re-resolving "latest scan per provider", which could race a scan + * completing in between. + */ +export const generateCrossProviderPdf = async ({ + complianceId, + filters, + reportName, +}: { + complianceId: string; + filters?: CrossProviderApiFilters; + /** Optional download filename; sanitized server-side. */ + reportName?: string; +}): Promise<{ taskId: string } | { error: string }> => { + const headers = await getAuthHeaders({ contentType: false }); + const url = new URL(`${apiBaseUrl}${CROSS_PROVIDER_API_PATH}/pdf`); + url.searchParams.set("filter[compliance_id]", complianceId); + applyFilters(url, filters); + if (reportName) url.searchParams.set("report_name", reportName); + + try { + const response = await fetch(url.toString(), { method: "POST", headers }); + + if (!response.ok) { + throw new Error( + await getPdfEndpointErrorMessage( + response, + "Unable to start PDF generation. Contact support if the issue continues.", + `POST ${CROSS_PROVIDER_API_PATH}/pdf`, + ), + ); + } + + const json = await response.json(); + const taskId = json?.data?.id; + if (!taskId) { + throw new Error("Unexpected response starting PDF generation."); + } + + return { taskId }; + } catch (error) { + return { error: getErrorMessage(error) }; + } +}; + +/** + * Fetch the finished cross-provider PDF for a task started by + * {@link generateCrossProviderPdf}. Speaks the same 202-pending / + * 2xx-binary / error-JSON protocol as the per-scan report endpoints, so it + * returns the shared {@link ScanBinaryResult} shape and callers can reuse the + * existing download plumbing unchanged. + */ +export const getCrossProviderPdfBinary = async ( + taskId: string, +): Promise => { + // The task id reaches the URL path: constrain it to the task-id charset + // (UUIDs) so a crafted value cannot smuggle `/`, `..` or a host. + const safeTaskId = taskId.trim(); + if (!/^[A-Za-z0-9_-]+$/.test(safeTaskId)) { + return { error: "Invalid task identifier." }; + } + + const headers = await getAuthHeaders({ contentType: false }); + const url = new URL( + `${apiBaseUrl}${CROSS_PROVIDER_API_PATH}/pdf/${encodeURIComponent(safeTaskId)}`, + ); + + try { + const response = await fetch(url.toString(), { headers }); + + if (response.status === 202) { + const json = await response.json(); + return { + pending: true, + state: json?.data?.attributes?.state, + taskId: json?.data?.id, + }; + } + + if (!response.ok) { + throw new Error( + await getPdfEndpointErrorMessage( + response, + "Unable to retrieve the compliance PDF report. Contact support if the issue continues.", + `GET ${CROSS_PROVIDER_API_PATH}/pdf/{taskId}`, + ), + ); + } + + const contentDisposition = + response.headers.get("content-disposition") || ""; + const filenameMatch = contentDisposition.match(/filename="?([^";]+)"?/i); + const filename = filenameMatch?.[1] || "cross-provider-compliance.pdf"; + + const arrayBuffer = await response.arrayBuffer(); + const base64 = Buffer.from(arrayBuffer).toString("base64"); + + return { success: true, data: base64, filename }; + } catch (error) { + return { error: getErrorMessage(error) }; + } +}; + +/** + * Check whether a cross-provider PDF already exists for the given filters so + * the UI can offer "Download" immediately instead of forcing a re-generate. + * + * 404 means "not generated yet" — a normal state, returned as `null` rather + * than an error. The backend only matches reports built from the exact scan + * set the filters resolve to, so a report goes stale (→ `null`) as soon as a + * contributing provider completes a new scan. Failures also degrade to + * `null`: this is an optional availability check and the caller's fallback + * (show "Generate") is always safe. + */ +export const getLatestCrossProviderPdf = async ({ + complianceId, + filters, +}: { + complianceId: string; + filters?: CrossProviderApiFilters; +}): Promise => { + const headers = await getAuthHeaders({ contentType: false }); + const url = new URL(`${apiBaseUrl}${CROSS_PROVIDER_API_PATH}/pdf/latest`); + url.searchParams.set("filter[compliance_id]", complianceId); + applyFilters(url, filters); + + try { + const response = await fetch(url.toString(), { headers }); + + if (response.status === 404) return null; + + if (!response.ok) { + throw new Error( + await getPdfEndpointErrorMessage( + response, + "Unable to check for an existing PDF report.", + `GET ${CROSS_PROVIDER_API_PATH}/pdf/latest`, + ), + ); + } + + const json = await response.json(); + const taskId = json?.data?.id; + if (!taskId) return null; + + return { + taskId, + filename: json?.data?.attributes?.result?.filename, + completedAt: json?.data?.attributes?.completed_at, + }; + } catch (error) { + // Degraded on purpose, but logged: without this a systematically failing + // endpoint would be indistinguishable from "never generated". + console.error("Error checking for an existing cross-provider PDF:", error); + return null; + } +}; diff --git a/ui/app/(prowler)/compliance/_components/compliance-page-tabs.shared.ts b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.shared.ts new file mode 100644 index 0000000000..7cb6b8cb39 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.shared.ts @@ -0,0 +1,17 @@ +import { COMPLIANCE_TAB, type ComplianceTab } from "../_types"; + +function isComplianceTab(value: string): value is ComplianceTab { + return Object.values(COMPLIANCE_TAB).includes(value as ComplianceTab); +} + +/** Resolves `?tab=` into a valid tab, defaulting to Per Scan so existing + * bookmarks (no query param) keep working. */ +function getComplianceTab(value: string | string[] | undefined): ComplianceTab { + if (typeof value !== "string") { + return COMPLIANCE_TAB.PER_SCAN; + } + + return isComplianceTab(value) ? value : COMPLIANCE_TAB.PER_SCAN; +} + +export { getComplianceTab }; diff --git a/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx new file mode 100644 index 0000000000..b5f25adfe7 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.test.tsx @@ -0,0 +1,83 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { COMPLIANCE_TAB } from "../_types"; +import { CompliancePageTabs } from "./compliance-page-tabs"; +import { getComplianceTab } from "./compliance-page-tabs.shared"; + +const { pushMock } = vi.hoisted(() => ({ + pushMock: vi.fn(), +})); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ + push: pushMock, + }), +})); + +describe("getComplianceTab", () => { + it("falls back to per-scan for missing or invalid values", () => { + expect(getComplianceTab(undefined)).toBe(COMPLIANCE_TAB.PER_SCAN); + expect(getComplianceTab(["cross-provider"])).toBe(COMPLIANCE_TAB.PER_SCAN); + expect(getComplianceTab("bogus")).toBe(COMPLIANCE_TAB.PER_SCAN); + expect(getComplianceTab("cross-provider")).toBe( + COMPLIANCE_TAB.CROSS_PROVIDER, + ); + }); +}); + +describe("CompliancePageTabs", () => { + beforeEach(() => { + pushMock.mockClear(); + }); + + it("navigates with ?tab=cross-provider and back to the bare route", async () => { + const user = userEvent.setup(); + const { rerender } = render( + Per scan content
} + crossProviderContent={
Cross provider content
} + />, + ); + + await user.click(screen.getByRole("tab", { name: /cross-provider/i })); + expect(pushMock).toHaveBeenCalledWith("/compliance?tab=cross-provider"); + + rerender( + Per scan content
} + crossProviderContent={
Cross provider content
} + />, + ); + + await user.click(screen.getByRole("tab", { name: /per scan/i })); + expect(pushMock).toHaveBeenCalledWith("/compliance"); + }); + + it("disables the cross-provider tab with the cloud upsell badge in OSS", () => { + render( + Per scan content} + crossProviderContent={null} + />, + ); + + const crossProviderTab = screen.getByRole("tab", { + name: /cross-provider/i, + }); + const tabLabel = screen.getByText("Cross-Provider", { exact: true }); + const cloudBadge = screen.getByText("Available in Prowler Cloud"); + + expect(crossProviderTab).toBeDisabled(); + expect(crossProviderTab).not.toHaveClass("disabled:opacity-50"); + expect(tabLabel).toHaveClass("opacity-50"); + expect(cloudBadge.parentElement).toHaveClass("gap-2"); + }); +}); diff --git a/ui/app/(prowler)/compliance/_components/compliance-page-tabs.tsx b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.tsx new file mode 100644 index 0000000000..fa6ab6ea70 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/compliance-page-tabs.tsx @@ -0,0 +1,70 @@ +"use client"; + +import { useRouter } from "next/navigation"; +import { ReactNode } from "react"; + +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/shadcn"; +import { CloudFeatureBadge } from "@/components/shared/cloud-feature-badge"; + +import { COMPLIANCE_TAB, type ComplianceTab } from "../_types"; + +interface CompliancePageTabsProps { + activeTab: ComplianceTab; + /** False in OSS: the Cross-Provider tab renders disabled with the + * "Available in Prowler Cloud" upsell badge. */ + crossProviderEnabled: boolean; + perScanContent: ReactNode; + crossProviderContent: ReactNode; +} + +export const CompliancePageTabs = ({ + activeTab, + crossProviderEnabled, + perScanContent, + crossProviderContent, +}: CompliancePageTabsProps) => { + const router = useRouter(); + + const handleTabChange = (tab: string) => { + const typedTab = tab as ComplianceTab; + + if (typedTab === activeTab) { + return; + } + + // Per Scan renders without the query param so existing bookmarks and + // shared links keep resolving to the default view. + if (typedTab === COMPLIANCE_TAB.PER_SCAN) { + router.push("/compliance"); + } else { + router.push(`/compliance?tab=${typedTab}`); + } + }; + + return ( + // Same layout spacing as the scans view tabs (scans-page-shell.tsx). + + + Per Scan + : undefined} + > + Cross-Provider + + + + + {perScanContent} + + + {crossProviderContent} + + + ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx new file mode 100644 index 0000000000..7972379b1f --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-detail.tsx @@ -0,0 +1,241 @@ +import { Info } from "lucide-react"; +import Image from "next/image"; + +import { getAllProviderGroups } from "@/actions/manage-groups/manage-groups"; +import { getAllProviders } from "@/actions/providers"; +import { + ClientAccordionWrapper, + RequirementsStatusCard, + TopFailedSectionsCard, +} from "@/components/compliance"; +import { getComplianceIcon } from "@/components/icons/compliance/IconCompliance"; +import { Alert, AlertDescription } from "@/components/shadcn/alert"; +import { Card } from "@/components/shadcn/card/card"; +import { getComplianceMapper } from "@/lib/compliance/compliance-mapper"; +import type { Framework, RequirementsTotals } from "@/types/compliance"; + +import { + getCrossProviderComplianceOverview, + getLatestCrossProviderPdf, +} from "../_actions/cross-provider"; +import { toCrossProviderAccordionItems } from "../_lib/cross-provider-accordion"; +import { + buildRequirementExtrasMap, + computeProviderBreakdown, + crossProviderToMapperInput, +} from "../_lib/cross-provider-adapter"; +import { + CROSS_PROVIDER_FRAMEWORKS, + parseCrossProviderFilters, +} from "../_lib/cross-provider-frameworks"; +import { CROSS_PROVIDER_OVERVIEW_RESULT_STATUS } from "../_types"; +import { CrossProviderErrorAlert } from "./cross-provider-error-alert"; +import type { + CrossProviderAccountOption, + CrossProviderGroupOption, +} from "./cross-provider-filters"; +import { CrossProviderFilters } from "./cross-provider-filters"; +import { CrossProviderHubLink } from "./cross-provider-hub-link"; +import { CrossProviderPdfButton } from "./cross-provider-pdf-button"; +import { ProviderCoverageCard } from "./provider-coverage-card"; + +interface CrossProviderDetailProps { + compliancetitle: string; + complianceId: string; + searchParams: Record; + targetSection?: string; +} + +/** + * Server island for the cross-provider detail (`?mode=cross-provider`): + * fetches the roll-up, funnels it through the real framework mapper via the + * adapter, and renders the same summary-charts + accordion layout as the + * per-scan detail with per-provider augmentations. + */ +export const CrossProviderDetail = async ({ + compliancetitle, + complianceId, + searchParams, + targetSection, +}: CrossProviderDetailProps) => { + const filters = parseCrossProviderFilters(searchParams); + + const [overviewResponse, providersData, providerGroupsData] = + await Promise.all([ + getCrossProviderComplianceOverview({ complianceId, filters }), + getAllProviders(), + getAllProviderGroups(), + ]); + + if ( + overviewResponse.status === + CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.ACTION_ERROR + ) { + return ; + } + + if ( + overviewResponse.status === CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.LOAD_ERROR + ) { + return ; + } + + const overviewData = overviewResponse.response.data; + + if (!overviewData?.attributes) { + return ( + + + + No cross-provider compliance data was returned for this framework. + Universal frameworks aggregate the latest completed scan of every + compatible provider — run a scan to populate this view. + + + ); + } + + const attrs = overviewData.attributes; + + // Scoped to the EXACT scans the overview resolved (not the raw filters), so + // an offered "Download latest" always matches the data on screen even if a + // provider finished a new scan between the two calls. The overview + // aggregation dominates wall-clock; serializing this quick check is cheap. + const latestPdf = await getLatestCrossProviderPdf({ + complianceId, + filters: { ...filters, scanIds: attrs.scan_ids }, + }); + + const mapper = getComplianceMapper(attrs.framework); + const { attributesData, requirementsData } = + crossProviderToMapperInput(attrs); + const data = mapper.mapComplianceData(attributesData, requirementsData); + const extras = buildRequirementExtrasMap(attrs); + const providerBreakdown = computeProviderBreakdown(attrs); + + const totals: RequirementsTotals = data.reduce( + (acc: RequirementsTotals, framework: Framework) => ({ + pass: acc.pass + framework.pass, + fail: acc.fail + framework.fail, + manual: acc.manual + framework.manual, + }), + { pass: 0, fail: 0, manual: 0 }, + ); + const accordionItems = toCrossProviderAccordionItems( + data, + extras, + attrs.framework, + ); + const topFailedResult = mapper.getTopFailedSections(data); + + // Same `${framework.name}-${category.name}` key scheme as the per-scan + // detail, so ?section= deep links (e.g. from Top Failed Sections) work. + const initialExpandedKeys: string[] = []; + if (targetSection) { + const candidates = new Set( + data.map((framework: Framework) => `${framework.name}-${targetSection}`), + ); + const match = accordionItems.find((item) => candidates.has(item.key)); + if (match) { + initialExpandedKeys.push(match.key); + } + } + + const catalogEntry = CROSS_PROVIDER_FRAMEWORKS.find( + (entry) => entry.complianceId === complianceId, + ); + const compatibleTypes = + catalogEntry?.compatibleProviders ?? + providerBreakdown.map((b) => b.provider); + const logoPath = getComplianceIcon(compliancetitle); + + const providerAccounts: CrossProviderAccountOption[] = ( + providersData?.data || [] + ) + .filter((provider) => + compatibleTypes.some((type) => type === provider.attributes.provider), + ) + .map((provider) => ({ + id: provider.id, + label: provider.attributes.alias + ? `${provider.attributes.alias} (${provider.attributes.uid})` + : provider.attributes.uid, + type: provider.attributes.provider, + })); + + const providerGroups: CrossProviderGroupOption[] = ( + providerGroupsData?.data || [] + ).map((group) => ({ id: group.id, name: group.attributes.name })); + + return ( +
+ {/* Header card — same structure as the per-scan detail: identity row + (logo + context) with the report action top-right, filters below + (lighthouse-settings card pattern). */} + +
+
+
+ {logoPath && ( +
+ {`${compliancetitle} +
+ )} +
+
+ + {attrs.name || compliancetitle.split("-").join(" ")} + + +
+

+ {attrs.providers.length} of {compatibleTypes.length}{" "} + compatible providers scanned · {attrs.scan_ids.length}{" "} + {attrs.scan_ids.length === 1 ? "scan" : "scans"} aggregated +

+
+
+
+ +
+
+ + +
+ + +
+ + + +
+ + +
+ ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-error-alert.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-error-alert.tsx new file mode 100644 index 0000000000..d692ea14b9 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-error-alert.tsx @@ -0,0 +1,38 @@ +import { AlertTriangle } from "lucide-react"; + +import { Alert, AlertDescription } from "@/components/shadcn/alert"; +import { UsageLimitMessage } from "@/components/shared/usage-limit-message"; +import { + ACTION_ERROR_STATUS, + type ActionErrorResult, + getActionErrorMessage, +} from "@/lib/action-errors"; + +import { CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE } from "../_types"; + +interface CrossProviderErrorAlertProps { + result?: ActionErrorResult; + message?: string; +} + +export const CrossProviderErrorAlert = ({ + result, + message = CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE, +}: CrossProviderErrorAlertProps) => { + const isUsageLimit = result?.status === ACTION_ERROR_STATUS.PAYMENT_REQUIRED; + + return ( + + + + {isUsageLimit ? ( + + ) : result ? ( + getActionErrorMessage(result, { fallback: message }) + ) : ( + message + )} + + + ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-filters.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-filters.test.tsx new file mode 100644 index 0000000000..6c27fbed74 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-filters.test.tsx @@ -0,0 +1,72 @@ +import { render, screen } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { CrossProviderFilters } from "./cross-provider-filters"; + +vi.mock("next/navigation", () => ({ + useSearchParams: () => new URLSearchParams(), +})); + +vi.mock("@/hooks/use-url-filters", () => ({ + useUrlFilters: () => ({ updateFilter: vi.fn() }), +})); + +vi.mock("@/components/filters/clear-filters-button", () => ({ + ClearFiltersButton: () => , +})); + +vi.mock("@/components/shadcn/select/multiselect", () => ({ + MultiSelect: ({ children }: { children: ReactNode }) =>
{children}
, + MultiSelectTrigger: ({ + children, + ...props + }: { + children: ReactNode; + "aria-label"?: string; + }) => ( + + ), + MultiSelectValue: ({ placeholder }: { placeholder: string }) => ( + {placeholder} + ), + MultiSelectContent: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + MultiSelectItem: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + MultiSelectSelectAll: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + MultiSelectSeparator: () =>
, +})); + +describe("CrossProviderFilters", () => { + it("shows only the three cross-provider filters with product copy", () => { + // Given / When + render( + , + ); + + // Then + expect( + screen.getByRole("combobox", { name: "Provider type" }), + ).toBeInTheDocument(); + expect( + screen.getByRole("combobox", { name: "Providers" }), + ).toBeInTheDocument(); + expect( + screen.getByRole("combobox", { name: "Provider group" }), + ).toBeInTheDocument(); + expect(screen.getAllByRole("combobox")).toHaveLength(3); + }); +}); diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-filters.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-filters.tsx new file mode 100644 index 0000000000..44c477837b --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-filters.tsx @@ -0,0 +1,119 @@ +"use client"; + +import { useSearchParams } from "next/navigation"; + +import { ClearFiltersButton } from "@/components/filters/clear-filters-button"; +import { + MultiSelect, + MultiSelectContent, + MultiSelectItem, + MultiSelectSelectAll, + MultiSelectSeparator, + MultiSelectTrigger, + MultiSelectValue, +} from "@/components/shadcn/select/multiselect"; +import { useUrlFilters } from "@/hooks/use-url-filters"; +import { + type KnownProviderType, + PROVIDER_DISPLAY_NAMES, + type ProviderType, +} from "@/types/providers"; + +export interface CrossProviderAccountOption { + id: string; + label: string; + type: ProviderType; +} + +export interface CrossProviderGroupOption { + id: string; + name: string; +} + +interface CrossProviderFiltersProps { + /** Provider types offered by the visible universal frameworks. */ + providerTypes: readonly KnownProviderType[]; + providerAccounts: CrossProviderAccountOption[]; + providerGroups: CrossProviderGroupOption[]; +} + +interface UrlMultiSelectOption { + value: string; + label: string; +} + +interface UrlMultiSelectProps { + filterKey: string; + placeholder: string; + options: UrlMultiSelectOption[]; +} + +const UrlMultiSelect = ({ + filterKey, + placeholder, + options, +}: UrlMultiSelectProps) => { + const searchParams = useSearchParams(); + const { updateFilter } = useUrlFilters(); + + const values = + searchParams.get(`filter[${filterKey}]`)?.split(",").filter(Boolean) ?? []; + + return ( +
+ updateFilter(filterKey, nextValues)} + > + + + + 8} width="wide"> + Select All + + {options.map((option) => ( + + {option.label} + + ))} + + +
+ ); +}; + +export const CrossProviderFilters = ({ + providerTypes, + providerAccounts, + providerGroups, +}: CrossProviderFiltersProps) => { + return ( +
+ ({ + value: type, + label: PROVIDER_DISPLAY_NAMES[type], + }))} + /> + ({ + value: account.id, + label: account.label, + }))} + /> + ({ + value: group.id, + label: group.name, + }))} + /> + +
+ ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-framework-card.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-framework-card.tsx new file mode 100644 index 0000000000..65b87806e1 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-framework-card.tsx @@ -0,0 +1,162 @@ +"use client"; + +import Image from "next/image"; +import { useRouter, useSearchParams } from "next/navigation"; + +import { getComplianceIcon } from "@/components/icons/compliance/IconCompliance"; +import { ProviderTypeIcon } from "@/components/icons/providers-badge/provider-type-icon"; +import { Card, CardContent } from "@/components/shadcn/card/card"; +import { Progress } from "@/components/shadcn/progress"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/shadcn/tooltip"; +import { + getScoreIndicatorClass, + type ScoreColorVariant, +} from "@/lib/compliance/score-utils"; +import { cn } from "@/lib/utils"; +import { PROVIDER_DISPLAY_NAMES } from "@/types/providers"; + +import { buildCrossProviderDetailHref } from "../_lib/cross-provider-frameworks"; +import type { CrossProviderFrameworkSummary } from "../_types"; + +export const CrossProviderFrameworkCard = ({ + complianceId, + title, + version, + description, + requirementsPassed, + requirementsFailed, + requirementsManual, + totalRequirements, + providerBreakdown, +}: CrossProviderFrameworkSummary) => { + const router = useRouter(); + const searchParams = useSearchParams(); + + const formattedTitle = `${title.split("-").join(" ")}${version ? ` - ${version}` : ""}`; + + const ratingPercentage = + totalRequirements > 0 + ? Math.floor((requirementsPassed / totalRequirements) * 100) + : 0; + + // Same thresholds as the per-scan ComplianceCard. + const getRatingVariant = (value: number): ScoreColorVariant => { + if (value <= 10) return "danger"; + if (value <= 40) return "warning"; + return "success"; + }; + + const navigateToDetail = () => { + router.push( + buildCrossProviderDetailHref( + { complianceId, title, version }, + Object.fromEntries(searchParams.entries()), + ), + ); + }; + + return ( + { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + navigateToDetail(); + } + }} + > + +
+
+ {getComplianceIcon(title) && ( +
+ {`${title} +
+ )} +
+ + +

+ {formattedTitle} +

+ + {description} + + + + {requirementsPassed} / {totalRequirements} + + Passing Requirements + +
+
+ +
+
+ + Score: + + + {ratingPercentage}% + +
+ +
+ +
+
+ {providerBreakdown.map((entry) => ( + + + + + + + + {PROVIDER_DISPLAY_NAMES[entry.provider]} + {entry.unscanned + ? " — no completed scan yet" + : ` — ${entry.score}% passing`} + + + ))} +
+ + {requirementsFailed} failed · {requirementsManual} manual + +
+
+ + + ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-hub-link.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-hub-link.test.tsx new file mode 100644 index 0000000000..202a92eb28 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-hub-link.test.tsx @@ -0,0 +1,20 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { CrossProviderHubLink } from "./cross-provider-hub-link"; + +describe("CrossProviderHubLink", () => { + it("opens the framework page in Prowler Hub safely", () => { + // Given / When + render(); + + // Then + const link = screen.getByRole("link", { name: /view on prowler hub/i }); + expect(link).toHaveAttribute( + "href", + "https://hub.prowler.com/compliance/cis_controls_8.1", + ); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); +}); diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-hub-link.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-hub-link.tsx new file mode 100644 index 0000000000..d531940fd1 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-hub-link.tsx @@ -0,0 +1,24 @@ +import { SquareArrowOutUpRight } from "lucide-react"; +import Link from "next/link"; + +import { Button } from "@/components/shadcn/button/button"; + +interface CrossProviderHubLinkProps { + complianceId: string; +} + +export const CrossProviderHubLink = ({ + complianceId, +}: CrossProviderHubLinkProps) => ( + +); diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx new file mode 100644 index 0000000000..b2fb3b66f4 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-overview.test.tsx @@ -0,0 +1,137 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { ACTION_ERROR_STATUS, USAGE_LIMIT_MESSAGE } from "@/lib/action-errors"; + +import { getCrossProviderComplianceOverview } from "../_actions/cross-provider"; +import { CROSS_PROVIDER_FRAMEWORKS } from "../_lib/cross-provider-frameworks"; +import type { CrossProviderOverviewResult } from "../_types"; +import { + CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE, + CROSS_PROVIDER_OVERVIEW_RESULT_STATUS, + CROSS_PROVIDER_OVERVIEW_TYPE, +} from "../_types"; +import { CrossProviderOverview } from "./cross-provider-overview"; + +vi.mock("../_actions/cross-provider", () => ({ + getCrossProviderComplianceOverview: vi.fn(), +})); + +vi.mock("@/actions/providers", () => ({ + getAllProviders: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("@/actions/manage-groups/manage-groups", () => ({ + getAllProviderGroups: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("./cross-provider-filters", () => ({ + CrossProviderFilters: () =>
, +})); + +vi.mock("./cross-provider-framework-card", () => ({ + CrossProviderFrameworkCard: ({ title }: { title: string }) => ( +
{title}
+ ), +})); + +const successResult = (complianceId: string): CrossProviderOverviewResult => ({ + status: CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.SUCCESS, + response: { + data: { + type: CROSS_PROVIDER_OVERVIEW_TYPE, + id: complianceId, + attributes: { + compliance_id: complianceId, + framework: complianceId, + name: complianceId, + version: "1.0", + description: "", + compatible_providers: ["aws"], + requested_providers: ["aws"], + providers: ["aws"], + scan_ids: [], + scan_ids_by_provider: {}, + requirements_passed: 1, + requirements_failed: 0, + requirements_manual: 0, + total_requirements: 1, + requirements: [], + }, + }, + }, +}); + +const loadErrorResult: CrossProviderOverviewResult = { + status: CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.LOAD_ERROR, + message: CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE, +}; + +const renderOverview = async () => + render(await CrossProviderOverview({ searchParams: {} })); + +describe("CrossProviderOverview", () => { + beforeEach(() => { + vi.mocked(getCrossProviderComplianceOverview).mockReset(); + }); + + it("degrades to a partial view when a single framework fails to load", async () => { + // Given: DORA fails, the other frameworks load + vi.mocked(getCrossProviderComplianceOverview).mockImplementation( + async ({ complianceId }) => + complianceId === "dora_2022_2554" + ? loadErrorResult + : successResult(complianceId), + ); + + // When + await renderOverview(); + + // Then: loaded cards render, the failed framework is called out by name + expect(screen.getAllByTestId("framework-card")).toHaveLength( + CROSS_PROVIDER_FRAMEWORKS.length - 1, + ); + expect(screen.getByText(/Could not load DORA/)).toBeInTheDocument(); + expect( + screen.queryByText(CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE), + ).not.toBeInTheDocument(); + }); + + it("replaces the tab with the error alert when every framework fails to load", async () => { + // Given + vi.mocked(getCrossProviderComplianceOverview).mockResolvedValue( + loadErrorResult, + ); + + // When + await renderOverview(); + + // Then + expect( + screen.getByText(CROSS_PROVIDER_OVERVIEW_LOAD_ERROR_MESSAGE), + ).toBeInTheDocument(); + expect(screen.queryByTestId("framework-card")).not.toBeInTheDocument(); + }); + + it("gates the whole tab on an action error even if other frameworks loaded", async () => { + // Given: one framework hits the usage limit (402) + vi.mocked(getCrossProviderComplianceOverview).mockImplementation( + async ({ complianceId }) => + complianceId === "dora_2022_2554" + ? { + status: CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.ACTION_ERROR, + result: { status: ACTION_ERROR_STATUS.PAYMENT_REQUIRED }, + } + : successResult(complianceId), + ); + + // When + await renderOverview(); + + // Then + expect( + screen.getByText(new RegExp(USAGE_LIMIT_MESSAGE)), + ).toBeInTheDocument(); + expect(screen.queryByTestId("framework-card")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx new file mode 100644 index 0000000000..17af2b38bf --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-overview.tsx @@ -0,0 +1,189 @@ +import { AlertTriangle, Info } from "lucide-react"; + +import { getAllProviderGroups } from "@/actions/manage-groups/manage-groups"; +import { getAllProviders } from "@/actions/providers"; +import { Alert, AlertDescription } from "@/components/shadcn/alert"; +import { SearchParamsProps } from "@/types"; +import type { KnownProviderType } from "@/types/providers"; + +import { getCrossProviderComplianceOverview } from "../_actions/cross-provider"; +import { computeProviderBreakdown } from "../_lib/cross-provider-adapter"; +import { + CROSS_PROVIDER_FRAMEWORKS, + type CrossProviderFrameworkEntry, + parseCrossProviderFilters, +} from "../_lib/cross-provider-frameworks"; +import type { CrossProviderFrameworkSummary } from "../_types"; +import { CROSS_PROVIDER_OVERVIEW_RESULT_STATUS } from "../_types"; +import { CrossProviderErrorAlert } from "./cross-provider-error-alert"; +import type { + CrossProviderAccountOption, + CrossProviderGroupOption, +} from "./cross-provider-filters"; +import { CrossProviderFilters } from "./cross-provider-filters"; +import { CrossProviderFrameworkCard } from "./cross-provider-framework-card"; + +/** Zero-state summary: the framework renders with every compatible provider + * chip dimmed when the API returned nothing usable (e.g. no scans yet). */ +const emptySummary = ( + entry: CrossProviderFrameworkEntry, +): CrossProviderFrameworkSummary => ({ + complianceId: entry.complianceId, + title: entry.title, + version: entry.version, + description: entry.description, + requirementsPassed: 0, + requirementsFailed: 0, + requirementsManual: 0, + totalRequirements: 0, + providerBreakdown: entry.compatibleProviders.map((provider) => ({ + provider, + pass: 0, + fail: 0, + manual: 0, + total: 0, + score: 0, + unscanned: true, + })), +}); + +/** + * Server island for the Cross-Provider tab: fetches the roll-up for every + * catalog framework in parallel and renders the filter row plus the cards + * grid. Rendered only in Prowler Cloud with the tab active, so OSS and the + * Per Scan tab never pay for these aggregation calls. + */ +export const CrossProviderOverview = async ({ + searchParams, +}: { + searchParams: SearchParamsProps; +}) => { + const filters = parseCrossProviderFilters(searchParams); + + const [responses, providersData, providerGroupsData] = await Promise.all([ + Promise.all( + CROSS_PROVIDER_FRAMEWORKS.map((entry) => + getCrossProviderComplianceOverview({ + complianceId: entry.complianceId, + filters, + }).then((result) => ({ entry, result })), + ), + ), + getAllProviders(), + getAllProviderGroups(), + ]); + + // Action errors (402 usage limit, 403) gate the whole feature, not one + // framework, so any of them replaces the tab instead of degrading it. + const actionError = responses.find( + ({ result }) => + result.status === CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.ACTION_ERROR, + ); + if ( + actionError?.result.status === + CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.ACTION_ERROR + ) { + return ; + } + + // Load errors are per-framework and often transient: degrade to a partial + // view with a warning, and only replace the tab when nothing loaded. + const loadErrors = responses.flatMap(({ entry, result }) => + result.status === CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.LOAD_ERROR + ? [{ entry, result }] + : [], + ); + if (loadErrors.length === responses.length && loadErrors.length > 0) { + return ; + } + + const summaries: CrossProviderFrameworkSummary[] = responses + .filter( + ({ result }) => + result.status !== CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.LOAD_ERROR, + ) + .map(({ entry, result }) => { + if (result.status !== CROSS_PROVIDER_OVERVIEW_RESULT_STATUS.SUCCESS) { + return emptySummary(entry); + } + + const data = result.response.data; + if (!data?.attributes) return emptySummary(entry); + + const attrs = data.attributes; + return { + complianceId: entry.complianceId, + title: entry.title, + version: entry.version, + description: entry.description, + requirementsPassed: attrs.requirements_passed, + requirementsFailed: attrs.requirements_failed, + requirementsManual: attrs.requirements_manual, + totalRequirements: attrs.total_requirements, + providerBreakdown: computeProviderBreakdown(attrs), + }; + }); + + const compatibleTypes = Array.from( + new Set( + CROSS_PROVIDER_FRAMEWORKS.flatMap((entry) => entry.compatibleProviders), + ), + ).sort(); + + const providerAccounts: CrossProviderAccountOption[] = ( + providersData?.data || [] + ) + .filter((provider) => + compatibleTypes.some((type) => type === provider.attributes.provider), + ) + .map((provider) => ({ + id: provider.id, + label: provider.attributes.alias + ? `${provider.attributes.alias} (${provider.attributes.uid})` + : provider.attributes.uid, + type: provider.attributes.provider, + })); + + const providerGroups: CrossProviderGroupOption[] = ( + providerGroupsData?.data || [] + ).map((group) => ({ id: group.id, name: group.attributes.name })); + + return ( +
+ + + {loadErrors.length > 0 && ( + + + + Could not load{" "} + {loadErrors.map(({ entry }) => entry.title).join(", ")}. Showing the + frameworks that loaded — try again later. + + + )} + + {loadErrors.length === 0 && + summaries.every((summary) => summary.totalRequirements === 0) && ( + + + + No cross-provider compliance data yet. Universal frameworks + aggregate the latest completed scan of every compatible provider — + run a scan to populate these cards. + + + )} + +
+ {summaries.map((summary) => ( + + ))} +
+
+ ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-pdf-button.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-pdf-button.test.tsx new file mode 100644 index 0000000000..2f04c202db --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-pdf-button.test.tsx @@ -0,0 +1,233 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { CrossProviderPdfButton } from "./cross-provider-pdf-button"; + +// Radix dialogs/dropdowns rely on pointer-capture and scrollIntoView, which +// jsdom does not implement. +beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "hasPointerCapture", { + configurable: true, + value: vi.fn(() => false), + }); + Object.defineProperty(HTMLElement.prototype, "setPointerCapture", { + configurable: true, + value: vi.fn(), + }); + Object.defineProperty(HTMLElement.prototype, "releasePointerCapture", { + configurable: true, + value: vi.fn(), + }); + Object.defineProperty(HTMLElement.prototype, "scrollIntoView", { + configurable: true, + value: vi.fn(), + }); +}); + +const { + generatePdfMock, + trackAndPollMock, + downloadPdfMock, + toastMock, + storeState, +} = vi.hoisted(() => ({ + generatePdfMock: vi.fn(), + trackAndPollMock: vi.fn(), + downloadPdfMock: vi.fn(), + toastMock: vi.fn(), + storeState: { + tasks: {} as Record< + string, + { + taskId: string; + kind: string; + status: string; + meta: Record; + startedAt: number; + } + >, + }, +})); + +vi.mock("../_actions/cross-provider", () => ({ + generateCrossProviderPdf: generatePdfMock, +})); + +vi.mock("../_lib/cross-provider-pdf", () => ({ + CROSS_PROVIDER_PDF_TASK_KIND: "cross-provider-pdf", + buildCrossProviderPdfTaskScope: vi.fn(() => "scope-1"), + downloadCrossProviderPdf: downloadPdfMock, + crossProviderPdfHandler: { onReady: vi.fn(), onError: vi.fn() }, +})); + +vi.mock("@/store/task-watcher/store", () => ({ + TASK_WATCHER_STATUS: { PENDING: "pending", READY: "ready", ERROR: "error" }, + trackAndPollTask: trackAndPollMock, + useTaskWatcherStore: (selector: (state: typeof storeState) => unknown) => + selector(storeState), +})); + +vi.mock("@/components/shadcn/toast", () => ({ + toast: toastMock, + ToastAction: () => null, +})); + +const props = { + complianceId: "csa_ccm_4.0", + filters: { scanIds: ["scan-1"], providerTypes: "aws" }, + latestPdf: null, +}; + +describe("CrossProviderPdfButton", () => { + beforeEach(() => { + vi.clearAllMocks(); + storeState.tasks = {}; + generatePdfMock.mockResolvedValue({ taskId: "task-1" }); + }); + + const openGenerateModal = async ( + user: ReturnType, + ) => { + await user.click(screen.getByRole("button", { name: /report/i })); + await user.click( + await screen.findByRole("menuitem", { name: /generate new report/i }), + ); + }; + + it("generates a named report without unsupported requirement options", async () => { + // Given + const user = userEvent.setup(); + render(); + + // When + await openGenerateModal(user); + await user.type( + screen.getByLabelText(/report name/i), + "quarterly-audit.pdf", + ); + expect(screen.queryByLabelText(/only failed/i)).not.toBeInTheDocument(); + expect(screen.queryByLabelText(/include manual/i)).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /^generate$/i })); + + // Then + await waitFor(() => expect(generatePdfMock).toHaveBeenCalledTimes(1)); + expect(generatePdfMock).toHaveBeenCalledWith({ + complianceId: "csa_ccm_4.0", + filters: props.filters, + reportName: "quarterly-audit.pdf", + }); + expect(trackAndPollMock).toHaveBeenCalledWith({ + taskId: "task-1", + kind: "cross-provider-pdf", + meta: expect.objectContaining({ complianceId: "csa_ccm_4.0" }), + }); + }); + + it("surfaces generation errors as a destructive toast without tracking", async () => { + generatePdfMock.mockResolvedValue({ error: "No compatible scans." }); + const user = userEvent.setup(); + render(); + + await openGenerateModal(user); + await user.click(screen.getByRole("button", { name: /^generate$/i })); + + await waitFor(() => + expect(toastMock).toHaveBeenCalledWith( + expect.objectContaining({ variant: "destructive" }), + ), + ); + expect(trackAndPollMock).not.toHaveBeenCalled(); + }); + + it("shows a generating state while a task for this framework is pending", () => { + storeState.tasks = { + "task-9": { + taskId: "task-9", + kind: "cross-provider-pdf", + status: "pending", + meta: { complianceId: "csa_ccm_4.0", scopeKey: "scope-1" }, + startedAt: Date.now(), + }, + }; + + render(); + + expect(screen.getByRole("button", { name: /generating/i })).toBeDisabled(); + }); + + it("offers an instant download when a matching report already exists", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByRole("button", { name: /report/i })); + await user.click( + await screen.findByRole("menuitem", { name: /download latest/i }), + ); + + await waitFor(() => expect(downloadPdfMock).toHaveBeenCalledWith("task-7")); + }); + + it("keeps a completed report downloadable after the ready toast closes", async () => { + // Given + storeState.tasks = { + "task-8": { + taskId: "task-8", + kind: "cross-provider-pdf", + status: "ready", + meta: { + complianceId: "csa_ccm_4.0", + scopeKey: "scope-1", + reportLabel: "quarterly-audit.pdf", + }, + startedAt: Date.now(), + }, + }; + const user = userEvent.setup(); + render(); + + // When + await user.click(screen.getByRole("button", { name: /report/i })); + await user.click( + await screen.findByRole("menuitem", { name: /download latest/i }), + ); + + // Then + await waitFor(() => expect(downloadPdfMock).toHaveBeenCalledWith("task-8")); + }); + + it("does not offer a completed report from a different filter scope", async () => { + // Given + storeState.tasks = { + "task-other-scope": { + taskId: "task-other-scope", + kind: "cross-provider-pdf", + status: "ready", + meta: { + complianceId: "csa_ccm_4.0", + scopeKey: "another-scope", + }, + startedAt: Date.now(), + }, + }; + const user = userEvent.setup(); + render(); + + // When + await user.click(screen.getByRole("button", { name: /report/i })); + + // Then + expect( + screen.queryByRole("menuitem", { name: /download latest/i }), + ).not.toBeInTheDocument(); + }); +}); diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-pdf-button.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-pdf-button.tsx new file mode 100644 index 0000000000..163ef91ed0 --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-pdf-button.tsx @@ -0,0 +1,208 @@ +"use client"; + +import { ChevronDownIcon, DownloadIcon, FileTextIcon } from "lucide-react"; +import { useState } from "react"; + +import { Button, Label } from "@/components/shadcn"; +import { + ActionDropdown, + ActionDropdownItem, +} from "@/components/shadcn/dropdown"; +import { FormButtons } from "@/components/shadcn/form"; +import { Input } from "@/components/shadcn/input/input"; +import { Modal } from "@/components/shadcn/modal"; +import { toast } from "@/components/shadcn/toast"; +import { + TASK_WATCHER_STATUS, + trackAndPollTask, + useTaskWatcherStore, +} from "@/store/task-watcher/store"; + +import { generateCrossProviderPdf } from "../_actions/cross-provider"; +import { + buildCrossProviderPdfTaskScope, + CROSS_PROVIDER_PDF_TASK_KIND, + downloadCrossProviderPdf, +} from "../_lib/cross-provider-pdf"; +import type { + CrossProviderApiFilters, + LatestCrossProviderPdf, +} from "../_types"; + +interface CrossProviderPdfButtonProps { + complianceId: string; + /** The filters (and exact scan ids) of the view currently on screen, so + * the generated PDF matches what the user is looking at. */ + filters: CrossProviderApiFilters; + /** Already-generated report matching these filters, if any — offered as an + * instant download instead of forcing a re-generate. */ + latestPdf: LatestCrossProviderPdf | null; +} + +export const CrossProviderPdfButton = ({ + complianceId, + filters, + latestPdf, +}: CrossProviderPdfButtonProps) => { + const [dialogOpen, setDialogOpen] = useState(false); + const [reportName, setReportName] = useState(""); + const [submitting, setSubmitting] = useState(false); + const taskScope = buildCrossProviderPdfTaskScope(complianceId, filters); + + const isGenerating = useTaskWatcherStore((state) => + Object.values(state.tasks).some( + (task) => + task.kind === CROSS_PROVIDER_PDF_TASK_KIND && + task.status === TASK_WATCHER_STATUS.PENDING && + task.meta.scopeKey === taskScope, + ), + ); + const completedTask = useTaskWatcherStore((state) => + Object.values(state.tasks).reduce<(typeof state.tasks)[string] | undefined>( + (latest, task) => { + if ( + task.kind !== CROSS_PROVIDER_PDF_TASK_KIND || + task.status !== TASK_WATCHER_STATUS.READY || + task.meta.scopeKey !== taskScope + ) { + return latest; + } + + return !latest || task.startedAt > latest.startedAt ? task : latest; + }, + undefined, + ), + ); + const availablePdf: LatestCrossProviderPdf | null = completedTask + ? { + taskId: completedTask.taskId, + filename: completedTask.meta.reportLabel, + } + : latestPdf; + + const handleGenerate = async () => { + setSubmitting(true); + try { + const result = await generateCrossProviderPdf({ + complianceId, + filters, + reportName: reportName.trim() || undefined, + }); + + if ("error" in result) { + toast({ + variant: "destructive", + title: "Could not start report generation", + description: result.error, + }); + return; + } + + setDialogOpen(false); + toast({ + title: "Report generation started", + description: + "We'll let you know when the PDF is ready — you can keep working meanwhile.", + }); + await trackAndPollTask({ + taskId: result.taskId, + kind: CROSS_PROVIDER_PDF_TASK_KIND, + meta: { + complianceId, + scopeKey: taskScope, + ...(reportName.trim() ? { reportLabel: reportName.trim() } : {}), + }, + }); + } catch { + // The action returns {error} for API failures; this guards the + // server-action RPC itself (e.g. a network drop mid-request). + toast({ + variant: "destructive", + title: "Could not start report generation", + description: "An unexpected error occurred. Please try again later.", + }); + } finally { + setSubmitting(false); + } + }; + + const formatGeneratedAt = (completedAt?: string) => { + if (!completedAt) return ""; + const date = new Date(completedAt); + return Number.isNaN(date.getTime()) + ? "" + : ` (${date.toLocaleDateString()})`; + }; + + return ( + <> + {/* Same trigger as the per-scan detail's export dropdown, so both + compliance headers expose one consistent "Report" action. */} + {isGenerating ? ( + + ) : ( + + Report + + + } + > + {availablePdf && ( + } + label={`Download latest${formatGeneratedAt(availablePdf.completedAt)}`} + description={availablePdf.filename} + onSelect={() => downloadCrossProviderPdf(availablePdf.taskId)} + /> + )} + } + label="Generate new report…" + onSelect={() => setDialogOpen(true)} + /> + + )} + + +
{ + event.preventDefault(); + handleGenerate(); + }} + className="flex flex-col gap-6" + > +
+
+ + setReportName(event.target.value)} + /> +
+
+ + setDialogOpen(false)} + submitText="Generate" + loadingText="Starting..." + isDisabled={submitting} + /> + +
+ + ); +}; diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx new file mode 100644 index 0000000000..98a9f11aab --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.test.tsx @@ -0,0 +1,120 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { CheckProviderTypesMap, Requirement } from "@/types/compliance"; + +import type { CrossProviderRequirementExtras } from "../_types"; +import { CrossProviderRequirementContent } from "./cross-provider-requirement-content"; + +const { clientAccordionContentMock } = vi.hoisted(() => ({ + clientAccordionContentMock: vi.fn( + ({ + requirement, + scanIds, + }: { + requirement: Requirement; + scanIds: string[]; + framework: string; + checkProviders: CheckProviderTypesMap; + disableFindings?: boolean; + }) => ( +
+ {scanIds.join("|")}:{requirement.check_ids.join("|")}: + {requirement.status} +
+ ), + ), +})); + +// The real ClientAccordionContent drags the findings/server-action chain into +// jsdom; its behavior has its own tests. Here we assert the combined-view +// composition around it. +vi.mock( + "@/components/compliance/compliance-accordion/client-accordion-content", + () => ({ ClientAccordionContent: clientAccordionContentMock }), +); + +const mappedRequirement: Requirement = { + name: "A&A-01 - Audit and Assurance Policy and Procedures", + description: "Establish audit policies.", + status: "FAIL", + pass: 0, + fail: 1, + manual: 0, + check_ids: ["aws_check", "azure_check", "shared_check"], + scope_applicability: "IaaS", +}; + +const extras: CrossProviderRequirementExtras = { + requirementId: "A&A-01", + providers: { aws: "FAIL", azure: "PASS" }, + checkIdsByProvider: { + aws: ["aws_check", "shared_check"], + azure: ["azure_check", "shared_check"], + }, + scanIdsByProvider: { + aws: ["scan-aws-1", "scan-aws-2"], + azure: ["scan-azure-1"], + }, +}; + +describe("CrossProviderRequirementContent", () => { + it("renders the requirement once with all contributing scans combined", () => { + clientAccordionContentMock.mockClear(); + render( + , + ); + + // One combined block — no per-provider sections repeating the detail. + expect(clientAccordionContentMock).toHaveBeenCalledTimes(1); + expect(screen.queryByText(/account 1 of/)).not.toBeInTheDocument(); + expect(screen.getByTestId("findings-content")).toHaveTextContent( + "scan-aws-1|scan-aws-2|scan-azure-1:aws_check|azure_check|shared_check:FAIL", + ); + + // The mapped requirement passes through untouched (union checks, + // roll-up status, detail fields for getDetailsComponent). + const call = clientAccordionContentMock.mock.calls.at(-1)?.[0]; + expect(call?.requirement).toBe(mappedRequirement); + expect(call?.framework).toBe("CSA-CCM"); + expect(call?.disableFindings).toBe(false); + }); + + it("disables findings when no provider contributed any check", () => { + clientAccordionContentMock.mockClear(); + render( + , + ); + + const call = clientAccordionContentMock.mock.calls.at(-1)?.[0]; + expect(call?.disableFindings).toBe(true); + }); + + it("shows an empty state when no provider scan contributed", () => { + clientAccordionContentMock.mockClear(); + render( + , + ); + + expect( + screen.getByText(/No provider scan contributed/), + ).toBeInTheDocument(); + expect(clientAccordionContentMock).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.tsx b/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.tsx new file mode 100644 index 0000000000..e0d05f8f4d --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/cross-provider-requirement-content.tsx @@ -0,0 +1,58 @@ +"use client"; + +import { ClientAccordionContent } from "@/components/compliance/compliance-accordion/client-accordion-content"; +import type { Requirement } from "@/types/compliance"; +import { PROVIDER_TYPES } from "@/types/providers"; + +import { invertCheckIdsByProvider } from "../_lib/cross-provider-adapter"; +import type { CrossProviderRequirementExtras } from "../_types"; + +interface CrossProviderRequirementContentProps { + /** The requirement as produced by the framework mapper (roll-up level). */ + requirement: Requirement; + extras: CrossProviderRequirementExtras; + framework: string; +} + +/** + * Combined findings view for a cross-provider requirement: the requirement + * detail rendered once, one checks list labeling each check with its provider + * types, and a single findings table querying every contributing scan at once + * (each row carries its own provider column). Mounts lazily — the requirement + * accordion unmounts collapsed content, so the combined fetch only fires on + * expand. + */ +export const CrossProviderRequirementContent = ({ + requirement, + extras, + framework, +}: CrossProviderRequirementContentProps) => { + const contributingTypes = PROVIDER_TYPES.filter( + (type) => extras.providers[type], + ); + + if (contributingTypes.length === 0) { + return ( +

+ No provider scan contributed to this requirement with the current + filters. +

+ ); + } + + const scanIds = Array.from( + new Set( + contributingTypes.flatMap((type) => extras.scanIdsByProvider[type] ?? []), + ), + ); + + return ( + + ); +}; diff --git a/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx b/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx new file mode 100644 index 0000000000..d1f73d264a --- /dev/null +++ b/ui/app/(prowler)/compliance/_components/provider-coverage-card.test.tsx @@ -0,0 +1,53 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { ProviderBreakdownEntry } from "../_types"; +import { ProviderCoverageCard } from "./provider-coverage-card"; + +vi.mock("@/components/icons/providers-badge/provider-type-icon", () => ({ + ProviderTypeIcon: () =>