fix(sdk): redact secret fields in resource metadata (#13129)

This commit is contained in:
Pedro Martín authored and GitHub committed 2026-10-23 08:37:27 +02:00
1 parent 69b80f6a3f
commit 7d71220a25
26 files changed
+329 -18

No files matched your search

@@ -0,0 +1 @@
Resource metadata attached to findings no longer carries the values that secrets checks scan (CloudFormation outputs, Lambda environment variables, EC2 user data and similar fields); they are replaced with `REDACTED` before reaching the Evidence tab, the API and the output files
+2
View File
@@ -12,6 +12,7 @@ from pydantic.v1 import BaseModel, Field, ValidationError, validator
from pydantic.v1.error_wrappers import ErrorWrapper
from prowler.lib.check.compliance_models import Compliance
from prowler.lib.check.redact import redact_sensitive_fields
from prowler.lib.check.utils import recover_checks_from_provider
from prowler.lib.logger import logger
from prowler.providers.common.provider import Provider as ProviderABC
@@ -747,6 +748,7 @@ class Check_Report:
f"Resource metadata {type(resource)} in {self.check_metadata.CheckID} could not be converted to dict"
)
self.resource = {}
self.resource = redact_sensitive_fields(resource, self.resource)
self.status_extended = ""
self.resource_details = ""
self.resource_tags = getattr(resource, "tags", []) if resource else []
+43
View File
@@ -0,0 +1,43 @@
from typing import Any
REDACTED_VALUE = "REDACTED"
def _mask(value: Any) -> Any:
"""Replace every leaf of value with REDACTED_VALUE, keeping dict keys and list lengths."""
if value is None or value == "" or value == [] or value == {}:
return value
if isinstance(value, dict):
return {key: _mask(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_mask(item) for item in value]
return REDACTED_VALUE
def redact_sensitive_fields(resource: Any, data: Any) -> Any:
"""Return data, the serialised form of resource, with the fields each model lists in sensitive_fields masked."""
if isinstance(data, dict):
if isinstance(resource, dict):
return {
key: redact_sensitive_fields(resource.get(key), item)
for key, item in data.items()
}
sensitive = getattr(type(resource), "sensitive_fields", ())
return {
key: (
_mask(item)
if key in sensitive
else redact_sensitive_fields(getattr(resource, key, None), item)
)
for key, item in data.items()
}
if (
isinstance(data, list)
and isinstance(resource, (list, tuple))
and len(data) == len(resource)
):
return [
redact_sensitive_fields(item_resource, item)
for item_resource, item in zip(resource, data)
]
return data
@@ -1,3 +1,5 @@
from typing import ClassVar
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel, Field
@@ -11,6 +13,7 @@ class Branch(BaseModel):
name: str
arn: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("environment_variables",)
environment_variables: dict = Field(default_factory=dict)
@@ -21,6 +24,10 @@ class App(BaseModel):
name: str
arn: str
region: str
sensitive_fields: ClassVar[tuple[str, ...]] = (
"environment_variables",
"build_spec",
)
environment_variables: dict = Field(default_factory=dict)
build_spec: str = ""
branches: list[Branch] = Field(default_factory=list)
@@ -1,4 +1,4 @@
from typing import Any, Optional
from typing import Any, ClassVar, Optional
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel
@@ -266,6 +266,7 @@ class Stage(BaseModel):
tracing_enabled: Optional[bool] = None
cache_enabled: Optional[bool] = None
cache_data_encrypted: Optional[bool] = None
sensitive_fields: ClassVar[tuple[str, ...]] = ("variables",)
variables: Optional[dict] = {}
@@ -1,3 +1,5 @@
from typing import ClassVar
from pydantic.v1 import BaseModel
from prowler.lib.logger import logger
@@ -155,6 +157,7 @@ class ApplicationAutoScaling(AWSService):
class LaunchConfiguration(BaseModel):
arn: str
name: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("user_data",)
user_data: str
image_id: str
region: str
@@ -3,7 +3,7 @@ import json
import zipfile
from concurrent.futures import as_completed
from enum import Enum
from typing import Any, Optional
from typing import Any, ClassVar, Optional
import requests
from botocore.client import ClientError
@@ -404,6 +404,7 @@ class Function(BaseModel):
security_groups: list
last_modified: Optional[str] = None
runtime: Optional[str] = None
sensitive_fields: ClassVar[tuple[str, ...]] = ("environment",)
environment: Optional[dict] = None
region: str
policy: dict = {}
@@ -1,5 +1,5 @@
from itertools import zip_longest
from typing import Optional
from typing import ClassVar, Optional
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel
@@ -14,6 +14,7 @@ class ContainerEnvVariable(BaseModel):
"""An environment variable name-value pair."""
name: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("value",)
value: str
@@ -21,6 +22,7 @@ class BatchContainerProperties(BaseModel):
"""Container properties for an AWS Batch job definition."""
image: Optional[str]
sensitive_fields: ClassVar[tuple[str, ...]] = ("command",)
command: list[str] = []
environment: list[ContainerEnvVariable] = []
@@ -1,4 +1,4 @@
from typing import Optional
from typing import ClassVar, Optional
from botocore.client import ClientError
from pydantic.v1 import BaseModel
@@ -82,6 +82,7 @@ class Stack(BaseModel):
"""In the CloudFormation API the "Stacks[].StackId" is the ARN"""
name: str
"""Stacks[].StackName"""
sensitive_fields: ClassVar[tuple[str, ...]] = ("outputs",)
outputs: list[str]
"""Stacks[].Outputs"""
enable_termination_protection: bool = False
@@ -1,6 +1,6 @@
import json
from datetime import datetime, timezone
from typing import Optional
from typing import ClassVar, Optional
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel
@@ -381,6 +381,7 @@ class LogGroup(BaseModel):
# Properties inherited from account-level settings, e.g. ACCOUNT_DATA_PROTECTION.
inherited_properties: list[str] = []
region: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("log_streams",)
log_streams: dict[str, list[str]] = (
{}
) # Log stream name as the key, array of events as the value
@@ -1,6 +1,6 @@
import datetime
from concurrent.futures import as_completed
from typing import List, Optional
from typing import ClassVar, List, Optional
from pydantic.v1 import BaseModel
@@ -286,6 +286,7 @@ class Source(BaseModel):
class EnvironmentVariable(BaseModel):
name: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("value",)
value: str
type: str
@@ -1,3 +1,5 @@
from typing import ClassVar
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel, Field
@@ -13,6 +15,7 @@ class Pipeline(BaseModel):
name: str
arn: str
region: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("definition",)
definition: dict = Field(default_factory=dict)
tags: list[dict] = Field(default_factory=list)
@@ -1,6 +1,6 @@
from datetime import datetime
from ipaddress import IPv4Address, IPv6Address, ip_address
from typing import Optional, Union
from typing import ClassVar, Optional, Union
from botocore.client import ClientError
from pydantic.v1 import BaseModel
@@ -776,6 +776,7 @@ class Instance(BaseModel):
private_ip: Optional[str]
public_dns: Optional[str]
public_ip: Optional[str]
sensitive_fields: ClassVar[tuple[str, ...]] = ("user_data",)
user_data: Optional[str]
http_tokens: Optional[str]
http_endpoint: Optional[str]
@@ -898,6 +899,7 @@ class InstanceMetadataDefaults(BaseModel):
class TemplateData(BaseModel):
sensitive_fields: ClassVar[tuple[str, ...]] = ("user_data",)
user_data: str
network_interfaces: Optional[list[NetworkInterface]]
associate_public_ip_address: Optional[bool]
@@ -1,7 +1,7 @@
from datetime import datetime
from itertools import zip_longest
from re import sub
from typing import Optional
from typing import ClassVar, Optional
from pydantic.v1 import BaseModel
@@ -286,6 +286,7 @@ class ECS(AWSService):
class ContainerEnvVariable(BaseModel):
name: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("value",)
value: str
@@ -1,4 +1,4 @@
from typing import Optional
from typing import ClassVar, Optional
from botocore.client import ClientError
from pydantic.v1 import BaseModel
@@ -125,4 +125,5 @@ class Environment(BaseModel):
managed_platform_updates: Optional[str]
cloudwatch_stream_logs: Optional[str]
tags: Optional[list] = []
sensitive_fields: ClassVar[tuple[str, ...]] = ("option_settings",)
option_settings: Optional[list] = None
@@ -1,5 +1,5 @@
import json
from typing import Dict, List, Optional
from typing import ClassVar, Dict, List, Optional
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel, Field
@@ -275,6 +275,7 @@ class Connection(BaseModel):
name: str
arn: str
type: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("properties",)
properties: dict
region: str
tags: Optional[List[Dict[str, str]]] = Field(default_factory=list)
@@ -308,6 +309,7 @@ class Job(BaseModel):
arn: str
name: str
security: Optional[str]
sensitive_fields: ClassVar[tuple[str, ...]] = ("arguments",)
arguments: Optional[Dict[str, str]] = Field(default_factory=dict)
region: str
tags: Optional[List[Dict[str, str]]] = Field(default_factory=list)
@@ -1,5 +1,5 @@
import base64
from typing import Optional
from typing import ClassVar, Optional
from botocore.client import ClientError
from pydantic.v1 import BaseModel
@@ -594,6 +594,7 @@ class NotebookInstance(BaseModel):
lifecycle_config_name: str = None
# Decoded lifecycle scripts keyed by "<hook>[<index>]" (e.g. "OnStart[0]"),
# populated by _describe_notebook_instance_lifecycle_config.
sensitive_fields: ClassVar[tuple[str, ...]] = ("lifecycle_scripts",)
lifecycle_scripts: dict = {}
# True if the lifecycle configuration could not be fully described/decoded,
# so the secrets check reports MANUAL instead of a false PASS.
@@ -1,7 +1,7 @@
import json
import time
from enum import Enum
from typing import Optional
from typing import ClassVar, Optional
from botocore.client import ClientError
from pydantic.v1 import BaseModel
@@ -194,6 +194,7 @@ class Document(BaseModel):
arn: str
name: str
region: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("content",)
content: dict = None
account_owners: list[str] = None
tags: Optional[list] = []
@@ -1,6 +1,6 @@
from datetime import datetime
from enum import Enum
from typing import Dict, List, Optional
from typing import ClassVar, Dict, List, Optional
from botocore.exceptions import ClientError
from pydantic.v1 import BaseModel, Field
@@ -130,6 +130,7 @@ class StateMachine(BaseModel):
arn: str
name: Optional[str] = None
status: StateMachineStatus
sensitive_fields: ClassVar[tuple[str, ...]] = ("definition",)
definition: Optional[str] = None
role_arn: Optional[str] = None
type: StateMachineType
@@ -1,7 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Dict, List
from typing import ClassVar, Dict, List
from openstack import exceptions as openstack_exceptions
@@ -135,6 +135,7 @@ class VolumeResource:
is_bootable: bool
is_multiattach: bool
attachments: List[Dict]
sensitive_fields: ClassVar[tuple[str, ...]] = ("metadata",)
metadata: Dict[str, str]
availability_zone: str
snapshot_id: str
@@ -152,6 +153,7 @@ class SnapshotResource:
status: str
size: int
volume_id: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("metadata",)
metadata: Dict[str, str]
project_id: str
region: str
@@ -2,7 +2,7 @@ from __future__ import annotations
import ipaddress
from dataclasses import dataclass
from typing import Dict, List
from typing import ClassVar, Dict, List
from openstack import exceptions as openstack_exceptions
@@ -169,6 +169,7 @@ class ComputeInstance:
# Configuration Security
has_config_drive: bool
sensitive_fields: ClassVar[tuple[str, ...]] = ("metadata", "user_data")
metadata: Dict[str, str]
user_data: str
@@ -1,7 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Dict, List
from typing import ClassVar, Dict, List
from openstack import exceptions as openstack_exceptions
@@ -89,4 +89,5 @@ class ObjectStorageContainer:
history_location: str
sync_to: str
sync_key: str
sensitive_fields: ClassVar[tuple[str, ...]] = ("metadata",)
metadata: Dict[str, str]
+224
View File
@@ -0,0 +1,224 @@
import json
from dataclasses import dataclass, fields, is_dataclass
from pathlib import Path
from typing import ClassVar, Optional
import pytest
from pydantic.v1 import BaseModel
from prowler.lib.check.models import Check_Report_AWS, CheckReportOpenStack
from prowler.lib.check.redact import REDACTED_VALUE, redact_sensitive_fields
from prowler.providers.aws.services.amplify.amplify_service import App, Branch
from prowler.providers.aws.services.apigateway.apigateway_service import Stage
from prowler.providers.aws.services.autoscaling.autoscaling_service import (
LaunchConfiguration,
)
from prowler.providers.aws.services.awslambda.awslambda_service import Function
from prowler.providers.aws.services.batch.batch_service import (
BatchContainerProperties,
)
from prowler.providers.aws.services.batch.batch_service import (
ContainerEnvVariable as BatchContainerEnvVariable,
)
from prowler.providers.aws.services.cloudformation.cloudformation_service import Stack
from prowler.providers.aws.services.cloudwatch.cloudwatch_service import LogGroup
from prowler.providers.aws.services.codebuild.codebuild_service import (
EnvironmentVariable,
)
from prowler.providers.aws.services.datapipeline.datapipeline_service import Pipeline
from prowler.providers.aws.services.ec2.ec2_service import Instance, TemplateData
from prowler.providers.aws.services.ecs.ecs_service import (
ContainerEnvVariable as EcsContainerEnvVariable,
)
from prowler.providers.aws.services.elasticbeanstalk.elasticbeanstalk_service import (
Environment,
)
from prowler.providers.aws.services.glue.glue_service import Connection, Job
from prowler.providers.aws.services.sagemaker.sagemaker_service import (
NotebookInstance,
)
from prowler.providers.aws.services.ssm.ssm_service import Document
from prowler.providers.aws.services.stepfunctions.stepfunctions_service import (
StateMachine,
)
from prowler.providers.openstack.services.blockstorage.blockstorage_service import (
SnapshotResource,
VolumeResource,
)
from prowler.providers.openstack.services.compute.compute_service import (
ComputeInstance,
)
from prowler.providers.openstack.services.objectstorage.objectstorage_service import (
ObjectStorageContainer,
)
SECRET = "AKIAIOSFODNN7EXAMPLE"
STACK_ARN = "arn:aws:cloudformation:eu-central-1:123456789012:stack/app/1"
CLOUDFORMATION_CHECKS = (
Path(__file__).parents[3] / "prowler/providers/aws/services/cloudformation"
)
MODELS_WITH_SECRETS = [
App,
Branch,
Stage,
LaunchConfiguration,
Function,
BatchContainerProperties,
BatchContainerEnvVariable,
Stack,
LogGroup,
EnvironmentVariable,
Pipeline,
Instance,
TemplateData,
EcsContainerEnvVariable,
Environment,
Connection,
Job,
NotebookInstance,
Document,
StateMachine,
SnapshotResource,
VolumeResource,
ComputeInstance,
ObjectStorageContainer,
]
class Secret(BaseModel):
sensitive_fields: ClassVar[tuple[str, ...]] = ("value",)
name: str
value: str
class Holder(BaseModel):
sensitive_fields: ClassVar[tuple[str, ...]] = ("env", "script")
name: str
env: dict = {}
script: Optional[str] = None
secrets: list[Secret] = []
by_name: dict[str, Secret] = {}
@dataclass
class DataclassHolder:
sensitive_fields: ClassVar[tuple[str, ...]] = ("metadata",)
name: str
metadata: dict
def _check_metadata(check_id: str) -> str:
return (CLOUDFORMATION_CHECKS / check_id / f"{check_id}.metadata.json").read_text()
class TestRedactSensitiveFields:
def test_masks_values_and_keeps_dict_keys(self):
holder = Holder(name="app", env={"DB_PASSWORD": SECRET, "nested": {"k": 1}})
redacted = redact_sensitive_fields(holder, holder.dict())
assert redacted["name"] == "app"
assert redacted["env"] == {
"DB_PASSWORD": REDACTED_VALUE,
"nested": {"k": REDACTED_VALUE},
}
def test_keeps_empty_values(self):
holder = Holder(name="app")
redacted = redact_sensitive_fields(holder, holder.dict())
assert redacted["env"] == {}
assert redacted["script"] is None
def test_redacts_nested_models_in_lists_and_dicts(self):
holder = Holder(
name="app",
secrets=[Secret(name="TOKEN", value=SECRET)],
by_name={"TOKEN": Secret(name="TOKEN", value=SECRET)},
)
redacted = redact_sensitive_fields(holder, holder.dict())
assert redacted["secrets"] == [{"name": "TOKEN", "value": REDACTED_VALUE}]
assert redacted["by_name"]["TOKEN"] == {
"name": "TOKEN",
"value": REDACTED_VALUE,
}
def test_redacts_dataclasses(self):
holder = DataclassHolder(name="vm", metadata={"password": SECRET})
redacted = redact_sensitive_fields(holder, {"name": "vm", **holder.__dict__})
assert redacted["metadata"] == {"password": REDACTED_VALUE}
def test_does_not_mutate_the_resource(self):
holder = Holder(name="app", secrets=[Secret(name="TOKEN", value=SECRET)])
data = holder.dict()
redact_sensitive_fields(holder, data)
assert holder.secrets[0].value == SECRET
assert data["secrets"][0]["value"] == SECRET
@pytest.mark.parametrize("model", MODELS_WITH_SECRETS, ids=lambda m: m.__name__)
def test_declared_fields_exist_on_model(self, model):
names = (
{f.name for f in fields(model)}
if is_dataclass(model)
else set(model.__fields__)
)
assert model.sensitive_fields
assert set(model.sensitive_fields) <= names
assert "sensitive_fields" not in names
class TestCheckReportRedaction:
def test_unrelated_check_on_same_stack_does_not_expose_outputs(self):
stack = Stack(
arn=STACK_ARN,
name="app",
outputs=[f"AccessKey:{SECRET}"],
region="eu-central-1",
)
report = Check_Report_AWS(
metadata=_check_metadata(
"cloudformation_stacks_termination_protection_enabled"
),
resource=stack,
)
assert report.resource["outputs"] == [REDACTED_VALUE]
assert SECRET not in json.dumps(report.resource)
assert report.resource_arn == STACK_ARN
assert stack.outputs == [f"AccessKey:{SECRET}"]
def test_lambda_environment_is_redacted(self):
function = Function(
name="fn",
arn="arn:aws:lambda:eu-central-1:123456789012:function:fn",
region="eu-central-1",
environment={"API_TOKEN": SECRET},
security_groups=[],
)
report = Check_Report_AWS(
metadata=_check_metadata("cloudformation_stack_outputs_find_secrets"),
resource=function,
)
assert report.resource["environment"] == {"API_TOKEN": REDACTED_VALUE}
def test_dict_resource_without_model_is_left_untouched(self):
resource = {"name": "x", "outputs": ["plain"]}
report = CheckReportOpenStack(
metadata=_check_metadata("cloudformation_stack_outputs_find_secrets"),
resource=resource,
)
assert report.resource == resource
@@ -78,6 +78,8 @@ class Test_cloudformation_stack_outputs_find_secrets:
)
assert result[0].region == AWS_REGION
assert result[0].resource_tags == []
assert result[0].resource["outputs"] == ["REDACTED", "REDACTED"]
assert cloudformation_client.stacks[0].outputs[1] == "ENV:DEV"
def test_stack_secret_in_outputs_false_case(self):
cloudformation_client = mock.MagicMock
@@ -139,4 +139,7 @@ def test_stepfunctions_statemachine_encrypted_with_cmk(
assert result[0].resource_id == STATE_MACHINE_ID
assert result[0].resource_arn == STATE_MACHINE_ARN
assert result[0].region == AWS_REGION_EU_WEST_1
assert result[0].resource == state_machines[STATE_MACHINE_ARN]
assert result[0].resource == {
**state_machines[STATE_MACHINE_ARN].dict(),
"definition": "REDACTED",
}
@@ -131,4 +131,7 @@ def test_stepfunctions_statemachine_logging(state_machines, expected_status):
assert result[0].resource_id == STATE_MACHINE_ID
assert result[0].resource_arn == STATE_MACHINE_ARN
assert result[0].region == AWS_REGION_EU_WEST_1
assert result[0].resource == state_machines[STATE_MACHINE_ARN]
assert result[0].resource == {
**state_machines[STATE_MACHINE_ARN].dict(),
"definition": "REDACTED",
}