mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-08-21 13:20:57 +00:00
Merge branch 'master' into cross-provider-compliance-view-ui-side
This commit is contained in:
+24
-22
@@ -30,17 +30,18 @@ updates:
|
||||
# - "pip"
|
||||
# - "component/api"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github_actions"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# Dependabot version updates disabled - migrated to Renovate - 2026/07/02
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "npm"
|
||||
@@ -54,17 +55,18 @@ updates:
|
||||
# - "npm"
|
||||
# - "component/ui"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# Dependabot version updates disabled - migrated to Renovate - 2026/07/02
|
||||
# - package-ecosystem: "docker"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "docker"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# - package-ecosystem: "pre-commit"
|
||||
# directory: "/"
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 1 * *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Minors: 8th of every 3 months, Madrid overnight window (22:00-06:00)",
|
||||
@@ -48,7 +48,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 8 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Majors: 15th of every 3 months, Madrid overnight window",
|
||||
@@ -58,7 +58,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 15 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions - single grouped PR, no changelog, scope=ci",
|
||||
|
||||
@@ -215,7 +215,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17@sha256:2cd82735a36356842d5eb1ef80db3ae8f1154172f0f653db48fde079b2a0b7f7
|
||||
image: postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e
|
||||
env:
|
||||
POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
|
||||
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine3.19
|
||||
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
|
||||
env:
|
||||
VALKEY_HOST: ${{ env.VALKEY_HOST }}
|
||||
VALKEY_PORT: ${{ env.VALKEY_PORT }}
|
||||
|
||||
@@ -206,7 +206,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -37,8 +37,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
persist-credentials: false # No write token in the untrusted PR-head tree; public repo so base fetch/changed-files work unauthenticated
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
@@ -50,6 +49,8 @@ jobs:
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: '**'
|
||||
safe_output: false # Raw paths (list read via env var, injection-safe); default escaping backslash-quotes chars like () and breaks the -f test
|
||||
separator: "\n" # Newline-delimited so the reader tolerates spaces and glob chars in paths
|
||||
|
||||
- name: Check for conflict markers
|
||||
id: conflict-check
|
||||
@@ -59,19 +60,18 @@ jobs:
|
||||
CONFLICT_FILES=""
|
||||
HAS_CONFLICTS=false
|
||||
|
||||
# Check each changed file for conflict markers
|
||||
for file in ${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}; do
|
||||
if [ -f "$file" ]; then
|
||||
echo "Checking file: $file"
|
||||
# Read newline-delimited paths so spaces/globs neither word-split nor glob-expand
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] || continue
|
||||
[ -f "$file" ] || continue
|
||||
echo "Checking file: $file"
|
||||
|
||||
# Look for conflict markers (more precise regex)
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
done
|
||||
done <<< "$STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES"
|
||||
|
||||
if [ "$HAS_CONFLICTS" = true ]; then
|
||||
echo "has_conflicts=true" >> $GITHUB_OUTPUT
|
||||
@@ -88,18 +88,49 @@ jobs:
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
|
||||
- name: Check base-branch mergeability
|
||||
id: merge-check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
MERGEABLE=null
|
||||
|
||||
# GitHub computes mergeability async, so .mergeable is null until ready; poll until resolved
|
||||
for attempt in 1 2 3 4 5; do
|
||||
MERGEABLE=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.mergeable')
|
||||
if [ "$MERGEABLE" != "null" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Attempt ${attempt}: mergeability not computed yet, retrying..."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# Keep 'unknown' distinct from 'clean' so we never assert a clean merge we could not confirm
|
||||
case "$MERGEABLE" in
|
||||
false) STATUS=conflict; echo "PR branch cannot be merged cleanly into its base branch" ;;
|
||||
true) STATUS=clean; echo "PR branch merges cleanly into its base branch" ;;
|
||||
*) STATUS=unknown; echo "::warning::Mergeability did not resolve after retries; leaving it undetermined" ;;
|
||||
esac
|
||||
|
||||
echo "merge_status=${STATUS}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Manage conflict label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
LABEL_NAME="has-conflicts"
|
||||
|
||||
# Add or remove label based on conflict status
|
||||
if [ "$HAS_CONFLICTS" = "true" ]; then
|
||||
if [ "$HAS_CONFLICTS" = "true" ] || [ "$MERGE_STATUS" = "conflict" ]; then
|
||||
echo "Adding conflict label to PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
elif [ "$MERGE_STATUS" = "unknown" ]; then
|
||||
# Don't drop the label on an undetermined merge state; a later run will settle it
|
||||
echo "Mergeability undetermined; leaving label unchanged"
|
||||
else
|
||||
echo "Removing conflict label from PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
@@ -121,20 +152,25 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- conflict-checker-comment -->
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠️ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }}
|
||||
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files:
|
||||
${{ (steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict') && '⚠️ **Conflicts Detected**' || (steps.merge-check.outputs.merge_status == 'unknown' && 'ℹ️ **Conflict Check Incomplete**' || '✅ **No Conflicts**') }}
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('
|
||||
**Conflict markers** are present in the following files:
|
||||
|
||||
{0}
|
||||
|
||||
Please resolve these conflicts by:
|
||||
1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>`
|
||||
2. Manually editing the files to resolve the conflicts
|
||||
3. Removing all conflict markers
|
||||
4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }}
|
||||
Resolve them by removing every `<<<<<<<`, `=======`, and `>>>>>>>` marker, then commit and push.', steps.conflict-check.outputs.conflict_files) || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'conflict' && '
|
||||
**Merge conflict with the base branch.** This PR cannot be merged cleanly. Update your branch with the latest base (rebase or merge) and resolve the conflicts.' || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'unknown' && '
|
||||
GitHub had not finished computing mergeability, so base-branch conflict status could not be verified on this run.' || '' }}
|
||||
${{ (steps.conflict-check.outputs.has_conflicts != 'true' && steps.merge-check.outputs.merge_status == 'clean') && '
|
||||
No conflict markers, and the branch merges cleanly into its base.' || '' }}
|
||||
|
||||
- name: Fail workflow if conflicts detected
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict'
|
||||
env:
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
echo "::error::Workflow failed due to conflict markers detected in the PR"
|
||||
[ "$HAS_CONFLICTS" = "true" ] && echo "::error::Conflict markers detected in changed files"
|
||||
[ "$MERGE_STATUS" = "conflict" ] && echo "::error::PR branch has merge conflicts with the base branch"
|
||||
exit 1
|
||||
|
||||
@@ -56,6 +56,6 @@ jobs:
|
||||
"PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }},
|
||||
"PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }},
|
||||
"PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}",
|
||||
"PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}",
|
||||
"PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}"
|
||||
"PROWLER_PR_BASE_BRANCH": ${{ toJson(github.event.pull_request.base.ref) }},
|
||||
"PROWLER_PR_HEAD_BRANCH": ${{ toJson(github.event.pull_request.head.ref) }}
|
||||
}
|
||||
|
||||
@@ -138,6 +138,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -147,6 +148,8 @@ jobs:
|
||||
allowed-endpoints: >
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
public.ecr.aws:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
@@ -173,14 +176,16 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
@@ -206,6 +211,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -221,6 +227,8 @@ jobs:
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
|
||||
|
||||
- name: Login to DockerHub
|
||||
@@ -229,14 +237,16 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -299,7 +309,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install PyYAML
|
||||
run: pip install pyyaml
|
||||
|
||||
@@ -201,7 +201,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -37,8 +37,12 @@ jobs:
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
fonts.googleapis.com:443
|
||||
fonts.gstatic.com:443
|
||||
api.iconify.design:443
|
||||
api.simplesvg.com:443
|
||||
api.unisvg.com:443
|
||||
api.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
cdn.playwright.dev:443
|
||||
|
||||
@@ -114,6 +114,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing Prowler UI components | `prowler-ui` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/prowler"
|
||||
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
|
||||
@@ -4,9 +4,19 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.33.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Added timestamp precision support for `/api/v1/findings` `inserted_at` and `updated_at` filters [(#11754)](https://github.com/prowler-cloud/prowler/pull/11754)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Attack Paths: AWS Neptune is now supported as a persistent sink database, selectable via `ATTACK_PATHS_SINK_DATABASE=neptune` (default `neo4j`), Cartography's (bumped to 0.138.1) per-scan ingest database stays on Neo4j [(#11524)](https://github.com/prowler-cloud/prowler/pull/11524)
|
||||
- Attack Paths: Scan task now checks the ingest Neo4j database and configured graph sink before starting graph ingestion [(#11743)](https://github.com/prowler-cloud/prowler/pull/11743)
|
||||
- Disable PowerShell telemetry in the API container image [(#11746)](https://github.com/prowler-cloud/prowler/pull/11746)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Provider graph cleanup now deletes Neo4j and Neptune relationships in directed batches before deleting nodes [(#11755)](https://github.com/prowler-cloud/prowler/pull/11755)
|
||||
|
||||
---
|
||||
|
||||
|
||||
+3
-1
@@ -1,9 +1,11 @@
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
|
||||
ENV POWERSHELL_TELEMETRY_OPTOUT=1
|
||||
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
+1
-1
@@ -218,7 +218,6 @@ constraint-dependencies = [
|
||||
"debugpy==1.8.20",
|
||||
"decorator==5.2.1",
|
||||
"defusedxml==0.7.1",
|
||||
"detect-secrets==1.5.0",
|
||||
"dill==0.4.1",
|
||||
"distro==1.9.0",
|
||||
"dj-rest-auth==7.0.1",
|
||||
@@ -301,6 +300,7 @@ constraint-dependencies = [
|
||||
"jsonschema==4.23.0",
|
||||
"jsonschema-specifications==2025.9.1",
|
||||
"keystoneauth1==5.13.0",
|
||||
"kingfisher-bin==1.104.0",
|
||||
"kiwisolver==1.4.9",
|
||||
"knack==0.11.0",
|
||||
"kombu==5.6.2",
|
||||
|
||||
@@ -96,6 +96,11 @@ def inject_provider_label(cypher: str, provider_id: str) -> str:
|
||||
node pattern.
|
||||
"""
|
||||
label = get_provider_label(provider_id)
|
||||
return inject_label(cypher, label)
|
||||
|
||||
|
||||
def inject_label(cypher: str, label: str) -> str:
|
||||
"""Rewrite a Cypher query to append a label to every node pattern."""
|
||||
|
||||
# Step 1: Protect strings and comments (single pass, leftmost-first)
|
||||
protected: list[str] = []
|
||||
|
||||
@@ -106,6 +106,31 @@ def verify_connectivity() -> None:
|
||||
sink_module.get_backend().verify_connectivity()
|
||||
|
||||
|
||||
def verify_scan_databases_available() -> None:
|
||||
"""Raise if either graph database needed by an Attack Paths scan is unavailable."""
|
||||
errors: list[str] = []
|
||||
first_error: Exception | None = None
|
||||
|
||||
try:
|
||||
ingest.get_driver().verify_connectivity()
|
||||
except Exception as exc:
|
||||
errors.append(f"ingest Neo4j: {exc}")
|
||||
first_error = exc
|
||||
|
||||
try:
|
||||
get_driver().verify_connectivity()
|
||||
except Exception as exc:
|
||||
errors.append(f"sink {settings.ATTACK_PATHS_SINK_DATABASE}: {exc}")
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"Attack Paths graph database unavailable before scan start: "
|
||||
+ "; ".join(errors)
|
||||
) from first_error
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
"""Return the sink URI. Retained for backwards compatibility."""
|
||||
if settings.ATTACK_PATHS_SINK_DATABASE == "neptune":
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Shared batched deletion helpers for sink backends."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES = {
|
||||
"outgoing relationship": """
|
||||
MATCH (n:`{provider_label}`)-[r]->()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
"incoming relationship": """
|
||||
MATCH (n:`{provider_label}`)<-[r]-()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
}
|
||||
|
||||
NODE_DELETE_QUERY_TEMPLATE = """
|
||||
MATCH (n:{provider_resource_label}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
"""
|
||||
|
||||
|
||||
def delete_batches(
|
||||
*,
|
||||
session: Any,
|
||||
logger: logging.Logger,
|
||||
log_target: str,
|
||||
provider_id: str,
|
||||
query: str,
|
||||
phase: str,
|
||||
count_key: str,
|
||||
total_key: str,
|
||||
deleted_key: str,
|
||||
initial_total: int,
|
||||
batch_size: int,
|
||||
drop_t0: float,
|
||||
) -> tuple[int, int]:
|
||||
deleted_total = initial_total
|
||||
batches = 0
|
||||
while True:
|
||||
logger.info(
|
||||
"Deleting %s batch from %s "
|
||||
"(provider=%s, batch=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches + 1,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
record = session.run(query, {"batch_size": batch_size}).single()
|
||||
deleted = (record[count_key] if record else 0) or 0
|
||||
if deleted == 0:
|
||||
return deleted_total, batches
|
||||
|
||||
batches += 1
|
||||
deleted_total += deleted
|
||||
logger.info(
|
||||
"Deleted %s batch from %s "
|
||||
"(provider=%s, batch=%s, %s=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches,
|
||||
deleted_key,
|
||||
deleted,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
@@ -17,6 +17,11 @@ import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
@@ -204,10 +209,8 @@ class Neo4jSink(SinkDatabase):
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_nodes = 0
|
||||
deleted_relationships = 0
|
||||
relationship_batches = 0
|
||||
node_batches = 0
|
||||
deleted_nodes = deleted_relationships = 0
|
||||
relationship_batches = node_batches = 0
|
||||
drop_t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
@@ -232,84 +235,44 @@ class Neo4jSink(SinkDatabase):
|
||||
database,
|
||||
provider_id,
|
||||
)
|
||||
# Phase 1: delete relationships incident to provider nodes in
|
||||
# batches. The undirected pattern matches an edge between two
|
||||
# provider nodes from both ends, so `DISTINCT r` dedupes it to
|
||||
# delete a full batch of unique relationships each round.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
next_batch = relationship_batches + 1
|
||||
logger.info(
|
||||
"Deleting relationship batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, total_rels=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
log_target = f"Neo4j sink database {database}"
|
||||
for (
|
||||
phase,
|
||||
query_template,
|
||||
) in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (:`{provider_label}`)-[r]-()
|
||||
WITH DISTINCT r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_rels_count", 0)
|
||||
if deleted_count > 0:
|
||||
relationship_batches += 1
|
||||
deleted_relationships += deleted_count
|
||||
logger.info(
|
||||
"Deleted relationship batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, deleted_rels=%s, "
|
||||
"total_rels=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_count,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
# Phase 2: delete the now relationship-free nodes in batches.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
next_batch = node_batches + 1
|
||||
logger.info(
|
||||
"Deleting node batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, total_nodes=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_nodes_count", 0)
|
||||
if deleted_count > 0:
|
||||
node_batches += 1
|
||||
deleted_nodes += deleted_count
|
||||
logger.info(
|
||||
"Deleted node batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, deleted_nodes=%s, "
|
||||
"total_nodes=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
node_batches,
|
||||
deleted_count,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == DATABASE_NOT_FOUND_CODE:
|
||||
|
||||
@@ -27,6 +27,11 @@ import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.session import Session as BotoSession
|
||||
@@ -296,78 +301,40 @@ class NeptuneSink(SinkDatabase):
|
||||
"Opened Neptune writer session for provider graph drop (provider=%s)",
|
||||
provider_id,
|
||||
)
|
||||
while True:
|
||||
next_batch = relationship_batches + 1
|
||||
logger.info(
|
||||
"Deleting relationship batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, total_rels=%s, elapsed=%.3fs)",
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (:`{provider_label}`)-[r]-()
|
||||
WITH DISTINCT r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
record = result.single()
|
||||
deleted_rels = (record["deleted_rels_count"] if record else 0) or 0
|
||||
if deleted_rels == 0:
|
||||
break
|
||||
relationship_batches += 1
|
||||
deleted_relationships += deleted_rels
|
||||
logger.info(
|
||||
"Deleted relationship batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, deleted_rels=%s, total_rels=%s, "
|
||||
"elapsed=%.3fs)",
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_rels,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
for phase, query_template in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
deleted_nodes = 0
|
||||
while True:
|
||||
next_batch = node_batches + 1
|
||||
logger.info(
|
||||
"Deleting node batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, total_nodes=%s, elapsed=%.3fs)",
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
record = result.single()
|
||||
deleted = (record["deleted_nodes_count"] if record else 0) or 0
|
||||
if deleted == 0:
|
||||
break
|
||||
node_batches += 1
|
||||
deleted_nodes += deleted
|
||||
logger.info(
|
||||
"Deleted node batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, deleted_nodes=%s, total_nodes=%s, "
|
||||
"elapsed=%.3fs)",
|
||||
provider_id,
|
||||
node_batches,
|
||||
deleted,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Finished dropping provider graph from Neptune sink "
|
||||
|
||||
+206
-59
@@ -67,6 +67,7 @@ from django_filters.rest_framework import (
|
||||
)
|
||||
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
from uuid6 import UUID
|
||||
|
||||
|
||||
class CustomDjangoFilterBackend(DjangoFilterBackend):
|
||||
@@ -672,35 +673,32 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
return queryset.filter(tags__text_search=value)
|
||||
|
||||
|
||||
class FindingFilter(CommonFindingFilters):
|
||||
FINDING_BASE_FILTER_FIELDS = {
|
||||
"id": ["exact", "in"],
|
||||
"uid": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
"delta": ["exact", "in"],
|
||||
"status": ["exact", "in"],
|
||||
"severity": ["exact", "in"],
|
||||
"impact": ["exact", "in"],
|
||||
"check_id": ["exact", "in", "icontains"],
|
||||
}
|
||||
|
||||
|
||||
class BaseFindingFilter(CommonFindingFilters):
|
||||
DATE_FILTER_FIELDS = ()
|
||||
DATE_FILTER_NAMES = ()
|
||||
DATE_RANGE_HELP_TEXT = (
|
||||
f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days."
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = "At least one date filter is required."
|
||||
|
||||
scan = UUIDFilter(method="filter_scan_id")
|
||||
scan__in = UUIDInFilter(method="filter_scan_id_in")
|
||||
|
||||
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(
|
||||
method="filter_inserted_at_gte",
|
||||
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
)
|
||||
inserted_at__lte = DateFilter(
|
||||
method="filter_inserted_at_lte",
|
||||
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Finding
|
||||
fields = {
|
||||
"id": ["exact", "in"],
|
||||
"uid": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
"delta": ["exact", "in"],
|
||||
"status": ["exact", "in"],
|
||||
"severity": ["exact", "in"],
|
||||
"impact": ["exact", "in"],
|
||||
"check_id": ["exact", "in", "icontains"],
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"updated_at": ["gte", "lte"],
|
||||
}
|
||||
fields = FINDING_BASE_FILTER_FIELDS
|
||||
filter_overrides = {
|
||||
FindingDeltaEnumField: {
|
||||
"filter_class": CharFilter,
|
||||
@@ -723,17 +721,13 @@ class FindingFilter(CommonFindingFilters):
|
||||
return queryset.filter(resource_services__contains=[value])
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
|
||||
self.data.get("inserted_at")
|
||||
or self.data.get("inserted_at__date")
|
||||
or self.data.get("inserted_at__gte")
|
||||
or self.data.get("inserted_at__lte")
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not any(
|
||||
self.data.get(filter_name) for filter_name in self.DATE_FILTER_NAMES
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
|
||||
"or filter[inserted_at.lte].",
|
||||
"detail": self.DATE_FILTER_REQUIRED_DETAIL,
|
||||
"status": 400,
|
||||
"source": {"pointer": "/data/attributes/inserted_at"},
|
||||
"code": "required",
|
||||
@@ -742,31 +736,42 @@ class FindingFilter(CommonFindingFilters):
|
||||
)
|
||||
|
||||
cleaned = self.form.cleaned_data
|
||||
exact_date = cleaned.get("inserted_at") or cleaned.get("inserted_at__date")
|
||||
gte_date = cleaned.get("inserted_at__gte") or exact_date
|
||||
lte_date = cleaned.get("inserted_at__lte") or exact_date
|
||||
|
||||
if gte_date is None:
|
||||
gte_date = datetime.now(UTC).date()
|
||||
if lte_date is None:
|
||||
lte_date = datetime.now(UTC).date()
|
||||
|
||||
if abs(lte_date - gte_date) > timedelta(
|
||||
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
"status": 400,
|
||||
"source": {"pointer": "/data/attributes/inserted_at"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
for field_name in self.DATE_FILTER_FIELDS:
|
||||
self.validate_datetime_filter_range(cleaned, field_name)
|
||||
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
def validate_datetime_filter_range(self, cleaned, field_name):
|
||||
exact_value = cleaned.get(field_name) or cleaned.get(f"{field_name}__date")
|
||||
gte_value = cleaned.get(f"{field_name}__gte") or exact_value
|
||||
lte_value = cleaned.get(f"{field_name}__lte") or exact_value
|
||||
|
||||
if not (exact_value or gte_value or lte_value):
|
||||
return
|
||||
|
||||
default_value = datetime.now(UTC).date()
|
||||
gte_value = gte_value or default_value
|
||||
lte_value = lte_value or default_value
|
||||
|
||||
gte_datetime = self.filter_value_to_datetime(gte_value, field_name)
|
||||
lte_datetime = self.filter_value_to_datetime(lte_value, field_name)
|
||||
|
||||
if abs(lte_datetime - gte_datetime) <= timedelta(
|
||||
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
|
||||
):
|
||||
return
|
||||
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
"status": 400,
|
||||
"source": {"pointer": f"/data/attributes/{field_name}"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Convert filter values to UUIDv7 values for use with partitioning
|
||||
def filter_scan_id(self, queryset, name, value):
|
||||
try:
|
||||
@@ -824,27 +829,169 @@ class FindingFilter(CommonFindingFilters):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
start = uuid7_start(datetime_to_uuid7(datetime_value))
|
||||
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
|
||||
|
||||
return queryset.filter(id__gte=start, id__lt=end)
|
||||
|
||||
def filter_inserted_at_gte(self, queryset, name, value):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
start = uuid7_start(datetime_to_uuid7(datetime_value))
|
||||
|
||||
return queryset.filter(id__gte=start)
|
||||
|
||||
def filter_inserted_at_lte(self, queryset, name, value):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
|
||||
|
||||
return queryset.filter(id__lt=end)
|
||||
|
||||
@staticmethod
|
||||
def maybe_date_to_datetime(value):
|
||||
dt = value
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, date):
|
||||
dt = datetime.combine(value, datetime.min.time(), tzinfo=UTC)
|
||||
return dt
|
||||
return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
|
||||
if isinstance(value, str):
|
||||
return parse(value)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def filter_value_to_datetime(cls, value, field_name):
|
||||
try:
|
||||
datetime_value = cls.maybe_date_to_datetime(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "Enter a valid date or datetime.",
|
||||
"status": 400,
|
||||
"source": {"pointer": f"/data/attributes/{field_name}"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
if datetime_value.tzinfo is None:
|
||||
return datetime_value.replace(tzinfo=UTC)
|
||||
return datetime_value.astimezone(UTC)
|
||||
|
||||
|
||||
class FindingFilter(BaseFindingFilter):
|
||||
DATE_FILTER_FIELDS = ("inserted_at", "updated_at")
|
||||
DATE_FILTER_NAMES = (
|
||||
"inserted_at",
|
||||
"inserted_at__date",
|
||||
"inserted_at__gte",
|
||||
"inserted_at__lte",
|
||||
"updated_at",
|
||||
"updated_at__date",
|
||||
"updated_at__gte",
|
||||
"updated_at__lte",
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = (
|
||||
"At least one date filter is required: filter[inserted_at], filter[updated_at], "
|
||||
"filter[inserted_at.gte], filter[updated_at.gte], filter[inserted_at.lte], "
|
||||
"or filter[updated_at.lte]."
|
||||
)
|
||||
|
||||
inserted_at = CharFilter(method="filter_inserted_at")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = CharFilter(
|
||||
method="filter_inserted_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
inserted_at__lte = CharFilter(
|
||||
method="filter_inserted_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
updated_at = CharFilter(method="filter_updated_at")
|
||||
updated_at__date = DateFilter(method="filter_updated_at", lookup_expr="date")
|
||||
updated_at__gte = CharFilter(
|
||||
method="filter_updated_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
updated_at__lte = CharFilter(
|
||||
method="filter_updated_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
|
||||
class Meta(BaseFindingFilter.Meta):
|
||||
fields = FINDING_BASE_FILTER_FIELDS | {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"updated_at": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
def filter_inserted_at(self, queryset, name, value):
|
||||
start, end = self.filter_value_to_datetime_bounds(value, "inserted_at")
|
||||
|
||||
if name.endswith("__gte"):
|
||||
return queryset.filter(id__gte=self.datetime_to_uuid7_boundary(start))
|
||||
if name.endswith("__lte"):
|
||||
return queryset.filter(id__lt=self.datetime_to_uuid7_boundary(end))
|
||||
|
||||
return queryset.filter(
|
||||
id__gte=self.datetime_to_uuid7_boundary(start),
|
||||
id__lt=self.datetime_to_uuid7_boundary(end),
|
||||
)
|
||||
|
||||
def filter_updated_at(self, queryset, name, value):
|
||||
start, end = self.filter_value_to_datetime_bounds(value, "updated_at")
|
||||
|
||||
if name.endswith("__gte"):
|
||||
return queryset.filter(updated_at__gte=start)
|
||||
if name.endswith("__lte"):
|
||||
return queryset.filter(updated_at__lt=end)
|
||||
|
||||
return queryset.filter(updated_at__gte=start, updated_at__lt=end)
|
||||
|
||||
@classmethod
|
||||
def filter_value_to_datetime_bounds(cls, value, field_name):
|
||||
start = cls.filter_value_to_datetime(value, field_name)
|
||||
if cls.is_date_filter_value(value):
|
||||
return start, start + timedelta(days=1)
|
||||
return start, start + timedelta(milliseconds=1)
|
||||
|
||||
@staticmethod
|
||||
def datetime_to_uuid7_boundary(datetime_value):
|
||||
timestamp_ms = int(datetime_value.timestamp() * 1000) & 0xFFFFFFFFFFFF
|
||||
uuid_int = timestamp_ms << 80
|
||||
uuid_int |= 0x7 << 76
|
||||
uuid_int |= 0x2 << 62
|
||||
return UUID(int=uuid_int)
|
||||
|
||||
@staticmethod
|
||||
def is_date_filter_value(value):
|
||||
if isinstance(value, datetime):
|
||||
return False
|
||||
if isinstance(value, date):
|
||||
return True
|
||||
return isinstance(value, str) and len(value.strip()) == 10
|
||||
|
||||
|
||||
class FindingMetadataFilter(BaseFindingFilter):
|
||||
DATE_FILTER_FIELDS = ("inserted_at",)
|
||||
DATE_FILTER_NAMES = (
|
||||
"inserted_at",
|
||||
"inserted_at__date",
|
||||
"inserted_at__gte",
|
||||
"inserted_at__lte",
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = (
|
||||
"At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
|
||||
"or filter[inserted_at.lte]."
|
||||
)
|
||||
|
||||
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(
|
||||
method="filter_inserted_at_gte",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
inserted_at__lte = DateFilter(
|
||||
method="filter_inserted_at_lte",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
|
||||
class Meta(BaseFindingFilter.Meta):
|
||||
fields = FINDING_BASE_FILTER_FIELDS | {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
|
||||
class LatestFindingFilter(CommonFindingFilters):
|
||||
|
||||
+1601
-313
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,7 @@ hierarchy; sink-internal behavior is exercised in `test_sink.py`.
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import api.attack_paths.database as db_module
|
||||
import pytest
|
||||
|
||||
|
||||
class TestDatabaseNameHelper:
|
||||
@@ -72,6 +73,71 @@ class TestExecuteReadQueryRoutes:
|
||||
)
|
||||
|
||||
|
||||
class TestScanDatabaseAvailability:
|
||||
def test_verify_scan_databases_available_checks_ingest_and_sink(self):
|
||||
with (
|
||||
patch("api.attack_paths.database.ingest") as mock_ingest,
|
||||
patch("api.attack_paths.database.get_driver") as mock_get_driver,
|
||||
):
|
||||
db_module.verify_scan_databases_available()
|
||||
|
||||
mock_ingest.get_driver.return_value.verify_connectivity.assert_called_once_with()
|
||||
mock_get_driver.return_value.verify_connectivity.assert_called_once_with()
|
||||
|
||||
def test_verify_scan_databases_available_raises_when_ingest_is_down(self):
|
||||
with (
|
||||
patch("api.attack_paths.database.ingest") as mock_ingest,
|
||||
patch("api.attack_paths.database.get_driver"),
|
||||
):
|
||||
mock_ingest.get_driver.return_value.verify_connectivity.side_effect = (
|
||||
RuntimeError("ingest down")
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
db_module.verify_scan_databases_available()
|
||||
|
||||
assert "Attack Paths graph database unavailable before scan start" in str(
|
||||
exc.value
|
||||
)
|
||||
assert "ingest Neo4j: ingest down" in str(exc.value)
|
||||
|
||||
def test_verify_scan_databases_available_raises_when_sink_is_down(self, settings):
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
|
||||
with (
|
||||
patch("api.attack_paths.database.ingest"),
|
||||
patch("api.attack_paths.database.get_driver") as mock_get_driver,
|
||||
):
|
||||
mock_get_driver.return_value.verify_connectivity.side_effect = RuntimeError(
|
||||
"writer down"
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
db_module.verify_scan_databases_available()
|
||||
|
||||
assert "sink neptune: writer down" in str(exc.value)
|
||||
|
||||
def test_verify_scan_databases_available_reports_both_failures(self, settings):
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
|
||||
with (
|
||||
patch("api.attack_paths.database.ingest") as mock_ingest,
|
||||
patch("api.attack_paths.database.get_driver") as mock_get_driver,
|
||||
):
|
||||
mock_ingest.get_driver.return_value.verify_connectivity.side_effect = (
|
||||
RuntimeError("ingest down")
|
||||
)
|
||||
mock_get_driver.return_value.verify_connectivity.side_effect = RuntimeError(
|
||||
"sink down"
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
db_module.verify_scan_databases_available()
|
||||
|
||||
assert "ingest Neo4j: ingest down" in str(exc.value)
|
||||
assert "sink neo4j: sink down" in str(exc.value)
|
||||
|
||||
|
||||
class TestSinkOperationsDelegation:
|
||||
def test_has_provider_data_delegates_to_sink(self, sink_backend_stub):
|
||||
sink_backend_stub.has_provider_data.return_value = True
|
||||
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.cypher_sanitizer import (
|
||||
inject_label,
|
||||
inject_provider_label,
|
||||
validate_custom_query,
|
||||
)
|
||||
@@ -21,6 +22,13 @@ def _inject(cypher: str) -> str:
|
||||
return inject_provider_label(cypher, PROVIDER_ID)
|
||||
|
||||
|
||||
def test_generic_inject_label_reuses_provider_injection_pipeline():
|
||||
result = inject_label("MATCH (n:AWSRole)--(m) RETURN n, m", "_Tenant_test")
|
||||
|
||||
assert "(n:AWSRole:_Tenant_test)" in result
|
||||
assert "(m:_Tenant_test)" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pass A - Labeled node patterns (all clauses)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -186,6 +186,25 @@ def _session_ctx(session: MagicMock) -> MagicMock:
|
||||
return ctx
|
||||
|
||||
|
||||
def _count_result(key: str, count: int) -> MagicMock:
|
||||
return MagicMock(single=MagicMock(return_value={key: count}))
|
||||
|
||||
|
||||
def _directed_drop_results(
|
||||
outgoing_rels: int,
|
||||
incoming_rels: int,
|
||||
nodes: int,
|
||||
) -> list[MagicMock]:
|
||||
return [
|
||||
_count_result("deleted_rels_count", outgoing_rels),
|
||||
_count_result("deleted_rels_count", 0),
|
||||
_count_result("deleted_rels_count", incoming_rels),
|
||||
_count_result("deleted_rels_count", 0),
|
||||
_count_result("deleted_nodes_count", nodes),
|
||||
_count_result("deleted_nodes_count", 0),
|
||||
]
|
||||
|
||||
|
||||
class TestNeo4jSinkSyncWrites:
|
||||
def test_ensure_sync_indexes_runs_create_index_idempotent(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
@@ -310,65 +329,48 @@ class TestNeptuneSinkSyncWrites:
|
||||
|
||||
|
||||
class TestNeptuneSinkDropSubgraph:
|
||||
def test_drop_subgraph_deletes_rels_before_nodes_in_bounded_batches(self):
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
|
||||
rel_record_first = MagicMock()
|
||||
rel_record_first.__getitem__ = lambda _self, key: 50
|
||||
rel_record_drain = MagicMock()
|
||||
rel_record_drain.__getitem__ = lambda _self, key: 0
|
||||
node_record_first = MagicMock()
|
||||
node_record_first.__getitem__ = lambda _self, key: 10
|
||||
node_record_drain = MagicMock()
|
||||
node_record_drain.__getitem__ = lambda _self, key: 0
|
||||
|
||||
run_results = [
|
||||
MagicMock(single=MagicMock(return_value=rel_record_first)),
|
||||
MagicMock(single=MagicMock(return_value=rel_record_drain)),
|
||||
MagicMock(single=MagicMock(return_value=node_record_first)),
|
||||
MagicMock(single=MagicMock(return_value=node_record_drain)),
|
||||
]
|
||||
session.run.side_effect = run_results
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
deleted = sink.drop_subgraph("ignored", "provider-1")
|
||||
|
||||
assert deleted == 10
|
||||
first_query = session.run.call_args_list[0].args[0]
|
||||
assert "DELETE r" in first_query
|
||||
assert "DETACH DELETE" not in first_query
|
||||
# DISTINCT avoids double-counting relationships matched from both ends.
|
||||
assert "DISTINCT r" in first_query
|
||||
third_query = session.run.call_args_list[2].args[0]
|
||||
assert "DELETE n" in third_query
|
||||
assert session.run.call_count == 6
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
|
||||
assert ")-[r]->()" in queries[0]
|
||||
assert ")<-[r]-()" in queries[2]
|
||||
assert "DELETE n" in queries[4]
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
assert all("DISTINCT r" not in query for query in queries)
|
||||
|
||||
first_node = next(i for i, q in enumerate(queries) if "DELETE n" in q)
|
||||
last_rel = max(i for i, q in enumerate(queries) if "DELETE r" in q)
|
||||
assert last_rel < first_node
|
||||
|
||||
|
||||
class TestNeo4jSinkDropSubgraph:
|
||||
"""Neo4j drop deletes relationships then nodes in batches (no ``DETACH DELETE``)."""
|
||||
|
||||
def test_drop_subgraph_deletes_rels_before_nodes_in_bounded_batches(self):
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
|
||||
rel_first = MagicMock()
|
||||
rel_first.get = lambda key, default=0: 50
|
||||
rel_drain = MagicMock()
|
||||
rel_drain.get = lambda key, default=0: 0
|
||||
node_first = MagicMock()
|
||||
node_first.get = lambda key, default=0: 10
|
||||
node_drain = MagicMock()
|
||||
node_drain.get = lambda key, default=0: 0
|
||||
session.run.side_effect = [
|
||||
MagicMock(single=MagicMock(return_value=rel_first)),
|
||||
MagicMock(single=MagicMock(return_value=rel_drain)),
|
||||
MagicMock(single=MagicMock(return_value=node_first)),
|
||||
MagicMock(single=MagicMock(return_value=node_drain)),
|
||||
]
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
|
||||
provider_id = "00000000-0000-0000-0000-000000000abc"
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -376,19 +378,20 @@ class TestNeo4jSinkDropSubgraph:
|
||||
|
||||
# Only phase-2 node counts contribute to the return value.
|
||||
assert deleted == 10
|
||||
assert session.run.call_count == 4
|
||||
assert session.run.call_count == 6
|
||||
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
# Regression guard: the memory blow-up was caused by DETACH DELETE.
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
assert all("DISTINCT r" not in query for query in queries)
|
||||
|
||||
first_query = queries[0]
|
||||
assert "DELETE r" in first_query
|
||||
# DISTINCT avoids double-counting relationships matched from both ends.
|
||||
assert "DISTINCT r" in first_query
|
||||
assert ")-[r]->()" in first_query
|
||||
assert ":`_Provider_00000000000000000000000000000abc`" in first_query
|
||||
|
||||
assert "DELETE n" in queries[2]
|
||||
assert ")<-[r]-()" in queries[2]
|
||||
assert "DELETE n" in queries[4]
|
||||
|
||||
# Relationships must be fully drained before nodes are deleted.
|
||||
first_node = next(i for i, q in enumerate(queries) if "DELETE n" in q)
|
||||
|
||||
@@ -57,6 +57,7 @@ from api.models import (
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from api.v1.serializers import TokenSerializer
|
||||
from api.v1.views import ComplianceOverviewViewSet, TenantFinishACSView
|
||||
from botocore.exceptions import ClientError, NoCredentialsError
|
||||
@@ -7218,6 +7219,26 @@ class TestFindingViewSet:
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert response.json()["errors"][0]["code"] == "invalid"
|
||||
|
||||
def test_findings_updated_at_range_too_large_with_inserted_at_filter(
|
||||
self, authenticated_client
|
||||
):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[inserted_at]": TODAY,
|
||||
"filter[updated_at.gte]": today_after_n_days(
|
||||
-(settings.FINDINGS_MAX_DAYS_IN_RANGE + 1)
|
||||
),
|
||||
"filter[updated_at.lte]": TODAY,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert response.json()["errors"][0]["code"] == "invalid"
|
||||
assert response.json()["errors"][0]["source"]["pointer"] == (
|
||||
"/data/attributes/updated_at"
|
||||
)
|
||||
|
||||
def test_findings_list(self, authenticated_client, findings_fixture):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"), {"filter[inserted_at]": TODAY}
|
||||
@@ -7229,6 +7250,170 @@ class TestFindingViewSet:
|
||||
== findings_fixture[0].status
|
||||
)
|
||||
|
||||
def test_findings_list_inserted_at_accepts_timestamp_precision_filters(
|
||||
self, authenticated_client, scans_fixture
|
||||
):
|
||||
scan, *_ = scans_fixture
|
||||
|
||||
def create_finding(uid, inserted_at):
|
||||
finding = Finding.objects.create(
|
||||
id=datetime_to_uuid7(inserted_at),
|
||||
tenant_id=scan.tenant_id,
|
||||
uid=uid,
|
||||
scan=scan,
|
||||
status=Status.FAIL,
|
||||
status_extended="timestamp precision status",
|
||||
impact=Severity.medium,
|
||||
severity=Severity.medium,
|
||||
check_id="timestamp_precision_check",
|
||||
check_metadata={
|
||||
"CheckId": "timestamp_precision_check",
|
||||
"Description": "timestamp precision check",
|
||||
"servicename": "ec2",
|
||||
},
|
||||
first_seen_at=inserted_at,
|
||||
)
|
||||
Finding.all_objects.filter(pk=finding.pk).update(
|
||||
inserted_at=inserted_at,
|
||||
updated_at=inserted_at,
|
||||
)
|
||||
finding.refresh_from_db()
|
||||
return finding
|
||||
|
||||
create_finding(
|
||||
"timestamp_precision_early",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 100000, tzinfo=UTC),
|
||||
)
|
||||
late_finding = create_finding(
|
||||
"timestamp_precision_late",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[inserted_at.gte]": "2026-01-15T10:30:00.150Z",
|
||||
"filter[inserted_at.lte]": "2026-01-15T10:30:00.250Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{"filter[inserted_at]": "2026-01-15T10:30:00.200Z"},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
def test_findings_list_updated_at_accepts_timestamp_precision_filters(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
early_finding, late_finding, *_ = findings_fixture
|
||||
early_updated_at = datetime(2026, 1, 15, 10, 30, 0, 100000, tzinfo=UTC)
|
||||
late_updated_at = datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC)
|
||||
Finding.all_objects.filter(pk=early_finding.pk).update(
|
||||
updated_at=early_updated_at
|
||||
)
|
||||
Finding.all_objects.filter(pk=late_finding.pk).update(
|
||||
updated_at=late_updated_at
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[updated_at.gte]": "2026-01-15T10:30:00.150Z",
|
||||
"filter[updated_at.lte]": "2026-01-15T10:30:00.250Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{"filter[updated_at]": "2026-01-15T10:30:00.200Z"},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
def test_findings_list_inserted_at_and_updated_at_filters_are_combined(
|
||||
self, authenticated_client, scans_fixture
|
||||
):
|
||||
scan, *_ = scans_fixture
|
||||
|
||||
def create_finding(uid, inserted_at, updated_at):
|
||||
finding = Finding.objects.create(
|
||||
id=datetime_to_uuid7(inserted_at),
|
||||
tenant_id=scan.tenant_id,
|
||||
uid=uid,
|
||||
scan=scan,
|
||||
status=Status.FAIL,
|
||||
status_extended="timestamp precision status",
|
||||
impact=Severity.medium,
|
||||
severity=Severity.medium,
|
||||
check_id="timestamp_precision_check",
|
||||
check_metadata={
|
||||
"CheckId": "timestamp_precision_check",
|
||||
"Description": "timestamp precision check",
|
||||
"servicename": "ec2",
|
||||
},
|
||||
first_seen_at=inserted_at,
|
||||
)
|
||||
Finding.all_objects.filter(pk=finding.pk).update(
|
||||
inserted_at=inserted_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
finding.refresh_from_db()
|
||||
return finding
|
||||
|
||||
matching_finding = create_finding(
|
||||
"timestamp_precision_combined_match",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC),
|
||||
datetime(2026, 1, 15, 11, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
create_finding(
|
||||
"timestamp_precision_combined_inserted_only",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC),
|
||||
datetime(2026, 1, 15, 12, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
create_finding(
|
||||
"timestamp_precision_combined_updated_only",
|
||||
datetime(2026, 1, 15, 9, 30, 0, 200000, tzinfo=UTC),
|
||||
datetime(2026, 1, 15, 11, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[inserted_at.gte]": "2026-01-15T10:30:00.150Z",
|
||||
"filter[inserted_at.lte]": "2026-01-15T10:30:00.250Z",
|
||||
"filter[updated_at.gte]": "2026-01-15T11:30:00.150Z",
|
||||
"filter[updated_at.lte]": "2026-01-15T11:30:00.250Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {matching_finding.uid}
|
||||
|
||||
def test_findings_list_resource_tags_no_n_plus_one(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
@@ -7694,6 +7879,23 @@ class TestFindingViewSet:
|
||||
]
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filter_name",
|
||||
["inserted_at", "inserted_at.gte", "inserted_at.lte"],
|
||||
)
|
||||
def test_findings_metadata_rejects_timestamp_precision_filters(
|
||||
self, authenticated_client, filter_name
|
||||
):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata"),
|
||||
{f"filter[{filter_name}]": "2048-01-01T10:30:00Z"},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
error = response.json()["errors"][0]
|
||||
assert error["detail"] == "Enter a valid date."
|
||||
assert error["code"] == "invalid"
|
||||
|
||||
def test_findings_metadata_backfill(
|
||||
self, authenticated_client, scans_fixture, findings_fixture
|
||||
):
|
||||
|
||||
@@ -50,6 +50,7 @@ from api.filters import (
|
||||
FindingGroupAggregatedComputedFilter,
|
||||
FindingGroupFilter,
|
||||
FindingGroupSummaryFilter,
|
||||
FindingMetadataFilter,
|
||||
IntegrationFilter,
|
||||
IntegrationJiraFindingsFilter,
|
||||
InvitationFilter,
|
||||
@@ -3833,6 +3834,8 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
def get_filterset_class(self):
|
||||
if self.action in ["latest", "metadata_latest"]:
|
||||
return LatestFindingFilter
|
||||
if self.action == "metadata":
|
||||
return FindingMetadataFilter
|
||||
return FindingFilter
|
||||
|
||||
def get_queryset(self):
|
||||
|
||||
@@ -171,6 +171,8 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
update_tag=tmp_cartography_config.update_tag,
|
||||
)
|
||||
|
||||
graph_database.verify_scan_databases_available()
|
||||
|
||||
# Starting the Attack Paths scan
|
||||
if not db_utils.starting_attack_paths_scan(
|
||||
attack_paths_scan, tenant_cartography_config
|
||||
|
||||
@@ -34,6 +34,13 @@ SYNC_RESULT_EMPTY = {
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAttackPathsRun:
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_graph_database_preflight(self):
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.verify_scan_databases_available"
|
||||
) as mock_preflight:
|
||||
yield mock_preflight
|
||||
|
||||
# Patching with decorators as we got a `SyntaxError: too many statically nested blocks` error if we use context managers
|
||||
@patch("tasks.jobs.attack_paths.scan.graph_database.drop_database")
|
||||
@patch(
|
||||
@@ -190,6 +197,64 @@ class TestAttackPathsRun:
|
||||
# don't switch to the new catalog/sink before the graph is live.
|
||||
mock_set_scan_migrated.assert_called_once_with(attack_paths_scan, True, "neo4j")
|
||||
|
||||
def test_run_preflight_failure_does_not_start_scan(
|
||||
self,
|
||||
mock_graph_database_preflight,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
scans_fixture,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
mock_graph_database_preflight.side_effect = RuntimeError("graph unavailable")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(_enabled_regions=["us-east-1"]),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_ingest_uri",
|
||||
return_value="bolt://neo4j",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
|
||||
return_value=attack_paths_scan,
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=MagicMock(return_value={}),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"
|
||||
) as mock_starting,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.create_database"
|
||||
) as mock_create_db,
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="graph unavailable"):
|
||||
attack_paths_run(str(tenant.id), str(scan.id), "task-123")
|
||||
|
||||
mock_graph_database_preflight.assert_called_once_with()
|
||||
mock_starting.assert_not_called()
|
||||
mock_create_db.assert_not_called()
|
||||
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
|
||||
return_value="Cartography failed: ingestion boom",
|
||||
|
||||
Generated
+43
-10
@@ -135,7 +135,6 @@ constraints = [
|
||||
{ name = "debugpy", specifier = "==1.8.20" },
|
||||
{ name = "decorator", specifier = "==5.2.1" },
|
||||
{ name = "defusedxml", specifier = "==0.7.1" },
|
||||
{ name = "detect-secrets", specifier = "==1.5.0" },
|
||||
{ name = "dill", specifier = "==0.4.1" },
|
||||
{ name = "distro", specifier = "==1.9.0" },
|
||||
{ name = "dj-rest-auth", specifier = "==7.0.1" },
|
||||
@@ -218,6 +217,7 @@ constraints = [
|
||||
{ name = "jsonschema", specifier = "==4.23.0" },
|
||||
{ name = "jsonschema-specifications", specifier = "==2025.9.1" },
|
||||
{ name = "keystoneauth1", specifier = "==5.13.0" },
|
||||
{ name = "kingfisher-bin", specifier = "==1.104.0" },
|
||||
{ name = "kiwisolver", specifier = "==1.4.9" },
|
||||
{ name = "knack", specifier = "==0.11.0" },
|
||||
{ name = "kombu", specifier = "==5.6.2" },
|
||||
@@ -2245,16 +2245,15 @@ wheels = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "detect-secrets"
|
||||
version = "1.5.0"
|
||||
name = "deprecated"
|
||||
version = "1.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
{ name = "wrapt" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/69/67/382a863fff94eae5a0cf05542179169a1c49a4c8784a9480621e2066ca7d/detect_secrets-1.5.0.tar.gz", hash = "sha256:6bb46dcc553c10df51475641bb30fd69d25645cc12339e46c824c1e0c388898a", size = 97351, upload-time = "2024-05-06T17:46:19.721Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/5e/4f5fe4b89fde1dc3ed0eb51bd4ce4c0bca406246673d370ea2ad0c58d747/detect_secrets-1.5.0-py3-none-any.whl", hash = "sha256:e24e7b9b5a35048c313e983f76c4bd09dad89f045ff059e354f9943bf45aa060", size = 120341, upload-time = "2024-05-06T17:46:16.628Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3613,6 +3612,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/99/76476a1057b349c860bae72e45d6ef438feb877c84ee7d565faf464e54c3/keystoneauth1-5.13.0-py3-none-any.whl", hash = "sha256:5ab81412eb0923ceb9c602cc3decce514b399523cb83d16b409ed3b0f9b03d41", size = 343585, upload-time = "2026-01-19T10:47:00.762Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kingfisher-bin"
|
||||
version = "1.104.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/2b/324212f1baf482a7d4b66a2edf33073336735b67bb6b04a38d18fd9e67fb/kingfisher_bin-1.104.0-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:8e3840e67004a971fef80aba240ee5c3c5f7a3a343a6d1083a2751aaf866d5d3", size = 14057606, upload-time = "2026-06-22T03:03:01.419Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/0a/cbf964da5102657cb9be4a59db7c9f7807ef88f9419673b7486daba785d3/kingfisher_bin-1.104.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b838313411fa2166a318a45aec2cfcc238e2f30f5292e309ca1129a73180c851", size = 12468386, upload-time = "2026-06-22T03:03:03.951Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/a0/cc7ef0ac28f147cdfc9d80e4239fff11c1329831c6f57510c929e848753c/kingfisher_bin-1.104.0-py3-none-manylinux_2_17_aarch64.musllinux_1_2_aarch64.whl", hash = "sha256:0a94abbf2154ef8a3b4845cc0240e2321cdc19e0f5c7f585ea5252e76b242f68", size = 13943188, upload-time = "2026-06-22T03:03:06.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/79/827cfd7787885798a00b5ab905bdc866ef6f8deeff0f708679b06bc9baaa/kingfisher_bin-1.104.0-py3-none-manylinux_2_17_x86_64.musllinux_1_2_x86_64.whl", hash = "sha256:f381274b946f7f68ed72911770fff72024f2192c6e2e2158f2a7fbfda8c482fb", size = 14757594, upload-time = "2026-06-22T03:03:08.66Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/93/b0061fc69cd10382f647f9266823f213fd0b3f168f8b5bd9151a2370abb1/kingfisher_bin-1.104.0-py3-none-win_amd64.whl", hash = "sha256:f228d0dd61a738673b1c536e965a5661a83b1ee6ca64186a46ba6ea81ab4fd0b", size = 27697957, upload-time = "2026-06-22T03:03:11.268Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/fb/f062665b4eb3f77e799cb6335e56bc2945aea83787888a6c1ab329858d0a/kingfisher_bin-1.104.0-py3-none-win_arm64.whl", hash = "sha256:a7774d9d11815ca946bd80b8c9df0f1d39c36cb5a21def3323b99d148dc63065", size = 26063704, upload-time = "2026-06-22T03:03:14.08Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kiwisolver"
|
||||
version = "1.4.9"
|
||||
@@ -3706,6 +3718,20 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/08/10/9f8af3e6f569685ce3af7faab51c8dd9d93b9c38eba339ca31c746119447/kubernetes-32.0.1-py2.py3-none-any.whl", hash = "sha256:35282ab8493b938b08ab5526c7ce66588232df00ef5e1dbe88a419107dc10998", size = 1988070, upload-time = "2025-02-18T21:06:31.391Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linode-api4"
|
||||
version = "5.45.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "deprecated" },
|
||||
{ name = "polling" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b2/b5/fce03d9b81008dcc0fe4961ce10e140ac3ae5ab17f2cdd659763e4964c0d/linode_api4-5.45.0.tar.gz", hash = "sha256:af8a0a5638345ad467447112dcf5d58ec47e7dd192b89ce0c8537a1e5c435d04", size = 283375, upload-time = "2026-06-11T18:05:13.671Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/18/38/19e3c8f7b7a9dbeea2aa5af61f70162bff5131b3d39acbe73e8d0dd12972/linode_api4-5.45.0-py3-none-any.whl", hash = "sha256:3cc2650b13d8d3bc7735fa8e92a639669618f320471dc8e519db778c6020eacd", size = 158336, upload-time = "2026-06-11T18:05:11.799Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lxml"
|
||||
version = "6.1.0"
|
||||
@@ -4525,6 +4551,12 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/41/f5/65b66420c275e9b26513fdd6d84687403d11ac8be4650b67d1e5572b8f48/policyuniverse-1.5.1.20231109-py2.py3-none-any.whl", hash = "sha256:0b0ece0ee8285af31fc39ce09c82a551ca62e62bc2842e23952503bccb973321", size = 484251, upload-time = "2023-11-30T19:12:43.463Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polling"
|
||||
version = "0.3.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8f/c5/4249317962180d97ec7a60fe38aa91f86216533bd478a427a5468945c5c9/polling-0.3.2.tar.gz", hash = "sha256:3afd62320c99b725c70f379964bf548b302fc7f04d4604e6c315d9012309cc9a", size = 5189, upload-time = "2021-05-22T19:48:41.466Z" }
|
||||
|
||||
[[package]]
|
||||
name = "portalocker"
|
||||
version = "2.10.1"
|
||||
@@ -4641,8 +4673,8 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "5.31.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#b5bb85c9564f6ca6a7f66c851bb56bde719205ee" }
|
||||
version = "5.32.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#5dac8a0a53272e4db68c476fb969dc03e88beb68" }
|
||||
dependencies = [
|
||||
{ name = "alibabacloud-actiontrail20200706" },
|
||||
{ name = "alibabacloud-credentials" },
|
||||
@@ -4693,13 +4725,14 @@ dependencies = [
|
||||
{ name = "dash" },
|
||||
{ name = "dash-bootstrap-components" },
|
||||
{ name = "defusedxml" },
|
||||
{ name = "detect-secrets" },
|
||||
{ name = "dulwich" },
|
||||
{ name = "google-api-python-client" },
|
||||
{ name = "google-auth-httplib2" },
|
||||
{ name = "h2" },
|
||||
{ name = "jsonschema" },
|
||||
{ name = "kingfisher-bin" },
|
||||
{ name = "kubernetes" },
|
||||
{ name = "linode-api4" },
|
||||
{ name = "markdown" },
|
||||
{ name = "microsoft-kiota-abstractions" },
|
||||
{ name = "msgraph-sdk" },
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Build command
|
||||
# docker build --platform=linux/amd64 --no-cache -t prowler:latest .
|
||||
|
||||
ARG PROWLER_VERSION=latest@sha256:4b796c6df40a3350c7947747b59bdda230d0da6222287500e13b0a8e1574aad4
|
||||
ARG PROWLER_VERSION=latest@sha256:ebb4ab999f10cb7e7c256226c2873de9b3bf2f3d855f385e0164bcf34104bfba
|
||||
|
||||
FROM toniblyx/prowler:${PROWLER_VERSION}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:alpine@sha256:8b1e78743a03dbb2c95171cc58639fef29abc8816598e27fb910ed2e621e589a
|
||||
image: nginx:alpine@sha256:54f2a904c251d5a34adf545a72d32515a15e08418dae0266e23be2e18c66fefa
|
||||
container_name: prowler-nginx
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
|
||||
@@ -445,3 +445,5 @@ The metadata structure is enforced in code using a Pydantic model. For reference
|
||||
## Specific Check Patterns
|
||||
|
||||
Details for specific providers can be found in documentation pages named using the pattern `<provider_name>-details`.
|
||||
|
||||
Checks that scan resources for plaintext secrets follow a dedicated batched structure. Refer to [Secret-Scanning Checks](/developer-guide/secret-scanning-checks) before creating or updating one.
|
||||
|
||||
@@ -42,7 +42,7 @@ When adding a new configurable check to Prowler, update the following files:
|
||||
```
|
||||
- **Provider Schema:** Add the typed field to the provider's Pydantic schema in `prowler/config/schema/<provider>.py`. This is required: the loader validates user configs against these schemas and the shipped `config.yaml` must round-trip with zero warnings. See [Adding a Parameter to the Provider Schema](#adding-a-parameter-to-the-provider-schema) below.
|
||||
- **Test Fixtures:** If tests depend on this configuration, add the variable to `tests/config/fixtures/config.yaml`.
|
||||
- **Documentation:** Document the new variable in the list of configurable checks in `docs/tutorials/configuration_file.md`.
|
||||
- **Documentation:** Document the new variable in the list of configurable checks in [Configuration File](/user-guide/cli/tutorials/configuration_file) (`docs/user-guide/cli/tutorials/configuration_file.mdx`).
|
||||
|
||||
For a complete list of checks that already support configuration, see the [Configuration File Tutorial](/user-guide/cli/tutorials/configuration_file).
|
||||
|
||||
@@ -153,7 +153,6 @@ Only fields with a numeric range, a fixed value set, or a length cap are listed.
|
||||
| `max_days_secret_unused` | `7..365` days | |
|
||||
| `max_days_secret_unrotated` | `1..180` days | NIST IA-5: rotate quarterly; CIS ≤90 |
|
||||
| `min_kinesis_stream_retention_hours` | `24..8760` h | 1 day .. 1 year |
|
||||
| `detect_secrets_plugins[].limit` | `0.0..10.0` | Shannon entropy threshold |
|
||||
| `shodan_api_key` | ≤512 chars | |
|
||||
|
||||
### Azure
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
---
|
||||
title: 'Secret-Scanning Checks'
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
Prowler scans audited resources for plaintext secrets using [Kingfisher](https://github.com/mongodb/kingfisher), an open-source secret-scanning engine that Prowler invokes as a subprocess. This guide explains the structure every secret-scanning check must follow to keep scanning correct and efficient on large accounts.
|
||||
|
||||
<Note>
|
||||
Since Prowler 5.32.0 the secret-scanning checks scan with Kingfisher. Earlier versions used the `detect-secrets` library.
|
||||
</Note>
|
||||
|
||||
## Overview
|
||||
|
||||
Secret detection runs through a single helper in `prowler/lib/utils/utils.py`:
|
||||
|
||||
- **`detect_secrets_scan_batch(payloads, excluded_secrets=..., validate=...)`** scans many payloads in chunked subprocess invocations and returns a `{key: [findings]}` dictionary. To scan a single payload, pass a one-entry mapping (for example, `{0: data}`).
|
||||
|
||||
Every Kingfisher invocation carries a fixed process-startup cost (around 100 ms). Scanning once per resource would spawn thousands of subprocesses on large accounts (for example, thousands of CloudWatch log groups). `detect_secrets_scan_batch` amortizes that cost: it writes each payload to a temporary file as it consumes them, runs one subprocess per chunk (500 payloads by default), and maps the findings back to each payload by key.
|
||||
|
||||
## The Batched Structure
|
||||
|
||||
Every secret-scanning check follows three phases.
|
||||
|
||||
### Phase 1: Collect
|
||||
|
||||
Define a generator that yields `(key, payload)` for each scannable unit. The generator builds payload strings only — it does not call Kingfisher. Lazy yielding keeps memory and temporary-disk usage bounded to a single chunk, which matters when an account holds thousands of resources.
|
||||
|
||||
### Phase 2: Batch
|
||||
|
||||
Call `detect_secrets_scan_batch` once with the generator. The helper consumes it in chunks, runs Kingfisher per chunk, and returns the keys that produced findings mapped to their finding lists.
|
||||
|
||||
### Phase 3: Report
|
||||
|
||||
Iterate the resources, look up the findings by key, and build one report per resource. Emit a finding for **every** iterated resource — never drop one silently. When a resource's payload cannot be prepared for scanning (for example, user data that fails to base64-decode or decompress), report it as `MANUAL` with a status explaining the scan could not inspect it, rather than omitting it or claiming `PASS`.
|
||||
|
||||
```python
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import (
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.example.example_client import example_client
|
||||
|
||||
|
||||
class example_resource_no_secrets(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
excluded = example_client.audit_config.get("secrets_ignore_patterns", [])
|
||||
validate = example_client.audit_config.get("secrets_validate", False)
|
||||
resources = list(example_client.resources)
|
||||
|
||||
# Phase 1: collect — builds strings only, no scan.
|
||||
def payloads():
|
||||
for index, resource in enumerate(resources):
|
||||
if resource.scannable_data:
|
||||
yield index, serialize(resource)
|
||||
|
||||
# Phase 2: batch — one call, chunked subprocesses.
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=excluded, validate=validate
|
||||
)
|
||||
|
||||
# Phase 3: report — look up findings by key.
|
||||
for index, resource in enumerate(resources):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=resource)
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in {resource.name}."
|
||||
detect_secrets_output = batch_results.get(index)
|
||||
if detect_secrets_output:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = (
|
||||
f"Potential secret found in {resource.name} -> ..."
|
||||
)
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
```
|
||||
|
||||
## Choosing the Key
|
||||
|
||||
The key maps each finding back to its source. Two shapes cover every check:
|
||||
|
||||
- **One payload per resource:** use the resource index. This fits checks that serialize a single payload per resource, such as launch configurations, CloudFormation outputs, SSM documents, Step Functions definitions, and OpenStack metadata.
|
||||
- **Several payloads per resource:** use a `(resource_index, fragment)` tuple, where the fragment identifies the variable, log stream, container, file, or version. Phase 3 groups the per-fragment findings to build the resource report. This fits CloudWatch log streams, ECS containers, CodeBuild variables, Glue arguments, and Lambda code files.
|
||||
|
||||
Derive the indices from the same `list(...)` of resources in both Phase 1 and Phase 3 so the order stays stable and the keys align.
|
||||
|
||||
## Preserving Per-Payload Results
|
||||
|
||||
`detect_secrets_scan_batch` runs Kingfisher with `--no-dedup`, so a secret that appears in more than one payload is reported for each one. This reproduces the result of scanning each payload individually. Build payload strings exactly as a single scan would: serialize the same data and keep line ordering, because messages often map a finding's `line_number` back to a variable name or metadata key.
|
||||
|
||||
## Validation and Severity
|
||||
|
||||
`detect_secrets_scan_batch` accepts `validate`, read from `secrets_validate` in the provider configuration or the `--scan-secrets-validate` flag. When enabled, Kingfisher confirms whether each secret is live, and confirmed secrets carry `is_verified: True`.
|
||||
|
||||
After marking a report as `FAIL`, pass the findings to `annotate_verified_secrets(report, findings)`. When any secret is verified, the helper escalates the finding to critical severity and appends a note that the secret was confirmed live. Validation stays off by default because it sends the discovered secret to the provider API.
|
||||
|
||||
## Excluded Secrets
|
||||
|
||||
`detect_secrets_scan_batch` applies `secrets_ignore_patterns` — regular expressions from the provider configuration — against each finding's source line and drops the matches, mirroring single-scan behavior.
|
||||
|
||||
## Testing
|
||||
|
||||
To assert on the verified-secret path, mock `detect_secrets_scan_batch` in the check module and return the keyed dictionary. For a single resource scanned at index `0`:
|
||||
|
||||
```python
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.example.example_resource_no_secrets.example_resource_no_secrets.detect_secrets_scan_batch",
|
||||
return_value={
|
||||
0: [{"type": "...", "line_number": 1, "is_verified": True}]
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
Most tests need no mock at all: they seed resources that contain example secrets and assert on the `FAIL` status and message, which exercises the real batched path. Refer to the [Testing](/developer-guide/unit-testing) documentation for the general structure.
|
||||
+4
-1
@@ -237,6 +237,7 @@
|
||||
"user-guide/providers/azure/authentication",
|
||||
"user-guide/providers/azure/use-non-default-cloud",
|
||||
"user-guide/providers/azure/subscriptions",
|
||||
"user-guide/providers/azure/resource-groups",
|
||||
"user-guide/providers/azure/create-prowler-service-principal"
|
||||
]
|
||||
},
|
||||
@@ -359,7 +360,8 @@
|
||||
"group": "Okta",
|
||||
"pages": [
|
||||
"user-guide/providers/okta/getting-started-okta",
|
||||
"user-guide/providers/okta/authentication"
|
||||
"user-guide/providers/okta/authentication",
|
||||
"user-guide/providers/okta/retry-configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -398,6 +400,7 @@
|
||||
"developer-guide/provider",
|
||||
"developer-guide/services",
|
||||
"developer-guide/checks",
|
||||
"developer-guide/secret-scanning-checks",
|
||||
"developer-guide/outputs",
|
||||
"developer-guide/integrations",
|
||||
"developer-guide/security-compliance-framework",
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
export const SubscriptionBanner = ({ children }) => {
|
||||
return (
|
||||
<Note>
|
||||
This feature is available exclusively in <b>Prowler Cloud</b> and <b>Prowler Enterprise</b> with a <a href="https://prowler.com/pricing">subscription</a>.
|
||||
{children}
|
||||
</Note>
|
||||
);
|
||||
};
|
||||
@@ -2,6 +2,8 @@
|
||||
title: "Configuration File"
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
Several Prowler's checks have user configurable variables that can be modified in a common **configuration file**. This file can be found in the following [path](https://github.com/prowler-cloud/prowler/blob/master/prowler/config/config.yaml):
|
||||
|
||||
```
|
||||
@@ -24,6 +26,7 @@ The following list includes all the AWS checks with configurable variables that
|
||||
|---------------------------------------------------------------|--------------------------------------------------|-----------------|
|
||||
| `acm_certificates_expiration_check` | `days_to_expire_threshold` | Integer |
|
||||
| `acmpca_certificate_authority_pqc_key_algorithm` | `acmpca_pqc_key_algorithms` | List of Strings |
|
||||
| `apigateway_restapi_no_secrets_in_stage_variables` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `appstream_fleet_maximum_session_duration` | `max_session_duration_seconds` | Integer |
|
||||
| `appstream_fleet_session_disconnect_timeout` | `max_disconnect_timeout_in_seconds` | Integer |
|
||||
| `appstream_fleet_session_idle_disconnect_timeout` | `max_idle_disconnect_timeout_in_seconds` | Integer |
|
||||
@@ -86,6 +89,91 @@ The following list includes all the AWS checks with configurable variables that
|
||||
| `vpc_endpoint_services_allowed_principals_trust_boundaries` | `trusted_account_ids` | List of Strings |
|
||||
| `opensearch_service_domains_not_publicly_accessible` | `trusted_ips` | List of Strings |
|
||||
|
||||
### Resource Scan Limit
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
Some AWS services accumulate large numbers of resources (EBS snapshots, backup recovery points, CloudWatch log groups, Lambda functions, ECS task definitions, and CodeArtifact packages). Scanning every resource increases scan time, cost, API throttling, and finding volume. By default, Prowler scans every resource. Configure a positive resource scan limit to cap how many resources Prowler analyzes for these high-volume AWS resource paths.
|
||||
|
||||
The global default applies to the supported resources below and is overridable per resource. The default global value is `0`, which disables the limit and scans every resource. A global `null` value is also unlimited. For per-resource values, `null` means inherit the global default; set `0` or a negative value to disable that resource limit explicitly. Positive values enable limits.
|
||||
|
||||
<Warning>
|
||||
When positive resource scan limits are configured, compliance results are based only on the selected resources, not on the full set of matching resources in the account. Treat compliance summaries and percentages as partial evidence, because unselected resources are not analyzed and can change the real compliance posture.
|
||||
</Warning>
|
||||
|
||||
#### Global Behavior
|
||||
|
||||
Resource scan limits select resources for analysis. They do not cap, prioritize, or reorder findings.
|
||||
|
||||
* **`0`, negative, or global `null` values:** Disable the limit and keep the legacy behavior for that resource path. Prowler analyzes every discovered matching resource.
|
||||
* **Positive values:** Select at most that many resources for the affected resource path. A selected resource can produce zero, one, or many findings.
|
||||
* **No PASS/FAIL prioritization:** Prowler does not inspect the compliance result before selecting resources. Limits do not prefer failed resources, passed resources, or resources with more findings.
|
||||
* **Latest-first where possible:** When AWS exposes timestamps or useful ordering, Prowler selects the newest resources first. When AWS only exposes API order, Prowler preserves that API order and documents the behavior as best effort.
|
||||
* **Findings are downstream:** Checks only evaluate the resources exposed by the service client after selection. Findings from unselected resources are not produced because those resources are not analyzed.
|
||||
|
||||
Exact list API call reduction depends on each AWS API's ordering and pagination capabilities. When Prowler must enumerate candidates locally to select the latest resources, list calls may still read candidates, but expensive per-resource enrichment calls are bounded to the selected resources for the supported paths below.
|
||||
|
||||
#### Full Collections Versus Limited Analysis Sets
|
||||
|
||||
Some checks need lightweight evidence from a complete resource collection to avoid incorrect cross-service conclusions, while other checks perform primary analysis on a limited resource set.
|
||||
|
||||
Prowler keeps full lightweight collections where they are needed for cross-service evidence. For example:
|
||||
|
||||
* **Lambda security groups and regions:** Prowler records security groups used by all discovered Lambda functions and the regions where functions exist before it limits Lambda functions for primary Lambda checks. This helps Amazon EC2 and Amazon Inspector checks avoid false positives such as treating Lambda security groups as unused or assuming a region has no Lambda functions.
|
||||
* **CloudWatch `all_log_groups`:** Prowler records all discovered CloudWatch log groups in `all_log_groups` before limiting the primary `log_groups` analysis set. Other services can still resolve log group evidence, while CloudWatch log group checks only analyze the selected log groups.
|
||||
|
||||
This split is intentional. It reduces expensive per-resource analysis calls without discarding lightweight context that other services need for accurate results.
|
||||
|
||||
#### Supported AWS Resource Limits
|
||||
|
||||
| Value | Scope | Type |
|
||||
|-------|-------|------|
|
||||
| `max_scanned_resources_per_service` | Global default for all supported high-volume AWS resources (default `0`, disabled/unlimited) | Integer |
|
||||
| `max_ebs_snapshots` | EBS snapshots (`ec2_ebs_*` checks) | Integer |
|
||||
| `max_backup_recovery_points` | Backup recovery points (`backup_recovery_point_*`) | Integer |
|
||||
| `max_cloudwatch_log_groups` | CloudWatch log groups (`cloudwatch_log_group_*`) | Integer |
|
||||
| `max_lambda_functions` | Lambda functions (`awslambda_function_*`) | Integer |
|
||||
| `max_ecs_task_definitions` | ECS task definitions (`ecs_task_definitions_*`) | Integer |
|
||||
| `max_codeartifact_packages` | CodeArtifact packages (`codeartifact_packages_*`) | Integer |
|
||||
|
||||
#### Resource Limit Behavior By Resource Path
|
||||
|
||||
| Resource Path | What Prowler Discovers | What A Positive Limit Selects For Analysis | Ordering And Latest Behavior | AWS Calls Reduced | Drawbacks And Consequences |
|
||||
|---------------|------------------------|--------------------------------------------|------------------------------|-------------------|----------------------------|
|
||||
| EBS snapshots (`max_ebs_snapshots`) | Prowler lists self-owned snapshots and keeps lightweight evidence that volumes and regions have snapshots. | The selected EBS snapshots exposed to `ec2_ebs_*` checks. | Prowler sorts discovered snapshots by `StartTime` newest first, then applies the limit. Snapshots without a timestamp sort last. | Bounds expensive per-snapshot public attribute checks to selected snapshots. Snapshot listing still runs so Prowler can choose the newest snapshots and keep volume/region evidence. | Older unselected snapshots are not analyzed by snapshot checks. A public, unencrypted, or otherwise noncompliant older snapshot can be missed when the limit is lower than the number of snapshots. |
|
||||
| Backup recovery points (`max_backup_recovery_points`) | Prowler lists backup vaults, plans, selections, and recovery point candidates in discovered vaults. | The selected recovery points exposed to `backup_recovery_point_*` checks and tag hydration. | Prowler sorts discovered recovery points by `CreationDate` newest first across vaults, then applies the limit. Recovery points without a timestamp sort last. | Bounds recovery point tag calls to selected recovery points. Vault and recovery point list calls still run so Prowler can choose the newest points. | Older unselected recovery points are not analyzed. A nonencrypted or otherwise noncompliant older recovery point can be missed. |
|
||||
| CloudWatch log groups (`max_cloudwatch_log_groups`) | Prowler lists log groups into both `all_log_groups` and the primary `log_groups` collection. `all_log_groups` remains available as lightweight cross-service evidence. | The selected log groups exposed to `cloudwatch_log_group_*` checks, tag hydration, and log event retrieval for checks that need log contents. | Prowler sorts discovered log groups by `creationTime` newest first, then applies the limit. Log groups without a creation time sort last. | Bounds tag calls and log event retrieval to selected log groups. Log group listing still runs to build `all_log_groups` and choose newest log groups. | Older unselected log groups are not analyzed by CloudWatch log group checks. Retention, encryption, or secrets-in-logs issues in older log groups can be missed, although cross-service evidence can still use `all_log_groups`. |
|
||||
| Lambda functions (`max_lambda_functions`) | Prowler lists Lambda functions and records lightweight security group and region evidence for all discovered functions. | The selected Lambda functions exposed to `awslambda_function_*` checks and per-function enrichment such as tags, policies, function URLs, and event source mappings. | Prowler sorts discovered functions by `LastModified` newest first, then applies the limit. Functions without `LastModified` sort last. | Bounds per-function enrichment calls to selected functions. Function listing still runs to choose newest functions and keep security group/region evidence. | Older unselected functions are not analyzed by Lambda checks. Runtime, policy, URL, environment secret, or dead-letter queue issues in older functions can be missed. Cross-service checks can still use full Lambda security group and region evidence to avoid false positives. |
|
||||
| ECS task definitions (`max_ecs_task_definitions`) | Prowler lists ECS task definition ARN candidates in each region. Candidate ARNs can remain visible and discoverable through AWS list operations, even when not all are described. | The selected task definitions that Prowler describes and exposes to `ecs_task_definitions_*` checks. | Selection is not random. Prowler calls `ListTaskDefinitions` with `sort=DESC`, which asks AWS to return task definition ARNs in descending family and revision order. Prowler then interleaves regional candidate lists to avoid starving later regions before applying the limit. This selects the latest task definition revisions according to the ARN order AWS provides, while preserving regional fairness. | Bounds `DescribeTaskDefinition` calls to selected task definitions. Prowler may still list candidates so it can select the bounded set and keep discovery deterministic. | Unselected task definitions are not described or analyzed. Issues in older task definition revisions, or in lower-priority families outside the selected AWS `sort=DESC` order, can be missed. Because ECS ordering is family/revision based rather than a registration timestamp sort across every family, this is latest-first according to AWS task definition ARN ordering, not a global newest-by-time guarantee. |
|
||||
| CodeArtifact packages (`max_codeartifact_packages`) | Prowler lists CodeArtifact repositories and lazily lists packages inside them. | The selected packages exposed to `codeartifact_packages_*` checks, including latest-version metadata for those packages. | AWS `ListPackages` does not provide a newest-package timestamp ordering in this path. Prowler preserves repository order and package API order, then applies the limit. Latest package version metadata is retrieved for selected packages with `sortBy=PUBLISHED_TIME` and `maxResults=1`. | Bounds `ListPackageVersions` calls to selected packages and can stop package listing once the limit is reached. Repository listing still runs. | Package selection is best effort by API order, not newest package order. Packages outside the selected repository/API order are not analyzed, so origin restriction or latest-version issues can be missed. |
|
||||
|
||||
Use limits when scan duration, API throttling, or cost are more important than exhaustive coverage for these high-volume resources. Keep limits disabled when you need complete evidence for every resource in the affected checks.
|
||||
|
||||
### Validating Discovered Secrets
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
By default, the secret-scanning checks run fully offline: secrets are detected but never sent anywhere. Setting `secrets_validate` to `True` additionally confirms whether each discovered secret is live by authenticating with it against the corresponding provider API. The discovered secret itself serves as the credential, so Prowler requires no additional permissions to validate it.
|
||||
|
||||
`secrets_validate` applies to every AWS secret-scanning check listed above (those that accept `secrets_ignore_patterns`). The `--scan-secrets-validate` CLI flag is provider-wide: it also enables validation for the secret-scanning checks of other providers, such as the OpenStack metadata checks.
|
||||
|
||||
To enable validation through the configuration file, set the value under the `aws` section:
|
||||
|
||||
```yaml
|
||||
aws:
|
||||
secrets_validate: True
|
||||
```
|
||||
|
||||
To enable validation for a single scan (any provider), use Prowler CLI:
|
||||
|
||||
```
|
||||
prowler aws --scan-secrets-validate
|
||||
```
|
||||
|
||||
<Warning>
|
||||
Secret validation makes outbound network calls that authenticate with each discovered secret. The credential is exercised against the provider, so the call appears in the audited account's logs and can trigger its monitoring (for example, AWS CloudTrail records the validation request). Validation stays disabled by default so that scans remain fully offline.
|
||||
</Warning>
|
||||
|
||||
|
||||
## Azure
|
||||
|
||||
@@ -191,6 +279,19 @@ aws:
|
||||
# AWS Global Configuration
|
||||
# aws.mute_non_default_regions --> Set to True to muted failed findings in non-default regions for AccessAnalyzer, GuardDuty, SecurityHub, DRS and Config
|
||||
mute_non_default_regions: False
|
||||
|
||||
# AWS Resource Scan Limit Configuration
|
||||
# Disabled by default: scan every resource unless a positive limit is configured.
|
||||
# Findings are not capped. Set to 0 (or a negative value) to disable the limit.
|
||||
# aws.max_scanned_resources_per_service --> global default for all services below
|
||||
max_scanned_resources_per_service: 0
|
||||
# Per-service overrides. Leave as null to fall back to the global default.
|
||||
max_ebs_snapshots: null
|
||||
max_backup_recovery_points: null
|
||||
max_cloudwatch_log_groups: null
|
||||
max_lambda_functions: null
|
||||
max_ecs_task_definitions: null
|
||||
max_codeartifact_packages: null
|
||||
# If you want to mute failed findings only in specific regions, create a file with the following syntax and run it with `prowler aws -w mutelist.yaml`:
|
||||
# Mutelist:
|
||||
# Accounts:
|
||||
|
||||
@@ -6,20 +6,34 @@ Prowler has some checks that analyse pentesting risks (Secrets, Internet Exposed
|
||||
|
||||
## Detect Secrets
|
||||
|
||||
Prowler uses `detect-secrets` library to search for any secrets that are stores in plaintext within your environment.
|
||||
Prowler scans for secrets stored in plaintext within the audited environment using [Kingfisher](https://github.com/mongodb/kingfisher), an open-source secret-scanning engine. By default these scans run fully offline, so no data leaves the audited environment. Discovered secrets can optionally be validated against the provider APIs to confirm whether they are live — see [Validating Discovered Secrets](/user-guide/cli/tutorials/configuration_file#validating-discovered-secrets).
|
||||
|
||||
The actual checks that have this functionality are the following:
|
||||
The checks with this functionality are the following.
|
||||
|
||||
AWS:
|
||||
|
||||
- apigateway\_restapi\_no\_secrets\_in\_stage\_variables
|
||||
- autoscaling\_find\_secrets\_ec2\_launch\_configuration
|
||||
- awslambda\_function\_no\_secrets\_in\_code
|
||||
- awslambda\_function\_no\_secrets\_in\_variables
|
||||
- cloudformation\_stack\_outputs\_find\_secrets
|
||||
- cloudwatch\_log\_group\_no\_secrets\_in\_logs
|
||||
- codebuild\_project\_no\_secrets\_in\_variables
|
||||
- ec2\_instance\_secrets\_user\_data
|
||||
- ec2\_launch\_template\_no\_secrets
|
||||
- ecs\_task\_definitions\_no\_environment\_secrets
|
||||
- glue\_etl\_jobs\_no\_secrets\_in\_arguments
|
||||
- ssm\_document\_secrets
|
||||
- stepfunctions\_statemachine\_no\_secrets\_in\_definition
|
||||
|
||||
To execute detect-secrets related checks, you can run the following command:
|
||||
OpenStack:
|
||||
|
||||
- compute\_instance\_metadata\_sensitive\_data
|
||||
- blockstorage\_volume\_metadata\_sensitive\_data
|
||||
- blockstorage\_snapshot\_metadata\_sensitive\_data
|
||||
- objectstorage\_container\_metadata\_sensitive\_data
|
||||
|
||||
To execute the secret-scanning checks, run the following command:
|
||||
|
||||
```console
|
||||
prowler <provider> --categories secrets
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
title: 'Azure Resource Group Scope'
|
||||
---
|
||||
|
||||
Prowler supports narrowing security scans to specific resource groups within Azure subscriptions. This is useful when you want to audit only a subset of resources rather than scanning an entire subscription.
|
||||
|
||||
By default, Prowler scans all resource groups it has permission to access. Passing `--azure-resource-group` limits the scan to only the specified resource groups across all accessible subscriptions.
|
||||
|
||||
## Configuring Resource Group Scoped Scans
|
||||
|
||||
To restrict a scan to one or more resource groups, pass them as arguments using the `--azure-resource-group` flag:
|
||||
|
||||
```console
|
||||
prowler azure --az-cli-auth --azure-resource-group <resource-group-1> <resource-group-2> ... <resource-group-N>
|
||||
```
|
||||
|
||||
For example, to scan only `rg-production` and `rg-staging`:
|
||||
|
||||
```console
|
||||
prowler azure --az-cli-auth --azure-resource-group rg-prod1 rg-prod2
|
||||
```
|
||||
|
||||
This works with all supported authentication methods:
|
||||
|
||||
```console
|
||||
# Service Principal
|
||||
prowler azure --sp-env-auth --azure-resource-group rg-production
|
||||
|
||||
# Browser
|
||||
prowler azure --browser-auth --tenant-id <tenant-id> --azure-resource-group rg-production
|
||||
|
||||
# Managed Identity
|
||||
prowler azure --managed-identity-auth --azure-resource-group rg-production
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
When `--azure-resource-group` is provided, Prowler validates each specified resource group against all accessible subscriptions. A resource group is included in the scan if it exists in **at least one** subscription.
|
||||
|
||||
- If a resource group is found in one or more subscriptions, it will be scanned in those subscriptions only.
|
||||
- If a resource group is **not found in any** subscription, Prowler logs a warning and skips it.
|
||||
- If **none** of the provided resource groups are found across any subscription, Prowler logs a warning and no resource group scoped checks will run.
|
||||
- Resource group names are matched case-insensitively, so `MyGroup` and `mygroup` are treated as the same group, mirroring Azure's own behavior.
|
||||
|
||||
<Warning>
|
||||
If `--azure-resource-group` is used, checks that apply to specific resources are limited to the relevant resource groups. But if checks that apply to tenant or subscription scope (identity, policy, or subscription-level configuration checks) are involved, then these checks will run in their natural scope.
|
||||
</Warning>
|
||||
@@ -0,0 +1,123 @@
|
||||
---
|
||||
title: "Okta Rate Limit Configuration in Prowler"
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
Prowler's Okta Provider manages API rate limits with two complementary controls:
|
||||
|
||||
- **Request throttling (proactive):** Prowler paces outbound requests through a shared limiter so scans stay under Okta's rate limits and rarely trigger a rate-limit response in the first place.
|
||||
- **Retries (reactive):** When Okta still returns a rate-limit response (HTTP 429), the official Okta Python SDK reads the `X-Rate-Limit-Reset` header and waits until the window resets before retrying. This acts as a safety net for occasional bursts.
|
||||
|
||||
Both controls are configurable through the configuration file or command line flags.
|
||||
|
||||
## Request Throttling (Requests per Second)
|
||||
|
||||
Throttling is the primary control for avoiding rate limits. Prowler limits the aggregate number of Okta API requests per second across every service in a scan.
|
||||
|
||||
### Using the Command Line Flag
|
||||
|
||||
```bash
|
||||
prowler okta --okta-requests-per-second 4
|
||||
```
|
||||
|
||||
Set the value to `0` to disable throttling.
|
||||
|
||||
### Using the Configuration File
|
||||
|
||||
```yaml
|
||||
okta:
|
||||
# Maximum aggregate Okta API requests per second. Default: 4. Set to 0 to disable.
|
||||
okta_requests_per_second: 4
|
||||
```
|
||||
|
||||
Okta enforces rate limits per endpoint, so this single global cap is a deliberately simple control. Lower the value if scans still hit limits on large organizations; raise it to scan faster when the organization has generous limits.
|
||||
|
||||
## Retries
|
||||
|
||||
Retries cover the cases throttling does not prevent, such as short bursts or per-endpoint limits lower than the global cap.
|
||||
|
||||
### Using the Command Line Flag
|
||||
|
||||
```bash
|
||||
prowler okta --okta-retries-max-attempts 8
|
||||
```
|
||||
|
||||
### Using the Configuration File
|
||||
|
||||
```yaml
|
||||
okta:
|
||||
# Maximum retries on HTTP 429. Default: 5.
|
||||
okta_max_retries: 8
|
||||
# Per-request timeout in seconds. Default: 300.
|
||||
okta_request_timeout: 300
|
||||
```
|
||||
|
||||
The command line flags override the configuration file values.
|
||||
|
||||
## How It Works
|
||||
|
||||
- **Automatic detection:** The Okta SDK retries the retryable statuses 429, 503, and 504.
|
||||
- **Reset-aware backoff:** On a 429 response the SDK sleeps until the `X-Rate-Limit-Reset` window before each retry, rather than using a fixed delay.
|
||||
- **Bounded attempts:** `okta_max_retries` caps how many times a single request is retried. The Okta SDK default is 2, which is often too low for large organizations, so Prowler defaults to 5.
|
||||
|
||||
## Request Timeout
|
||||
|
||||
The `okta_request_timeout` setting plays a dual role in the Okta SDK:
|
||||
|
||||
- It is the per-request socket timeout, bounding how long a single HTTP call can hang.
|
||||
- It is also the total wall-clock budget for the whole retry-and-backoff loop of one request.
|
||||
|
||||
For this reason, the value defaults to 300 seconds rather than 0 (no timeout). A value of 0 leaves hung connections unbounded, while a value that is too low cuts the rate-limit waits short and reintroduces the errors. As a guideline, keep `okta_request_timeout` greater than or equal to `okta_max_retries` multiplied by 60 when raising the retry count, because Okta reset windows are typically up to one minute.
|
||||
|
||||
## Error Example Handled
|
||||
|
||||
```
|
||||
Okta HTTP 429: Too Many Requests. Hit rate limit. Retry request in 42 seconds.
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
### Debug Logging
|
||||
|
||||
To confirm that throttling and retries are active, run a scan with debug logging:
|
||||
|
||||
```bash
|
||||
prowler okta --okta-requests-per-second 4 --log-level DEBUG --log-file debuglogs.txt
|
||||
```
|
||||
|
||||
### Check the Messages
|
||||
|
||||
```bash
|
||||
grep -i "throttling\|rate limit\|retry" debuglogs.txt
|
||||
```
|
||||
|
||||
### Expected Output
|
||||
|
||||
When throttling is enabled, Prowler logs the configured rate at startup:
|
||||
|
||||
```
|
||||
Okta request throttling enabled at 4 req/s
|
||||
```
|
||||
|
||||
If a rate limit is still hit, the SDK logs the backoff:
|
||||
|
||||
```
|
||||
Hit rate limit. Retry request in 42 seconds.
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If scans continue to hit rate limits:
|
||||
|
||||
1. Lower `--okta-requests-per-second` so requests are paced more conservatively.
|
||||
2. Raise `--okta-retries-max-attempts` (and keep `okta_request_timeout` proportionally large) so the safety net absorbs more bursts.
|
||||
3. Review the rate-limit allocation for the Okta organization and request an increase if needed.
|
||||
4. Verify throttling and retry behavior with debug logging.
|
||||
|
||||
## Official References
|
||||
|
||||
- [Okta Rate Limits](https://developer.okta.com/docs/reference/rate-limits/)
|
||||
- [Okta SDK for Python](https://github.com/okta/okta-sdk-python)
|
||||
@@ -4,14 +4,13 @@ description: 'Create email alerts from Prowler Cloud findings to monitor relevan
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.26.0" />
|
||||
|
||||
Alerts notify recipients by email when security findings match saved filter conditions. Use Alerts to track high-priority findings, monitor specific providers or services, and keep teams informed about scan results that match defined criteria.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [subscription](https://prowler.com/pricing).
|
||||
</Note>
|
||||
<SubscriptionBanner />
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
@@ -226,8 +226,8 @@ Assign administrative permissions by selecting from the following options:
|
||||
|------------|-------|-------------|
|
||||
| Invite and Manage Users | All | Invite new users and manage existing ones. |
|
||||
| Manage Account | All | Adjust account settings, delete users and read/manage users permissions. |
|
||||
| Manage Scans | All | Run and review scans. |
|
||||
| Manage Providers | All | Add or modify connected providers. |
|
||||
| Manage Scans | All | Run and review scans, and manage [Scan Configuration](/user-guide/tutorials/prowler-app-scan-configuration) settings. |
|
||||
| Manage Providers | All | Add or modify connected providers, and attach or detach providers from a [Scan Configuration](/user-guide/tutorials/prowler-app-scan-configuration) (in addition to Manage Scans). |
|
||||
| Manage Integrations | All | Add or modify the Prowler Integrations. |
|
||||
| Manage Ingestions | Prowler Cloud | Allow or deny the ability to submit findings ingestion batches via the API. |
|
||||
| Manage Billing | Prowler Cloud | Access and manage billing settings and subscription information. |
|
||||
|
||||
@@ -3,53 +3,40 @@ title: 'Scan Configuration'
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
Scan Configuration lets you override Prowler's built-in scan defaults per tenant and per provider, directly from Prowler App — without editing files or redeploying. Each configuration is a small YAML document that changes how specific checks behave (thresholds, allowed values, retention windows, and so on), and you attach it to the cloud providers that should use it on their next scan.
|
||||
Scan Configuration lets you override, per provider, specific values in the default configuration Prowler's checks use during a scan. Each configuration modifies how specific checks behave, e.g.: thresholds, allowed values, retention windows, and you attach it to the providers that you want to use it on their next scan.
|
||||
|
||||
<Note>
|
||||
Scan Configuration is a **Prowler Cloud-only** feature. The open-source API does not expose the `scan-configurations` endpoints, so the menu item and provider actions described here only appear in Prowler Cloud.
|
||||
</Note>
|
||||
<SubscriptionBanner />
|
||||
|
||||
## What Is a Scan Configuration?
|
||||
|
||||
Every Prowler scan reads a set of tunable values documented in [`prowler/config/config.yaml`](https://github.com/prowler-cloud/prowler/blob/master/prowler/config/config.yaml) — for example, how many days an access key can stay unused before it's flagged, or the minimum retention period for a storage bucket. A Scan Configuration is a **partial override** of those defaults:
|
||||
A Scan Configuration lets you **override specific values, per provider**, on top of Prowler's defaults. It's merged with those defaults, not a full replacement:
|
||||
|
||||
- You include **only** the keys you want to change. Everything else falls back to Prowler's built-in defaults.
|
||||
- It is stored per tenant and applied to the **providers you attach** to it.
|
||||
- A provider type you don't add a section for keeps using `config.yaml` untouched.
|
||||
- **A provider type you do add a section for has its keys merged over `config.yaml` for that provider.** Only the keys you set are overridden; every key you leave out keeps its default from `config.yaml`, because each check falls back to the default configuration when a value isn't provided. See [How It's Applied](#how-its-applied) for details.
|
||||
- It is stored per organization and applied to the **providers you attach** to it.
|
||||
- **Attaching a provider is optional at creation time.** You can save a Scan Configuration with no providers attached and associate them later, either from the configuration's editor or from the **Providers** page (see [Attaching Providers](#attaching-providers)). It has no effect on any scan until at least one provider is attached.
|
||||
- A provider can be attached to **at most one** Scan Configuration at a time.
|
||||
- Changes take effect on the provider's **next scan** — they do not re-run past scans.
|
||||
- Changes take effect on the provider's **next scan** and do not re-run past scans.
|
||||
|
||||
This is different from the [Mutelist](/user-guide/tutorials/prowler-app-mute-findings), which hides findings. A Scan Configuration changes how the checks themselves evaluate your resources.
|
||||
|
||||
## Where to Find It
|
||||
The full set of configurable values and their defaults lives in [`prowler/config/config.yaml`](https://github.com/prowler-cloud/prowler/blob/master/prowler/config/config.yaml). For what each key means and which checks read it, see the [Configuration File tutorial](/user-guide/cli/tutorials/configuration_file).
|
||||
|
||||
In Prowler Cloud, open **Configuration → Scan** in the sidebar, or go directly to `/scans/config`. The page lists every Scan Configuration in your tenant, with search by name and a filter by provider.
|
||||
## Required Permissions
|
||||
|
||||
## Creating a Scan Configuration
|
||||
Scan Configuration access is governed by Role-Based Access Control (RBAC). See [RBAC Administrative Permissions](/user-guide/tutorials/prowler-app-rbac#rbac-administrative-permissions) for details on each permission.
|
||||
|
||||
<Steps>
|
||||
<Step title="Open the editor">
|
||||
On the **Scan** page, click **New Scan Configuration**.
|
||||
</Step>
|
||||
<Step title="Name it">
|
||||
Give the configuration a descriptive **Name** (3–100 characters), e.g. `stricter-iam-aws`. Names must be unique within your tenant.
|
||||
</Step>
|
||||
<Step title="Write the YAML overrides">
|
||||
In the **Configuration (YAML)** field, add only the keys you want to override, grouped by provider. The editor is pre-filled with a representative default placeholder you can use as a starting point.
|
||||
</Step>
|
||||
<Step title="Attach providers (optional)">
|
||||
Under **Attach to providers**, pick the providers that should use this configuration. This is optional — you can save without any provider and attach them later.
|
||||
</Step>
|
||||
<Step title="Save">
|
||||
Click **Save**. The server validates the configuration values and, if everything is valid, stores it and attaches the selected providers.
|
||||
</Step>
|
||||
</Steps>
|
||||
- **Viewing** a Scan Configuration doesn't require any specific permission.
|
||||
- **Creating, editing, and deleting** a Scan Configuration requires the **Manage Scans** permission.
|
||||
- **Attaching or detaching providers** requires the **Manage Providers** permission as well, in addition to Manage Scans. This applies to explicit changes to the attached providers, and to deleting a Scan Configuration that still has providers attached (deleting it detaches them too).
|
||||
- Attaching or detaching a provider also requires that provider to be **visible to your role**. Visibility comes from the [Provider Groups](/user-guide/tutorials/prowler-app-rbac#provider-groups) assigned to your role, or from **Unlimited Visibility**. You can't attach a provider you can't see, and you can't detach one either, whether by removing it from the list or by deleting the Scan Configuration it's attached to.
|
||||
|
||||
### YAML Structure
|
||||
## Config Schema
|
||||
|
||||
The YAML follows the structure of `config.yaml`: a mapping keyed by provider, with each provider section holding the keys you want to override.
|
||||
The YAML follows the structure of `config.yaml`: a mapping keyed by provider, with each provider section holding the keys you want to change. You only list the keys you want to override; they're merged over that provider's `config.yaml` defaults.
|
||||
|
||||
```yaml
|
||||
aws:
|
||||
@@ -64,35 +51,36 @@ gcp:
|
||||
storage_min_retention_days: 30
|
||||
```
|
||||
|
||||
Scan Configuration works for **every provider Prowler scans** — you key your overrides by provider using the same section names as `config.yaml`. Each provider below ships a configuration schema, so its values are checked on save (ranges, enums, and types):
|
||||
## Creating a Scan Configuration
|
||||
|
||||
| Provider | Section key |
|
||||
| --- | --- |
|
||||
| AWS | `aws` |
|
||||
| Azure | `azure` |
|
||||
| Google Cloud | `gcp` |
|
||||
| Kubernetes | `kubernetes` |
|
||||
| Microsoft 365 | `m365` |
|
||||
| GitHub | `github` |
|
||||
| MongoDB Atlas | `mongodbatlas` |
|
||||
| Cloudflare | `cloudflare` |
|
||||
| Vercel | `vercel` |
|
||||
| Okta | `okta` |
|
||||
| Alibaba Cloud | `alibabacloud` |
|
||||
| OpenStack | `openstack` |
|
||||
|
||||
Sections that aren't listed here — those contributed by third-party check plugins, or providers that don't yet ship tunable defaults — are **accepted as-is** and applied without server-side value validation.
|
||||
<Steps>
|
||||
<Step title="Open the editor">
|
||||
On the **Scan** page (under **Configuration**), click **New Scan Configuration**.
|
||||
</Step>
|
||||
<Step title="Name it">
|
||||
Give the configuration a descriptive **Name** (3–100 characters), e.g. `stricter-iam-aws`.
|
||||
</Step>
|
||||
<Step title="Write the configuration file">
|
||||
In the **Configuration (YAML)** field, add the keys you want to change, grouped by provider. Only the keys you set are overridden; every other key keeps its `config.yaml` default. The editor is pre-filled with a representative default placeholder you can use as a starting point.
|
||||
</Step>
|
||||
<Step title="Attach providers (optional)">
|
||||
Under **Attach to providers**, pick the providers that should use this configuration. This is optional, you can save without any provider and attach them later, either by editing this configuration or from the **Providers** page (see [Attaching Providers](#attaching-providers)).
|
||||
</Step>
|
||||
<Step title="Save">
|
||||
Click **Save**. The server validates the configuration values and, if everything is valid, stores it and attaches the selected providers.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
<Tip>
|
||||
You don't need to fill in every provider — include only the sections and keys you actually want to change. The placeholder shown in the editor is just an example; if you leave the field with only the placeholder (greyed-out) text, nothing is saved.
|
||||
You don't need to fill in every provider. A section you don't include leaves that provider's `config.yaml` untouched. Within a section you do include, only add the keys you want to change; the rest keep their `config.yaml` defaults. The placeholder shown in the editor is just an example; if you leave the field with only the placeholder (greyed-out) text, nothing is saved.
|
||||
</Tip>
|
||||
|
||||
## How Validation Works
|
||||
|
||||
Validation happens in two layers, mirroring the Advanced Mutelist editor:
|
||||
Prowler checks your configuration in two stages, the same way the Advanced Mutelist editor does:
|
||||
|
||||
1. **Client-side (live): YAML syntax only.** As you type, the editor checks that the text parses to a valid YAML mapping. If it doesn't, you'll see an `Invalid YAML format` message and the **Save** button is disabled. When the syntax is valid, it shows **Valid YAML format**.
|
||||
2. **Server-side (on save): configuration values.** When you click Save (or Update), the API validates the actual values — ranges, enums, and types — against Prowler's schema. Any problems are returned and shown **inline beneath the field**, for both create and edit.
|
||||
1. **As you type: is it well-formed?** The editor checks that what you've written is valid YAML. If something is off, you'll see an `Invalid YAML format` message and **Save** stays disabled until you fix it. Once it's clean, it shows **Valid YAML format**.
|
||||
2. **When you save: are the values allowed?** Saving checks that each value is one Prowler accepts, the right type, within range, and one of the allowed options where a key only takes a fixed set of choices. If a value isn't allowed, the editor points to the exact key and explains why, right beneath the field, so you can correct it and save again.
|
||||
|
||||
For example, `azure.defender_attack_path_minimal_risk_level` only accepts `Low`, `Medium`, `High`, or `Critical`. Saving any other value returns an inline error like:
|
||||
|
||||
@@ -101,9 +89,9 @@ azure.defender_attack_path_minimal_risk_level: Input should be 'Low', 'Medium',
|
||||
```
|
||||
|
||||
<Warning>
|
||||
"Valid YAML format" confirms only that the document is **syntactically** correct — it does **not** mean the values are valid. Value validation (ranges and enums) is performed by the server when you save.
|
||||
**Valid YAML format** only means the text is well-formed; it doesn't mean your values are accepted. Those are checked when you save.
|
||||
|
||||
Be careful with indentation. A line like `azure: defender_attack_path_minimal_risk_level: Critical` (no newline/indent after `azure:`) is *valid YAML*, but it parses to a single top-level key named `azure:defender_attack_path_minimal_risk_level` instead of the nested `azure` section — so the value is never applied. Always nest provider keys:
|
||||
Be careful with indentation. A line like `azure: defender_attack_path_minimal_risk_level: Critical` (no line break and indent after `azure:`) is still valid YAML, but Prowler reads it as one long key name instead of a setting inside the `azure` section, so the value is silently ignored. Always nest provider keys:
|
||||
|
||||
```yaml
|
||||
azure:
|
||||
@@ -112,7 +100,7 @@ azure:
|
||||
</Warning>
|
||||
|
||||
<Info>
|
||||
Unknown top-level sections and unknown keys inside a known provider section are **tolerated** (accepted without error) for backward compatibility with third-party check plugins. This means typos in section or key names won't be rejected on save — double-check your structure against `config.yaml`.
|
||||
Prowler won't flag a section or key it doesn't recognize. It accepts them without error, so custom checks and plugins can add their own settings. The trade-off: a typo in a key or section name isn't rejected either, and that misspelled setting simply won't apply. Double-check your spelling against `config.yaml`.
|
||||
</Info>
|
||||
|
||||
## Attaching Providers
|
||||
@@ -131,11 +119,10 @@ You can also manage a provider's configuration from **Providers**:
|
||||
<Step title="Open the provider menu">
|
||||
On the **Providers** page, open the **⋮** menu on a provider row.
|
||||
</Step>
|
||||
<Step title="Choose the scan-config action">
|
||||
Click **Edit Scan Configuration**.
|
||||
<Step title="Click **Edit Scan Configuration**.">
|
||||
</Step>
|
||||
<Step title="Pick a configuration">
|
||||
In the dialog, choose an existing configuration from the dropdown to associate it, pick a different one to move the provider, or select **Default** to detach it. **Default** means the provider uses Prowler's built-in scan defaults from the SDK (no custom configuration), and it's always available — even if no custom configurations exist yet. Then click **Save**.
|
||||
In the dialog, pick a configuration from the dropdown to apply it (picking a different one moves the provider), or pick **Default** to detach it and go back to the built-in `config.yaml` defaults. Then click **Save**.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
@@ -144,7 +131,7 @@ This dialog only **associates or disassociates** an existing configuration. To c
|
||||
</Note>
|
||||
|
||||
<Info>
|
||||
Because a provider can belong to only one configuration, associating a provider that is already attached elsewhere **moves** it to the new configuration automatically — it is removed from the previous one.
|
||||
Because a provider can belong to only one configuration, associating a provider that is already attached elsewhere **moves** it to the new configuration automatically; it is removed from the previous one.
|
||||
</Info>
|
||||
|
||||
## Editing and Deleting
|
||||
@@ -152,20 +139,43 @@ Because a provider can belong to only one configuration, associating a provider
|
||||
On the **Scan Config** page, open the **⋮** menu on a configuration row:
|
||||
|
||||
- **Edit:** Choose **Edit** to open the editor, change its name, YAML, or attached providers, and click **Update**. Editing the YAML always happens here, never from the provider row.
|
||||
- **Delete:** Choose **Delete** (in the danger zone) and confirm. Providers that were attached fall back to Prowler's built-in scan defaults on their next scan.
|
||||
- **Delete:** Choose **Delete** (in the danger zone) and confirm. Providers that were attached fall back to the built-in defaults from `config.yaml` on their next scan.
|
||||
|
||||
## How It's Applied
|
||||
|
||||
When a scan runs for a provider:
|
||||
|
||||
1. If the provider is attached to a Scan Configuration, Prowler applies that configuration's overrides on top of the built-in defaults.
|
||||
2. If it isn't attached to any, the built-in defaults from `config.yaml` are used.
|
||||
1. If the provider is attached to a Scan Configuration **and that configuration has a section for the provider's type** (e.g. `aws` for an AWS provider), Prowler merges that section over `config.yaml` for that provider's scan: the keys you set win, and every key you didn't set keeps its `config.yaml` default.
|
||||
2. If the provider is attached to a Scan Configuration, but that configuration **has no section for the provider's type** (for example, a configuration that only defines an `aws` section, attached to a GCP provider), the scan uses the built-in defaults from `config.yaml` for that provider, exactly as if no Scan Configuration were attached at all.
|
||||
3. If the provider isn't attached to any Scan Configuration, the built-in defaults from `config.yaml` are used.
|
||||
|
||||
Overrides are merged key by key: any value you don't set keeps its default.
|
||||
<Note>
|
||||
The merge is per key. A key you don't set keeps its `config.yaml` default, because each check falls back to the default configuration when a value isn't provided. You only need to list the keys you want to change.
|
||||
</Note>
|
||||
|
||||
<Tip>
|
||||
A single Scan Configuration can hold sections for several provider types at once (see [Config Schema](#config-schema)) and be attached to providers of different types. Each provider only ever picks up the section matching its own type; the rest of the YAML is ignored for that provider.
|
||||
</Tip>
|
||||
|
||||
## Effect on Compliance Results
|
||||
|
||||
Some compliance requirements only hold if the checks they map to ran with a strict-enough configuration. For example, a requirement expecting unused access keys to be disabled within 45 days loses its meaning if a Scan Configuration raises `max_unused_access_keys_days` to 120: the check would still PASS, but the requirement wouldn't really be met.
|
||||
|
||||
When a scan's applied configuration doesn't meet a requirement's expectations, Prowler marks that requirement as **FAIL** on the Compliance page, even if every individual finding passed. The requirement shows an info icon (and, when expanded, an inline alert) with:
|
||||
|
||||
> Marked as FAIL because the applied scan configuration does not meet this requirement, even though all findings passed.
|
||||
|
||||
This only affects requirements built around a configurable check that declares this kind of expectation; requirements without one are never affected by an attached Scan Configuration.
|
||||
|
||||
## Common Examples
|
||||
|
||||
**Stricter IAM hygiene for AWS:**
|
||||
Each example below shows only the keys being changed for that provider.
|
||||
|
||||
<Note>
|
||||
Only the keys shown are overridden. Every other key for that provider keeps its `config.yaml` default (see [How It's Applied](#how-its-applied)).
|
||||
</Note>
|
||||
|
||||
**Stricter IAM (Identity and Access Management) hygiene for AWS:**
|
||||
|
||||
```yaml
|
||||
aws:
|
||||
@@ -191,18 +201,10 @@ gcp:
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
<Note>
|
||||
**Save is disabled.** The YAML has a syntax error (or the field is empty). Fix the `Invalid YAML format` message shown beneath the editor.
|
||||
</Note>
|
||||
### A Provider Doesn't Appear in the Selector
|
||||
|
||||
<Note>
|
||||
**An inline error appears after saving.** The server rejected a value (out of range or not an allowed enum). The message names the exact path, e.g. `aws.max_unused_access_keys_days: ...`. Correct the value and save again.
|
||||
</Note>
|
||||
The provider is already attached to another Scan Configuration. Detach it there first, or use the provider row menu to move it.
|
||||
|
||||
<Note>
|
||||
**A provider doesn't appear in the selector.** It's already attached to another Scan Configuration. Detach it there first, or use the provider row menu to move it.
|
||||
</Note>
|
||||
### You Can't Attach or Detach Providers
|
||||
|
||||
<Note>
|
||||
**My override doesn't seem to apply.** Check indentation (provider keys must be nested under their section) and key spelling — unknown keys are silently accepted. Compare against [`config.yaml`](https://github.com/prowler-cloud/prowler/blob/master/prowler/config/config.yaml).
|
||||
</Note>
|
||||
If you can edit the configuration but not change its providers, or an error mentions a provider ID that "wasn't found", you're missing the **Manage Providers** permission or the provider isn't visible to your role. A provider outside your visibility is reported the same way as one that doesn't exist, so it isn't revealed to roles that shouldn't see it. See [Required Permissions](#required-permissions).
|
||||
|
||||
@@ -98,6 +98,12 @@ Choose a Method:
|
||||
|
||||
</Info>
|
||||
<Warning>
|
||||
**Single-Value `userType` Required**
|
||||
|
||||
Map `userType` to an IdP attribute that always contains a single value. If the IdP sends multiple values, Prowler App uses only the first value and does not assign multiple roles or select the highest-privilege role.
|
||||
|
||||
</Warning>
|
||||
<Warning>
|
||||
**Dynamic Updates**
|
||||
|
||||
Prowler App updates these attributes each time a user logs in. Any changes made in the Identity Provider (IdP) will be reflected when the user logs in again.
|
||||
@@ -154,6 +160,7 @@ Choose a Method:
|
||||
* If a role with the specified name already exists in Prowler App, the user automatically receives that role.
|
||||
* If the role does not exist, Prowler App creates a new role with that exact name with read-only access: the user can see all providers and their findings but cannot manage anything. A Prowler administrator (a user whose role includes the "Manage Account" permission) can adjust its permissions afterward through the [RBAC Management tab](/user-guide/tutorials/prowler-app-rbac).
|
||||
* If `userType` is not defined in the user's Okta profile, the user's existing roles in Prowler App are left unchanged.
|
||||
* `userType` must contain a single value. If the IdP sends multiple values, Prowler App uses only the first value and does not assign multiple roles.
|
||||
|
||||
**Example:** To assign the `IT` role to a user, set the `userType` value to `IT` in Okta. If a role named `IT` already exists in Prowler App, the user receives it automatically upon login. If it does not exist, Prowler App creates a new role called `IT` with read-only access, and a Prowler administrator can adjust its permissions as needed.
|
||||
|
||||
|
||||
@@ -18,17 +18,14 @@ After [installing](/getting-started/installation/prowler-app) **Prowler App**, a
|
||||
To view the auto-generated **Prowler API** documentation, navigate to [http://localhost:8080/api/v1/docs](http://localhost:8080/api/v1/docs). This documentation provides details on available endpoints, parameters, and responses.
|
||||
</Note>
|
||||
|
||||
## **Step 1: Sign Up**
|
||||
|
||||
### **Sign Up with Email**
|
||||
|
||||
## Step 1: Sign Up
|
||||
### Sign Up with Email
|
||||
To get started, sign up using your email and password:
|
||||
|
||||
<img src="/images/sign-up-button.png" alt="Sign Up Button" width="320" />
|
||||
<img src="/images/sign-up.png" alt="Sign Up" width="285" />
|
||||
|
||||
### **Sign Up with Social Login**
|
||||
|
||||
### Sign Up with Social Login
|
||||
If Social Login is enabled, you can sign up using your preferred provider (e.g., Google, GitHub).
|
||||
|
||||
<Note>
|
||||
@@ -44,16 +41,14 @@ If your email is not registered, a new account will be created using your social
|
||||
See [how to configure Social Login for Prowler](/user-guide/tutorials/prowler-app-social-login) to enable this feature in your own deployments.
|
||||
|
||||
</Note>
|
||||
## **Step 2: Log In**
|
||||
|
||||
## Step 2: Log In
|
||||
Once registered, log in with your email and password to access Prowler App.
|
||||
|
||||
<img src="/images/log-in.png" alt="Log In" width="350" />
|
||||
|
||||
Upon logging in, the Overview page will display. At this stage, no data is present: add a provider to begin scanning your cloud environment.
|
||||
|
||||
## **Step 3: Add a Provider**
|
||||
|
||||
## Step 3: Add a Provider
|
||||
To perform security scans, link a cloud provider account. Prowler supports the following providers and more:
|
||||
|
||||
- **AWS**
|
||||
@@ -77,8 +72,7 @@ Steps to add a provider:
|
||||
|
||||
<img src="/images/add-provider.png" alt="Add Provider" width="700" />
|
||||
|
||||
## **Step 4: Configure the Provider**
|
||||
|
||||
## Step 4: Configure the Provider
|
||||
Select the cloud provider to scan and configure authentication credentials. Each provider has specific requirements and authentication methods.
|
||||
|
||||
<img src="/images/select-provider.png" alt="Select a Provider" width="700" />
|
||||
@@ -111,14 +105,12 @@ For detailed instructions on configuring credentials for each provider, refer to
|
||||
Scan IaC public or private repositories for security issues.
|
||||
</Card>
|
||||
</Columns>
|
||||
## **Step 5: Test Connection**
|
||||
|
||||
## Step 5: Test Connection
|
||||
After adding your credentials of your cloud account, click the `Launch` button to verify that Prowler App can successfully connect to your provider:
|
||||
|
||||
<img src="/images/test-connection-button.png" alt="Test Connection" width="700" />
|
||||
|
||||
## **Step 6: Scan started**
|
||||
|
||||
## Step 6: Scan Started
|
||||
After successfully adding and testing your credentials, Prowler will start scanning your cloud environment, click the `Go to Scans` button to see the progress:
|
||||
|
||||
<img src="/images/provider-added.png" alt="Start Now" width="700" />
|
||||
@@ -127,8 +119,7 @@ After successfully adding and testing your credentials, Prowler will start scann
|
||||
Prowler will automatically scan all configured providers every **24 hours**, ensuring your cloud environment stays continuously monitored.
|
||||
|
||||
</Note>
|
||||
## **Step 7: Monitor Scan Progress**
|
||||
|
||||
## Step 7: Monitor Scan Progress
|
||||
Track the progress of your scan in the `Scans` section:
|
||||
|
||||
<img src="/images/scan-progress.png" alt="Scan Progress" width="700" />
|
||||
@@ -146,8 +137,7 @@ Each dashboard handles scan data differently:
|
||||
When a new scan completes or a new data ingestion is processed, the dashboards automatically reflect the updated results.
|
||||
</Note>
|
||||
|
||||
## **Step 8: Analyze the Findings**
|
||||
|
||||
## Step 8: Analyze the Findings
|
||||
While the scan is running, start exploring the findings in these sections:
|
||||
|
||||
- **Overview**: High-level summary of the scans.
|
||||
@@ -168,8 +158,7 @@ While the scan is running, start exploring the findings in these sections:
|
||||
|
||||
To view all `new` findings that have not been seen prior to this scan, click the `Delta` filter and select `new`. To view all `changed` findings that have had a status change (from `PASS` to `FAIL` for example), click the `Delta` filter and select `changed`.
|
||||
|
||||
## **Step 9: Download the Outputs**
|
||||
|
||||
## Step 9: Download the Outputs
|
||||
Once a scan is complete, navigate to the Scan Jobs section to download the output files generated by Prowler:
|
||||
|
||||
<img src="/images/scan_jobs_section.png" alt="Scan Jobs section" width="700" />
|
||||
@@ -190,8 +179,7 @@ The `zip` file unpacks into a folder named like `prowler-output-<provider_id>-<t
|
||||
For more information about the API endpoint used by the UI to download the ZIP archive, refer to: [Prowler API Reference - Download Scan Output](https://api.prowler.com/api/v1/docs#tag/Scan/operation/scans_report_retrieve)
|
||||
|
||||
</Note>
|
||||
## **Step 10: Download specified compliance report**
|
||||
|
||||
## Step 10: Download Specified Compliance Report
|
||||
Once your scan has finished, you don’t need to grab the entire ZIP—just pull down the specific compliance report you want:
|
||||
|
||||
- Navigate to the **Compliance** section of the UI.
|
||||
|
||||
@@ -4,14 +4,15 @@ description: 'Onboard all AWS accounts in your Organization through a single gui
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.19.0" />
|
||||
|
||||
Prowler Cloud enables you to onboard all AWS accounts in your Organization through a single guided wizard. Instead of connecting accounts one by one, you can discover every account in your AWS Organization, select the ones you want to monitor, test connectivity, and launch scans — all from the Prowler Cloud UI.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [subscription](https://prowler.com/pricing). For CLI-based multi-account scanning, see [AWS Organizations in Prowler CLI](/user-guide/providers/aws/organizations).
|
||||
</Note>
|
||||
<SubscriptionBanner>
|
||||
For CLI-based multi-account scanning, see [AWS Organizations in Prowler CLI](/user-guide/providers/aws/organizations).
|
||||
</SubscriptionBanner>
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
@@ -4,16 +4,15 @@ description: 'Upload OCSF scan results to Prowler Cloud from external sources or
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.19.0" />
|
||||
|
||||
Findings Ingestion enables uploading OCSF (Open Cybersecurity Schema Framework) scan results to Prowler Cloud. This feature supports importing findings from Prowler CLI output files that use the [Detection Finding](https://schema.ocsf.io/classes/detection_finding) class.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [subscription](https://prowler.com/pricing).
|
||||
</Note>
|
||||
<SubscriptionBanner />
|
||||
|
||||
## OCSF Detection Finding format
|
||||
## OCSF Detection Finding Format
|
||||
|
||||
The ingestion API accepts `.ocsf.json` files containing a JSON array of OCSF Detection Finding records. Each finding represents a security check result from Prowler.
|
||||
|
||||
@@ -130,7 +129,7 @@ The ingestion API accepts `.ocsf.json` files containing a JSON array of OCSF Det
|
||||
Only **Detection Finding** (`class_uid: 2004`) records are accepted. Other OCSF classes are not supported for ingestion.
|
||||
</Note>
|
||||
|
||||
## Required permissions
|
||||
## Required Permissions
|
||||
|
||||
The **Manage Ingestions** RBAC permission controls access to the ingestion endpoints. Without this permission, findings cannot be submitted via the API or `--push-to-cloud`.
|
||||
|
||||
@@ -145,7 +144,7 @@ The `--push-to-cloud` flag uploads scan results directly to Prowler Cloud after
|
||||
- A valid Prowler Cloud API key (see [API Keys](/user-guide/tutorials/prowler-app-api-keys))
|
||||
- The `PROWLER_CLOUD_API_KEY` environment variable configured
|
||||
|
||||
### Basic usage
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
export PROWLER_CLOUD_API_KEY="pk_your_api_key_here"
|
||||
@@ -153,7 +152,7 @@ export PROWLER_CLOUD_API_KEY="pk_your_api_key_here"
|
||||
prowler aws --push-to-cloud
|
||||
```
|
||||
|
||||
### Combining with output formats
|
||||
### Combining with Output Formats
|
||||
|
||||
When using `--push-to-cloud` with custom output formats that exclude OCSF, Prowler generates a temporary OCSF file for upload:
|
||||
|
||||
@@ -169,7 +168,7 @@ When default output formats include OCSF, Prowler reuses the existing file. Defa
|
||||
prowler aws --services accessanalyzer --push-to-cloud -o /tmp/scan-output
|
||||
```
|
||||
|
||||
### CLI output examples
|
||||
### CLI Output Examples
|
||||
|
||||
**Successful upload:**
|
||||
```
|
||||
@@ -228,7 +227,7 @@ curl -X POST \
|
||||
https://api.prowler.com/api/v1/ingestions
|
||||
```
|
||||
|
||||
### Submit an ingestion batch
|
||||
### Submit an Ingestion Batch
|
||||
|
||||
Upload a `.ocsf.json` file containing a JSON array of OCSF Detection Finding records. See [OCSF Detection Finding format](#ocsf-detection-finding-format) for the expected structure.
|
||||
|
||||
@@ -266,7 +265,7 @@ curl -X POST \
|
||||
}
|
||||
```
|
||||
|
||||
### Get ingestion status
|
||||
### Get Ingestion Status
|
||||
|
||||
Monitor the progress of an ingestion job.
|
||||
|
||||
@@ -304,7 +303,7 @@ curl -X GET \
|
||||
}
|
||||
```
|
||||
|
||||
### List ingestion jobs
|
||||
### List Ingestion Jobs
|
||||
|
||||
Retrieve a list of ingestion jobs for the tenant.
|
||||
|
||||
@@ -332,7 +331,7 @@ curl -X GET \
|
||||
"https://api.prowler.com/api/v1/ingestions?filter[status]=completed&page[size]=10"
|
||||
```
|
||||
|
||||
### Get ingestion errors
|
||||
### Get Ingestion Errors
|
||||
|
||||
Retrieve error details for a specific ingestion job.
|
||||
|
||||
@@ -346,7 +345,7 @@ curl -X GET \
|
||||
https://api.prowler.com/api/v1/ingestions/3650fef9-8e5f-4808-a95f-74f0afae8499/errors
|
||||
```
|
||||
|
||||
## Ingestion status values
|
||||
## Ingestion Status Values
|
||||
|
||||
| Status | Description |
|
||||
|--------|-------------|
|
||||
@@ -355,7 +354,7 @@ curl -X GET \
|
||||
| `completed` | All records processed successfully |
|
||||
| `failed` | Job encountered errors during processing |
|
||||
|
||||
## CI/CD integration
|
||||
## CI/CD Integration
|
||||
|
||||
Automate findings ingestion in CI/CD pipelines by setting the API key as a secret.
|
||||
|
||||
@@ -391,7 +390,7 @@ prowler_scan:
|
||||
PROWLER_CLOUD_API_KEY: $PROWLER_CLOUD_API_KEY
|
||||
```
|
||||
|
||||
## Billing impact
|
||||
## Billing Impact
|
||||
|
||||
Each unique cloud account discovered in ingested OCSF findings counts as one **provider** in the Prowler Cloud subscription.
|
||||
|
||||
|
||||
@@ -4,14 +4,13 @@ description: 'Create, edit, and monitor recurring scans in Prowler Cloud and Ent
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.31.0" />
|
||||
|
||||
Scan Scheduling lets Prowler run recurring scans for connected providers. Use it to keep findings, compliance results, and resource inventory up to date without launching every scan manually.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [subscription](https://prowler.com/pricing).
|
||||
</Note>
|
||||
<SubscriptionBanner />
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
# =============================================================================
|
||||
# Final stage - Minimal runtime environment
|
||||
# =============================================================================
|
||||
FROM python:3.13.14-alpine3.23@sha256:b0513989fa9be54569cac73f48a60320b74bb0f9ffa886568eea7e48a2432c04
|
||||
FROM python:3.13.14-alpine3.23@sha256:9fdbf2e3e82628351513560b121e2ee6ce31cac212be9e070c5a5e2769fb5e76
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud"
|
||||
|
||||
|
||||
@@ -39,6 +39,8 @@
|
||||
"rolesanywhere:ListTagsForResource",
|
||||
"rolesanywhere:ListTrustAnchors",
|
||||
"s3:GetAccountPublicAccessBlock",
|
||||
"s3:GetObjectAcl",
|
||||
"s3:ListBucket",
|
||||
"shield:DescribeProtection",
|
||||
"shield:GetSubscriptionState",
|
||||
"securityhub:BatchImportFindings",
|
||||
|
||||
+19
-1
@@ -6,23 +6,42 @@ All notable changes to the **Prowler SDK** are documented in this file.
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `exchange_application_access_policy_restricts_mailbox_apps` check for M365 provider, verifying every service principal with Microsoft Graph application-level Exchange mailbox permissions is restricted by an Exchange Online Application Access Policy, preventing tenant-wide mailbox access by unscoped applications [(#11247)](https://github.com/prowler-cloud/prowler/pull/11247)
|
||||
- Per-requirement configuration validation for compliance frameworks via `ConfigRequirements`, so a requirement is reported as FAIL when its configurable checks ran with a configuration too loose to satisfy it (applied across all compliance outputs: CSV, OCSF, and console tables) [(#11669)](https://github.com/prowler-cloud/prowler/pull/11669)
|
||||
- `entra_conditional_access_policy_explicitly_targets_azure_devops` check for M365 provider, verifying at least one enabled Conditional Access policy explicitly includes the Azure DevOps cloud application instead of relying on a broad "All cloud apps" policy [(#11182)](https://github.com/prowler-cloud/prowler/pull/11182)
|
||||
- `entra_conditional_access_policy_no_exclusion_gaps` check for M365 provider, verifying every user, group, role, or application excluded from an enabled Conditional Access policy stays in scope of another enabled policy [(#11577)](https://github.com/prowler-cloud/prowler/pull/11577)
|
||||
- `entra_conditional_access_policy_groups_management_restricted` check for M365 provider, verifying every security group referenced by an enabled or report-only Conditional Access policy is management-restricted or role-assignable [(#11342)](https://github.com/prowler-cloud/prowler/pull/11342)
|
||||
- `stepfunctions_statemachine_encrypted_with_cmk` check for AWS provider, verifying that each Step Functions state machine uses a customer-managed KMS key for encryption at rest rather than the default AWS-owned key [(#11538)](https://github.com/prowler-cloud/prowler/pull/11538)
|
||||
- CIS Controls v8.1 universal compliance framework mapping existing checks across 18 providers (AWS, Azure, GCP, Kubernetes, M365, GitHub, AlibabaCloud, OracleCloud, GoogleWorkspace, Okta, Cloudflare, Vercel, MongoDB Atlas, OpenStack, Linode, StackIT, NHN, and Scaleway) to the 18 CIS Critical Security Controls and their Safeguards [(#11700)](https://github.com/prowler-cloud/prowler/pull/11700)
|
||||
- CIS Microsoft 365 Foundations Benchmark v7.0.0 compliance framework for the M365 provider [(#11699)](https://github.com/prowler-cloud/prowler/pull/11699)
|
||||
- `waf_regional_webacl_logging_enabled` check for AWS provider, verifying that each AWS WAF Classic Regional Web ACL has logging enabled to a Kinesis Data Firehose stream [(#11539)](https://github.com/prowler-cloud/prowler/pull/11539)
|
||||
- `sdk_only` provider property (default `true`) and `Provider.get_app_providers()`, so a provider (built-in or external) stays CLI/SDK-only and hidden from the app unless it declares `sdk_only = False` [(#11427)](https://github.com/prowler-cloud/prowler/pull/11427)
|
||||
- `Provider.get_scan_arguments()`, `Provider.get_connection_arguments()` and `Provider.get_credentials_schema()` contract methods, so a provider persisted as a stored uid plus a secret dict can be constructed and validated programmatically (to be consumed by the API in a later change) [(#11578)](https://github.com/prowler-cloud/prowler/pull/11578)
|
||||
- Okta API request throttling to proactively stay under rate limits, configurable via `okta_requests_per_second` in the config file and the `--okta-requests-per-second` CLI flag, plus configurable retries via `okta_max_retries` / `--okta-retries-max-attempts` as a safety net [(#11702)](https://github.com/prowler-cloud/prowler/pull/11702)
|
||||
- CIS Amazon Web Services Foundations Benchmark v7.0.0 compliance framework for the AWS provider, adding the new Organizations section (2.1.1-2.1.6), resource policy (2.21), web front-end access logging (4.10), and VPC Endpoints (6.8) recommendations [(#11707)](https://github.com/prowler-cloud/prowler/pull/11707)
|
||||
- CIS Microsoft Azure Foundations Benchmark v6.0.0 compliance framework for the Azure provider [(#11708)](https://github.com/prowler-cloud/prowler/pull/11708)
|
||||
- CIS Google Cloud Platform Foundation Benchmark v5.0.0 compliance framework for the GCP provider [(#11714)](https://github.com/prowler-cloud/prowler/pull/11714)
|
||||
- CIS Kubernetes Benchmark v2.0.1 compliance framework for the Kubernetes provider [(#11722)](https://github.com/prowler-cloud/prowler/pull/11722)
|
||||
- CIS GitHub Benchmark v1.2.0 compliance framework for the GitHub provider [(#11719)](https://github.com/prowler-cloud/prowler/pull/11719)
|
||||
- AWS Bedrock AgentCore privilege escalation paths in the IAM privilege escalation checks, covering Runtime, Harness, Code Interpreter and Custom Browser [(#11726)](https://github.com/prowler-cloud/prowler/pull/11726)
|
||||
- `--scan-secrets-validate` flag and `aws.secrets_validate` configuration option to optionally validate the secrets discovered by the secret-scanning checks against the provider APIs; secrets confirmed to be live are reported as critical [(#11694)](https://github.com/prowler-cloud/prowler/pull/11694)
|
||||
- `apigateway_restapi_no_secrets_in_stage_variables` check for AWS provider, scanning API Gateway REST API stage variables for hardcoded secrets such as passwords, API keys, and tokens [(#11188)](https://github.com/prowler-cloud/prowler/pull/11188)
|
||||
- `s3_bucket_object_public` check for AWS provider, spot-checking a configurable sample of object ACLs in each bucket and flagging objects granted to the AllUsers or AuthenticatedUsers groups; disabled by default and opted into via the `s3_bucket_object_public_enabled` configuration option [(#9517)](https://github.com/prowler-cloud/prowler/pull/9517)
|
||||
- Azure provider now supports `--azure-resource-group` to scope resource-level checks to specific resource groups across all accessible subscriptions [(#10657)](https://github.com/prowler-cloud/prowler/pull/10657)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Replaced the `detect-secrets` library with [Kingfisher](https://github.com/mongodb/kingfisher) as the engine for the secret-scanning checks; scans run fully offline by default and obvious placeholder values are no longer reported as findings [(#11694)](https://github.com/prowler-cloud/prowler/pull/11694)
|
||||
- Removed the `detect_secrets_plugins` configuration option, which is no longer used by the new secret-scanning engine [(#11694)](https://github.com/prowler-cloud/prowler/pull/11694)
|
||||
- `awslambda_function_no_secrets_in_code` now supports a `secrets_ignore_files` audit-config option to skip files inside the deployment package by glob pattern (e.g. `*.deps.json`), suppressing .NET dependency-manifest false positives without masking real secrets [(#11222)](https://github.com/prowler-cloud/prowler/pull/11222)
|
||||
- AWS scans for EBS snapshots, Backup recovery points, CloudWatch log groups, Lambda functions, ECS task definitions, and CodeArtifact packages now support configurable resource analysis limits via `aws.max_scanned_resources_per_service`; limits are disabled by default and only positive values cap analyzed resources [(#11228)](https://github.com/prowler-cloud/prowler/pull/11228)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- GitHub `repository_has_codeowners_file` check no longer flags archived repositories, since they are read-only and cannot be updated without first being unarchived, making the finding not actionable [(#11735)](https://github.com/prowler-cloud/prowler/pull/11735)
|
||||
- Report secret-scanning checks as `MANUAL` instead of `PASS` when the scanner fails (non-zero exit, timeout, unparseable output or missing binary), so a scanner failure is no longer indistinguishable from "no secrets found" [(#11694)](https://github.com/prowler-cloud/prowler/pull/11694)
|
||||
- Avoid a false `FAIL` in `cloudwatch_log_group_no_secrets_in_logs` when a multiline event's secrets are all removed by `secrets_ignore_patterns` during the rescan [(#11694)](https://github.com/prowler-cloud/prowler/pull/11694)
|
||||
- Key the `cloudwatch_log_group_no_secrets_in_logs` secret scan by log group ARN instead of name, so same-named log groups and streams in different regions no longer collide and reuse each other's findings [(#11694)](https://github.com/prowler-cloud/prowler/pull/11694)
|
||||
- Compliance frameworks contributed by several external packages under the same provider are now merged instead of overwritten, so every entry-point directory a provider contributes is discovered [(#11578)](https://github.com/prowler-cloud/prowler/pull/11578)
|
||||
- Azure PostgreSQL flexible server collection no longer drops the remaining servers in a subscription when one server fails to collect; the `connection_throttle.enable` parameter (removed in PostgreSQL 16+) is treated as absent only when the Azure SDK reports it as not found, so unexpected lookup failures are not silently reported as throttling disabled [(#11595)](https://github.com/prowler-cloud/prowler/pull/11595)
|
||||
- Azure `keyvault_logging_enabled` now accepts Key Vault diagnostic settings that enable the explicit `AuditEvent` category, avoiding false failures when Azure returns category-based logs without category groups [(#11660)](https://github.com/prowler-cloud/prowler/pull/11660)
|
||||
@@ -304,7 +323,6 @@ All notable changes to the **Prowler SDK** are documented in this file.
|
||||
- `bedrock_prompt_management_exists` check for AWS provider [(#10878)](https://github.com/prowler-cloud/prowler/pull/10878)
|
||||
- 8 Gmail attachment safety and spoofing protection checks for Google Workspace provider using the Cloud Identity Policy API [(#10980)](https://github.com/prowler-cloud/prowler/pull/10980)
|
||||
- `bedrock_prompt_encrypted_with_cmk` check for AWS provider [(#10905)](https://github.com/prowler-cloud/prowler/pull/10905)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Azure Network Watcher flow log checks now require workspace-backed Traffic Analytics for `network_flow_log_captured_sent` and align metadata with VNet-compatible flow log guidance [(#10645)](https://github.com/prowler-cloud/prowler/pull/10645)
|
||||
|
||||
+80
-34
@@ -3,6 +3,32 @@ aws:
|
||||
# AWS Global Configuration
|
||||
# aws.mute_non_default_regions --> Set to True to muted failed findings in non-default regions for AccessAnalyzer, GuardDuty, SecurityHub, DRS and Config
|
||||
mute_non_default_regions: False
|
||||
|
||||
# AWS Resource Scan Limit Configuration
|
||||
# Limits the number of resources scanned per service for services that can
|
||||
# accumulate huge numbers of resources (EBS snapshots, backup recovery
|
||||
# points, CloudWatch log groups, Lambda functions, ECS task definitions,
|
||||
# CodeArtifact packages). Limits apply to resources analyzed, not findings:
|
||||
# a selected resource can produce zero, one, or many findings. Where the AWS
|
||||
# API supports server-side ordering the latest resources are scanned first;
|
||||
# otherwise it is best-effort API order.
|
||||
# Disabled by default: scan every resource unless a positive limit is configured.
|
||||
# Set to 0 (or a negative value) to disable the limit (scan every resource).
|
||||
# aws.max_scanned_resources_per_service --> global default for all services below
|
||||
max_scanned_resources_per_service: 0
|
||||
# Per-service overrides. Leave as null to fall back to the global default.
|
||||
# aws.max_ebs_snapshots --> ec2_ebs_* checks (EBS snapshots)
|
||||
max_ebs_snapshots: null
|
||||
# aws.max_backup_recovery_points --> backup_recovery_point_* checks
|
||||
max_backup_recovery_points: null
|
||||
# aws.max_cloudwatch_log_groups --> cloudwatch_log_group_* checks
|
||||
max_cloudwatch_log_groups: null
|
||||
# aws.max_lambda_functions --> awslambda_function_* checks
|
||||
max_lambda_functions: null
|
||||
# aws.max_ecs_task_definitions --> ecs_task_definitions_* checks
|
||||
max_ecs_task_definitions: null
|
||||
# aws.max_codeartifact_packages --> codeartifact_packages_* checks
|
||||
max_codeartifact_packages: null
|
||||
# aws.disallowed_regions --> List of AWS regions to exclude from the scan.
|
||||
# Also settable via the PROWLER_AWS_DISALLOWED_REGIONS environment variable or
|
||||
# the --excluded-region CLI flag. Precedence: CLI > env var > config file.
|
||||
@@ -423,6 +449,27 @@ aws:
|
||||
# Patterns to ignore in the secrets checks
|
||||
secrets_ignore_patterns: []
|
||||
|
||||
# aws.awslambda_function_no_secrets_in_code
|
||||
# Glob patterns of file names inside the Lambda deployment package to skip
|
||||
# when scanning for secrets. Useful to suppress known false positives such
|
||||
# as .NET dependency manifests.
|
||||
# Example:
|
||||
# secrets_ignore_files:
|
||||
# - "*.deps.json"
|
||||
# WARNING: use at your own risk. Any file whose name matches one of these
|
||||
# patterns is fully excluded from secret scanning, so a real secret placed
|
||||
# in such a file will NOT be detected. Keep patterns as narrow and specific
|
||||
# as possible; this is not recommended unless you have confirmed the matched
|
||||
# files only ever contain false positives.
|
||||
secrets_ignore_files: []
|
||||
|
||||
# Validate discovered secrets by checking whether they are live against the
|
||||
# provider APIs. WARNING: this makes outbound network calls that authenticate
|
||||
# with the discovered secret itself; the credential is exercised against the
|
||||
# provider and the call will appear in the audited account's logs (and may
|
||||
# trigger its monitoring). Disabled by default (scans stay fully offline).
|
||||
secrets_validate: False
|
||||
|
||||
# AWS Secrets Manager Configuration
|
||||
# aws.secretsmanager_secret_unused
|
||||
# Maximum number of days a secret can be unused
|
||||
@@ -436,36 +483,17 @@ aws:
|
||||
# Minimum retention period in hours for Kinesis streams
|
||||
min_kinesis_stream_retention_hours: 168 # 7 days
|
||||
|
||||
# Detect Secrets plugin configuration
|
||||
detect_secrets_plugins: [
|
||||
{"name": "ArtifactoryDetector"},
|
||||
{"name": "AWSKeyDetector"},
|
||||
{"name": "AzureStorageKeyDetector"},
|
||||
{"name": "BasicAuthDetector"},
|
||||
{"name": "CloudantDetector"},
|
||||
{"name": "DiscordBotTokenDetector"},
|
||||
{"name": "GitHubTokenDetector"},
|
||||
{"name": "GitLabTokenDetector"},
|
||||
{"name": "Base64HighEntropyString", "limit": 6.0},
|
||||
{"name": "HexHighEntropyString", "limit": 3.0},
|
||||
{"name": "IbmCloudIamDetector"},
|
||||
{"name": "IbmCosHmacDetector"},
|
||||
# {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885
|
||||
{"name": "JwtTokenDetector"},
|
||||
{"name": "KeywordDetector"},
|
||||
{"name": "MailchimpDetector"},
|
||||
{"name": "NpmDetector"},
|
||||
{"name": "OpenAIDetector"},
|
||||
{"name": "PrivateKeyDetector"},
|
||||
{"name": "PypiTokenDetector"},
|
||||
{"name": "SendGridDetector"},
|
||||
{"name": "SlackDetector"},
|
||||
{"name": "SoftlayerDetector"},
|
||||
{"name": "SquareOAuthDetector"},
|
||||
{"name": "StripeDetector"},
|
||||
# {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878
|
||||
{"name": "TwilioKeyDetector"},
|
||||
]
|
||||
# AWS S3 Configuration
|
||||
# aws.s3_bucket_object_public
|
||||
# This check performs a spot-check by sampling object ACLs within a bucket, so
|
||||
# it is disabled by default. For complete coverage, rely on s3_bucket_acl_prohibited
|
||||
# which enforces BucketOwnerEnforced Object Ownership (AWS's recommended approach).
|
||||
# Set s3_bucket_object_public_enabled to True to opt in.
|
||||
s3_bucket_object_public_enabled: False
|
||||
# Maximum number of objects to list per bucket (upper bound for the sampling pool)
|
||||
s3_bucket_object_public_max_objects: 100
|
||||
# Number of objects to randomly sample from the listed pool and inspect ACLs for
|
||||
s3_bucket_object_public_sample_size: 3
|
||||
|
||||
# AWS CodeBuild Configuration
|
||||
# aws.codebuild_project_uses_allowed_github_organizations
|
||||
@@ -715,6 +743,27 @@ okta:
|
||||
# Defaults to 15 per DISA STIG V-273187 (OKTA-APP-000025); raise it only
|
||||
# with an explicit risk acceptance.
|
||||
okta_admin_console_idle_timeout_max_minutes: 15
|
||||
# Okta API rate limiting
|
||||
# Max retries on HTTP 429. The Okta SDK sleeps until the X-Rate-Limit-Reset
|
||||
# window before each retry, so raising this lets scans ride out more rate-limit
|
||||
# windows on busy orgs instead of failing with partial data. SDK default is 2.
|
||||
okta_max_retries: 5
|
||||
# Per-request timeout in seconds. In the Okta SDK this value plays a DUAL role:
|
||||
# it is both the per-HTTP-call socket timeout AND the total wall-clock budget
|
||||
# across the whole retry+backoff loop. It defaults to 300 (not 0) because it is
|
||||
# the only effective hang guard, and 300s is the smallest value that still lets
|
||||
# all okta_max_retries rate-limit waits (~60s Okta reset windows) complete
|
||||
# without being cut short. Keep it roughly >= okta_max_retries * 60 if you
|
||||
# raise okta_max_retries.
|
||||
okta_request_timeout: 300
|
||||
# Maximum aggregate Okta API requests per second. Prowler paces all requests
|
||||
# through a shared limiter so scans stay under Okta's rate limits proactively,
|
||||
# rather than relying on the 429 retry above as a safety net. Okta enforces
|
||||
# limits per endpoint, so this is a deliberately simple global cap; lower it if
|
||||
# scans still hit limits, raise it to scan faster. Set to 0 to disable. Valid
|
||||
# range: 0 or 0.1..100 — non-zero rates below 0.1 are rejected because they
|
||||
# would make a scan impractically slow.
|
||||
okta_requests_per_second: 4
|
||||
# Okta Users
|
||||
# okta.user_inactivity_automation_35d_enabled
|
||||
# Maximum number of days a user can stay inactive before the
|
||||
@@ -749,9 +798,6 @@ openstack:
|
||||
# being flagged. Defaults to 5.
|
||||
image_sharing_threshold: 5
|
||||
# openstack.<compute|blockstorage|objectstorage>_*_metadata_sensitive_data
|
||||
# Regex patterns whose matches are excluded from detect-secrets scanning of
|
||||
# Regex patterns whose matches are excluded from secret scanning of
|
||||
# resource metadata.
|
||||
secrets_ignore_patterns: []
|
||||
# Custom detect-secrets plugin configuration for metadata scanning. Each
|
||||
# entry requires a `name`; entropy plugins also accept a `limit` (0..10).
|
||||
detect_secrets_plugins: []
|
||||
|
||||
@@ -14,7 +14,7 @@ thresholds) and avoids ints that obviously break downstream maths
|
||||
|
||||
from typing import Annotated, Literal, Optional
|
||||
|
||||
from pydantic import AfterValidator, Field
|
||||
from pydantic import AfterValidator, BeforeValidator, Field
|
||||
|
||||
from prowler.config.schema.base import ProviderConfigBase
|
||||
from prowler.config.schema.validators import (
|
||||
@@ -101,33 +101,65 @@ def _validate_account_ids(v: Optional[list[str]]) -> Optional[list[str]]:
|
||||
return v
|
||||
|
||||
|
||||
# ---- Nested models ----------------------------------------------------------
|
||||
def _reject_bool_resource_limit(v):
|
||||
if isinstance(v, bool):
|
||||
raise ValueError("resource scan limits must be integers, not booleans")
|
||||
return v
|
||||
|
||||
|
||||
class _DetectSecretsPlugin(ProviderConfigBase):
|
||||
"""One entry inside ``detect_secrets_plugins``.
|
||||
|
||||
Only ``name`` is required by the upstream library. ``limit`` is used by
|
||||
the entropy detectors. Any other plugin-specific kwarg is preserved by
|
||||
the ``extra="allow"`` policy inherited from ProviderConfigBase.
|
||||
"""
|
||||
|
||||
name: str
|
||||
limit: Optional[float] = Field(
|
||||
default=None,
|
||||
ge=0.0,
|
||||
le=10.0,
|
||||
description=(
|
||||
"Entropy threshold for detect-secrets entropy plugins. Range: 0..10 "
|
||||
"(Shannon entropy is bounded by log2(256)=8; >10 is meaningless)."
|
||||
),
|
||||
)
|
||||
ResourceScanLimit = Annotated[
|
||||
Optional[int], BeforeValidator(_reject_bool_resource_limit)
|
||||
]
|
||||
|
||||
|
||||
# ---- Main schema ------------------------------------------------------------
|
||||
|
||||
|
||||
class AWSProviderConfig(ProviderConfigBase):
|
||||
# --- Resource scan limits ---------------------------------------------
|
||||
max_scanned_resources_per_service: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Global resource scan limit for high-volume AWS services. Use 0 or -1 to disable.",
|
||||
)
|
||||
max_ebs_snapshots: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Resource scan limit for EBS snapshots. Use 0 or -1 to disable.",
|
||||
)
|
||||
max_backup_recovery_points: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Resource scan limit for AWS Backup recovery points. Use 0 or -1 to disable.",
|
||||
)
|
||||
max_cloudwatch_log_groups: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Resource scan limit for CloudWatch log groups. Use 0 or -1 to disable.",
|
||||
)
|
||||
max_lambda_functions: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Resource scan limit for Lambda functions. Use 0 or -1 to disable.",
|
||||
)
|
||||
max_ecs_task_definitions: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Resource scan limit for ECS task definitions. Use 0 or -1 to disable.",
|
||||
)
|
||||
max_codeartifact_packages: ResourceScanLimit = Field(
|
||||
default=None,
|
||||
ge=-1,
|
||||
le=1_000_000,
|
||||
description="Resource scan limit for CodeArtifact packages. Use 0 or -1 to disable.",
|
||||
)
|
||||
|
||||
# --- IAM ---------------------------------------------------------------
|
||||
mute_non_default_regions: Optional[bool] = None
|
||||
disallowed_regions: Optional[list[str]] = None
|
||||
@@ -394,6 +426,15 @@ class AWSProviderConfig(ProviderConfigBase):
|
||||
|
||||
# --- Secrets ---------------------------------------------------------
|
||||
secrets_ignore_patterns: Optional[list[str]] = None
|
||||
secrets_ignore_files: Optional[list[str]] = None
|
||||
secrets_validate: Optional[bool] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Validate discovered secrets against the provider APIs (live check). "
|
||||
"Makes outbound network calls that authenticate with the discovered "
|
||||
"secret. Disabled by default."
|
||||
),
|
||||
)
|
||||
max_days_secret_unused: Optional[int] = Field(
|
||||
default=None,
|
||||
ge=7,
|
||||
@@ -418,5 +459,30 @@ class AWSProviderConfig(ProviderConfigBase):
|
||||
description="Hours of Kinesis stream retention. Range: 24..8760 (1 day .. 1 year).",
|
||||
)
|
||||
|
||||
# --- detect-secrets plugin list -------------------------------------
|
||||
detect_secrets_plugins: Optional[list[_DetectSecretsPlugin]] = None
|
||||
# --- S3 --------------------------------------------------------------
|
||||
s3_bucket_object_public_enabled: Optional[bool] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Enable the s3_bucket_object_public spot-check, which samples object "
|
||||
"ACLs per bucket. Disabled by default because it lists and reads object "
|
||||
"ACLs, which is expensive on large buckets."
|
||||
),
|
||||
)
|
||||
s3_bucket_object_public_max_objects: Optional[int] = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
le=1000,
|
||||
description=(
|
||||
"Max objects to list per bucket as the sampling pool. Range: 1..1000 "
|
||||
"(ListObjectsV2 returns at most 1000 keys per page)."
|
||||
),
|
||||
)
|
||||
s3_bucket_object_public_sample_size: Optional[int] = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
le=1000,
|
||||
description=(
|
||||
"Number of objects sampled from the listed pool for ACL inspection. "
|
||||
"Range: 1..1000. Must be positive to avoid a no-op or invalid sample."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1,19 +1,45 @@
|
||||
"""Okta provider config schema with safety bounds."""
|
||||
|
||||
from typing import Optional
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic import AfterValidator, Field
|
||||
|
||||
from prowler.config.schema.base import ProviderConfigBase
|
||||
|
||||
# Lowest non-zero request rate we accept. Below this a scan is paced so slowly
|
||||
# it becomes impractical (e.g. 0.001 req/s is ~1000s per request, turning a
|
||||
# routine scan into days or years). 0 stays valid as the "disable throttling"
|
||||
# sentinel; anything between 0 and this floor is rejected so a typo can never
|
||||
# stall a scan.
|
||||
MIN_REQUESTS_PER_SECOND = 0.1
|
||||
|
||||
|
||||
def _validate_requests_per_second(value: Optional[float]) -> Optional[float]:
|
||||
"""Reject impractically slow non-zero request rates.
|
||||
|
||||
``0`` (and ``None``) pass through unchanged — ``0`` is the documented
|
||||
"disable throttling" sentinel. Any positive value below
|
||||
``MIN_REQUESTS_PER_SECOND`` is rejected; the ``ge``/``le`` bounds on the
|
||||
field already handle negatives and the upper cap.
|
||||
"""
|
||||
if value is None or value == 0:
|
||||
return value
|
||||
if value < MIN_REQUESTS_PER_SECOND:
|
||||
raise ValueError(
|
||||
f"must be 0 (disable throttling) or >= {MIN_REQUESTS_PER_SECOND}; "
|
||||
"smaller rates make scans impractically slow"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class OktaProviderConfig(ProviderConfigBase):
|
||||
"""Okta provider configuration schema.
|
||||
|
||||
Bounds the session, idle-timeout and inactivity thresholds consumed by
|
||||
the Okta checks. Every field is optional: when omitted (or dropped for
|
||||
being out of range) the check falls back to its own DISA STIG-derived
|
||||
default via ``audit_config.get(key, default)``.
|
||||
the Okta checks, plus the provider's API rate-limit handling (proactive
|
||||
request throttling and the SDK retry safety net). Every field is optional:
|
||||
when omitted (or dropped for being out of range) the check falls back to
|
||||
its own DISA STIG-derived default via ``audit_config.get(key, default)``.
|
||||
"""
|
||||
|
||||
okta_max_session_idle_minutes: Optional[int] = Field(
|
||||
@@ -63,3 +89,36 @@ class OktaProviderConfig(ProviderConfigBase):
|
||||
"the built-in `OU=DoD` / `OU=ECA` patterns."
|
||||
),
|
||||
)
|
||||
|
||||
# API rate limiting
|
||||
okta_requests_per_second: Annotated[
|
||||
Optional[float], AfterValidator(_validate_requests_per_second)
|
||||
] = Field(
|
||||
default=None,
|
||||
ge=0,
|
||||
le=100,
|
||||
description=(
|
||||
"Maximum aggregate Okta API requests per second. Range: 0 or "
|
||||
f"{MIN_REQUESTS_PER_SECOND}..100 (0 disables throttling). Non-zero "
|
||||
f"values below {MIN_REQUESTS_PER_SECOND} are rejected to avoid "
|
||||
"impractically slow scans."
|
||||
),
|
||||
)
|
||||
okta_max_retries: Optional[int] = Field(
|
||||
default=None,
|
||||
ge=0,
|
||||
le=10,
|
||||
description=(
|
||||
"Max retries on Okta API rate limiting (HTTP 429). Range: 0..10 "
|
||||
"(0 disables retries)."
|
||||
),
|
||||
)
|
||||
okta_request_timeout: Optional[int] = Field(
|
||||
default=None,
|
||||
ge=0,
|
||||
le=3600,
|
||||
description=(
|
||||
"Per-request timeout in seconds; also the total budget for the SDK "
|
||||
"retry loop. Range: 0..3600 (0 disables the timeout)."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -4,18 +4,16 @@ from typing import Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from prowler.config.schema.aws import _DetectSecretsPlugin
|
||||
from prowler.config.schema.base import ProviderConfigBase
|
||||
|
||||
|
||||
class OpenStackProviderConfig(ProviderConfigBase):
|
||||
"""OpenStack provider configuration schema.
|
||||
|
||||
Bounds the image-sharing threshold and reuses the AWS secret-scanning
|
||||
config models (``detect_secrets_plugins`` / ``secrets_ignore_patterns``)
|
||||
consumed by the metadata sensitive-data checks. Every field is optional:
|
||||
when omitted (or dropped for being out of range) the check falls back to
|
||||
its own default via ``audit_config.get(key, default)``.
|
||||
Bounds the image-sharing threshold and reuses the ``secrets_ignore_patterns``
|
||||
config consumed by the metadata sensitive-data checks. Every field is
|
||||
optional: when omitted (or dropped for being out of range) the check falls
|
||||
back to its own default via ``audit_config.get(key, default)``.
|
||||
"""
|
||||
|
||||
image_sharing_threshold: Optional[int] = Field(
|
||||
@@ -30,11 +28,7 @@ class OpenStackProviderConfig(ProviderConfigBase):
|
||||
secrets_ignore_patterns: Optional[list[str]] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Regex patterns whose matches are excluded from detect-secrets "
|
||||
"Regex patterns whose matches are excluded from secret "
|
||||
"scanning of resource metadata."
|
||||
),
|
||||
)
|
||||
detect_secrets_plugins: Optional[list[_DetectSecretsPlugin]] = Field(
|
||||
default=None,
|
||||
description="Custom detect-secrets plugin configuration for metadata scanning.",
|
||||
)
|
||||
|
||||
@@ -473,6 +473,18 @@ Detailed documentation at https://docs.prowler.com
|
||||
default=default_fixer_config_file_path,
|
||||
help="Set configuration fixer file path",
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--scan-secrets-validate",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Validate secrets discovered by the secrets checks by checking "
|
||||
"whether they are live against the provider APIs. WARNING: this "
|
||||
"makes outbound network calls using the discovered secret itself; "
|
||||
"the credential is exercised against the provider and the call "
|
||||
"appears in the audited account's logs. Disabled by default."
|
||||
),
|
||||
)
|
||||
|
||||
def __init_custom_checks_metadata_parser__(self):
|
||||
# CustomChecksMetadata
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Scoped resource scan limits for high-volume resources.
|
||||
|
||||
Some services accumulate huge numbers of resources (EBS snapshots, backup
|
||||
recovery points, log groups, Lambda functions, ECS task definitions,
|
||||
CodeArtifact packages). Scanning all of them causes API throttling, slow
|
||||
scans, cost and noisy findings.
|
||||
|
||||
``get_resource_scan_limit`` resolves the configured number of resources to
|
||||
analyze for a supported resource path. A limited resource can produce zero,
|
||||
one, or many findings; findings are not capped or re-ordered here.
|
||||
|
||||
Tradeoff: for newest-based resources, services may need to list lightweight or
|
||||
base metadata broadly to select the truly newest resources, then apply limits
|
||||
only to expensive hydration or analysis. The helper must not send
|
||||
user-configured limits as unsafe paginator ``PageSize`` values because AWS
|
||||
services validate page sizes differently.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from itertools import islice
|
||||
from typing import Any, Optional, Protocol, TypeVar
|
||||
|
||||
GLOBAL_LIMIT_KEY = "max_scanned_resources_per_service"
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class PaginatorProtocol(Protocol):
|
||||
"""Minimal boto3-compatible paginator interface used by this module."""
|
||||
|
||||
def paginate(self, **operation_parameters: Any) -> Iterable[Mapping[str, Any]]:
|
||||
"""Return paginator pages for the provided operation parameters."""
|
||||
|
||||
|
||||
def get_resource_scan_limit(audit_config: dict, service_key: str) -> Optional[int]:
|
||||
"""Resolve the resource scan limit for a service.
|
||||
|
||||
Precedence: per-service key (``service_key``) > global
|
||||
``max_scanned_resources_per_service`` > unlimited.
|
||||
|
||||
A non-positive resolved value means **unlimited** (``None``), preserving
|
||||
the legacy behavior as an explicit opt-out.
|
||||
|
||||
Args:
|
||||
audit_config: The provider ``audit_config`` dictionary.
|
||||
service_key: The per-service config key, e.g. ``max_lambda_functions``.
|
||||
|
||||
Returns:
|
||||
The limit as a positive ``int``, or ``None`` for unlimited.
|
||||
"""
|
||||
value = audit_config.get(service_key)
|
||||
if value is None:
|
||||
value = audit_config.get(GLOBAL_LIMIT_KEY)
|
||||
if value is None or value <= 0:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def limit_resources(resources: Iterable[T], limit: Optional[int]) -> Iterator[T]:
|
||||
"""Yield up to ``limit`` resources without changing resource order."""
|
||||
if not limit or limit <= 0:
|
||||
yield from resources
|
||||
return
|
||||
yield from islice(resources, limit)
|
||||
|
||||
|
||||
def iter_limited_paginator_items(
|
||||
paginator: PaginatorProtocol,
|
||||
result_key: str,
|
||||
limit: Optional[int],
|
||||
item_filter: Optional[Callable[[T], bool]] = None,
|
||||
**operation_parameters: Any,
|
||||
) -> Iterator[T]:
|
||||
"""Yield paginator result items, stopping after ``limit`` selected items.
|
||||
|
||||
The configured resource-analysis limit is intentionally not sent as
|
||||
``PageSize`` because AWS services validate page sizes differently. The
|
||||
paginator receives only the operation parameters needed by the AWS API,
|
||||
while this iterator applies the analysis limit defensively client-side.
|
||||
"""
|
||||
selected = 0
|
||||
for page in paginator.paginate(**operation_parameters):
|
||||
for item in page.get(result_key, []):
|
||||
if item_filter and not item_filter(item):
|
||||
continue
|
||||
yield item
|
||||
selected += 1
|
||||
if limit and selected >= limit:
|
||||
return
|
||||
+272
-97
@@ -9,52 +9,116 @@ except ImportError:
|
||||
pass
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from hashlib import sha512
|
||||
from functools import lru_cache
|
||||
from hashlib import sha1, sha512
|
||||
from io import TextIOWrapper
|
||||
from ipaddress import ip_address
|
||||
from os.path import exists
|
||||
from time import mktime
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Iterable, Mapping, Optional, Union
|
||||
|
||||
from colorama import Style
|
||||
from detect_secrets import SecretsCollection
|
||||
from detect_secrets.settings import transient_settings
|
||||
|
||||
from prowler.config.config import encoding_format_utf_8
|
||||
from prowler.lib.logger import logger
|
||||
|
||||
default_detect_secrets_plugins = [
|
||||
{"name": "ArtifactoryDetector"},
|
||||
{"name": "AWSKeyDetector"},
|
||||
{"name": "AzureStorageKeyDetector"},
|
||||
{"name": "BasicAuthDetector"},
|
||||
{"name": "CloudantDetector"},
|
||||
{"name": "DiscordBotTokenDetector"},
|
||||
{"name": "GitHubTokenDetector"},
|
||||
{"name": "GitLabTokenDetector"},
|
||||
{"name": "Base64HighEntropyString", "limit": 6.0},
|
||||
{"name": "HexHighEntropyString", "limit": 3.0},
|
||||
{"name": "IbmCloudIamDetector"},
|
||||
{"name": "IbmCosHmacDetector"},
|
||||
# {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885
|
||||
{"name": "JwtTokenDetector"},
|
||||
{"name": "KeywordDetector"},
|
||||
{"name": "MailchimpDetector"},
|
||||
{"name": "NpmDetector"},
|
||||
{"name": "OpenAIDetector"},
|
||||
{"name": "PrivateKeyDetector"},
|
||||
{"name": "PypiTokenDetector"},
|
||||
{"name": "SendGridDetector"},
|
||||
{"name": "SlackDetector"},
|
||||
{"name": "SoftlayerDetector"},
|
||||
{"name": "SquareOAuthDetector"},
|
||||
{"name": "StripeDetector"},
|
||||
# {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878
|
||||
{"name": "TwilioKeyDetector"},
|
||||
]
|
||||
# Default minimum confidence level for reporting findings. "low" is required to
|
||||
# enable Kingfisher's built-in generic rules (Generic Password / Secret / API
|
||||
# Key), which preserve the keyword-based coverage Prowler had with
|
||||
# detect-secrets' KeywordDetector; at "medium" those generic rules do not fire.
|
||||
# Possible values: "low", "medium", "high".
|
||||
default_secrets_confidence = "low"
|
||||
|
||||
# Kingfisher exit codes considered successful: 0 (no findings), 200 (findings),
|
||||
# 205 (validated findings).
|
||||
_kingfisher_success_exit_codes = (0, 200, 205)
|
||||
|
||||
# Number of payloads scanned per Kingfisher invocation in batch mode. Bounds
|
||||
# peak temp-disk and memory while still amortizing the per-process spawn cost
|
||||
# across many fragments (see detect_secrets_scan_batch).
|
||||
default_secrets_batch_chunk_size = 500
|
||||
|
||||
# Wall-clock cap (seconds) for a single Kingfisher subprocess, so a hung binary
|
||||
# cannot block the audit indefinitely.
|
||||
default_secrets_scan_timeout = 300
|
||||
|
||||
|
||||
class SecretsScanError(Exception):
|
||||
"""The secret scanner could not produce a trustworthy result.
|
||||
|
||||
Raised when Kingfisher exits with a non-success code, times out, cannot be
|
||||
located/executed, or returns output that cannot be parsed. This is distinct
|
||||
from "no secrets found": a security check must never treat a scanner failure
|
||||
as a clean result, so callers are expected to surface it as ``MANUAL``
|
||||
(manual review required) instead of ``PASS``.
|
||||
"""
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_kingfisher_binary() -> str:
|
||||
"""Return the path to the bundled Kingfisher binary (cached)."""
|
||||
from kingfisher import get_binary_path
|
||||
|
||||
return get_binary_path()
|
||||
|
||||
|
||||
def _build_kingfisher_command(
|
||||
scan_paths: list,
|
||||
output_path: str,
|
||||
confidence: str,
|
||||
validate: bool,
|
||||
no_dedup: bool = False,
|
||||
) -> list:
|
||||
"""Build the Kingfisher ``scan`` command shared by single and batch scans."""
|
||||
command = [
|
||||
get_kingfisher_binary(),
|
||||
"scan",
|
||||
*scan_paths,
|
||||
"--format",
|
||||
"json",
|
||||
"--output",
|
||||
output_path,
|
||||
"--no-update-check",
|
||||
"--confidence",
|
||||
confidence,
|
||||
]
|
||||
if validate:
|
||||
# Live-validate discovered secrets against provider APIs. Use
|
||||
# conservative defaults (short timeout, no retries) to limit the blast
|
||||
# radius of the outbound calls.
|
||||
command += ["--validation-timeout", "5", "--validation-retries", "0"]
|
||||
else:
|
||||
command.append("--no-validate")
|
||||
if no_dedup:
|
||||
# Report every occurrence (one per file) so batched results match
|
||||
# scanning each payload individually.
|
||||
command.append("--no-dedup")
|
||||
return command
|
||||
|
||||
|
||||
def _finding_to_dict(entry: dict, fallback_filename: str) -> dict:
|
||||
"""Convert a Kingfisher finding entry into Prowler's finding dict shape."""
|
||||
rule = entry.get("rule", {})
|
||||
finding = entry.get("finding", {})
|
||||
snippet = finding.get("snippet", "") or ""
|
||||
return {
|
||||
"filename": finding.get("path", fallback_filename),
|
||||
"line_number": finding.get("line"),
|
||||
"type": rule.get("name"),
|
||||
# Non-security identifier for the matched secret (matches the
|
||||
# detect-secrets output shape); not used for security.
|
||||
"hashed_secret": (
|
||||
sha1(snippet.encode(), usedforsecurity=False).hexdigest()
|
||||
if snippet
|
||||
else None
|
||||
),
|
||||
"is_verified": finding.get("validation", {}).get("status") == "Active",
|
||||
}
|
||||
|
||||
|
||||
def open_file(input_file: str, mode: str = "r") -> TextIOWrapper:
|
||||
@@ -111,77 +175,188 @@ def hash_sha512(string: str) -> str:
|
||||
return sha512(string.encode(encoding_format_utf_8)).hexdigest()[0:9]
|
||||
|
||||
|
||||
def detect_secrets_scan(
|
||||
data: str = None,
|
||||
file=None,
|
||||
excluded_secrets: list[str] = None,
|
||||
detect_secrets_plugins: dict = None,
|
||||
) -> list[dict[str, str]]:
|
||||
"""detect_secrets_scan scans the data or file for secrets using the detect-secrets library.
|
||||
Args:
|
||||
data (str): The data to scan for secrets.
|
||||
file (str): The file to scan for secrets.
|
||||
excluded_secrets (list): A list of regex patterns to exclude from the scan.
|
||||
detect_secrets_plugins (dict): The settings to use for the scan.
|
||||
Returns:
|
||||
dict: The secrets found in the
|
||||
Raises:
|
||||
Exception: If an error occurs during the scan.
|
||||
Examples:
|
||||
>>> detect_secrets_scan(data="password=password")
|
||||
[{'filename': 'data', 'hashed_secret': 'f7c3bc1d808e04732adf679965ccc34ca7ae3441', 'is_verified': False, 'line_number': 1, 'type': 'Secret Keyword'}]
|
||||
>>> detect_secrets_scan(file="file.txt")
|
||||
{'file.txt': [{'filename': 'file.txt', 'hashed_secret': 'f7c3bc1d808e04732adf679965ccc34ca7ae3441', 'is_verified': False, 'line_number': 1, 'type': 'Secret Keyword'}]}
|
||||
def _scan_batch_chunk(
|
||||
chunk: list,
|
||||
excluded_secrets: list,
|
||||
confidence: str,
|
||||
validate: bool,
|
||||
results: dict,
|
||||
) -> None:
|
||||
"""Scan one chunk of ``(key, data)`` payloads in a single Kingfisher call.
|
||||
|
||||
Writes each payload to its own file in a temp directory, scans the whole
|
||||
directory once (``--no-dedup`` so per-file results match individual scans),
|
||||
maps findings back to their key by file path, and appends them to
|
||||
``results``. The temp directory is always removed.
|
||||
"""
|
||||
if not chunk:
|
||||
return
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
temp_output_file = None
|
||||
try:
|
||||
if not file:
|
||||
temp_data_file = tempfile.NamedTemporaryFile(delete=False)
|
||||
temp_data_file.write(bytes(data, encoding="raw_unicode_escape"))
|
||||
temp_data_file.close()
|
||||
index_to_key = {}
|
||||
for index, (key, data) in enumerate(chunk):
|
||||
content = data if data.endswith("\n") else data + "\n"
|
||||
name = str(index)
|
||||
with open(os.path.join(tmp_dir, name), "wb") as fh:
|
||||
fh.write(bytes(content, encoding="raw_unicode_escape"))
|
||||
index_to_key[name] = key
|
||||
|
||||
secrets = SecretsCollection()
|
||||
|
||||
if not detect_secrets_plugins:
|
||||
detect_secrets_plugins = default_detect_secrets_plugins
|
||||
|
||||
settings = {
|
||||
"plugins_used": detect_secrets_plugins,
|
||||
"filters_used": [
|
||||
{"path": "detect_secrets.filters.common.is_invalid_file"},
|
||||
{"path": "detect_secrets.filters.common.is_known_false_positive"},
|
||||
{"path": "detect_secrets.filters.heuristic.is_likely_id_string"},
|
||||
{"path": "detect_secrets.filters.heuristic.is_potential_secret"},
|
||||
],
|
||||
}
|
||||
|
||||
if excluded_secrets and len(excluded_secrets) > 0:
|
||||
settings["filters_used"].append(
|
||||
{
|
||||
"path": "detect_secrets.filters.regex.should_exclude_line",
|
||||
"pattern": excluded_secrets,
|
||||
}
|
||||
temp_output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
||||
temp_output_file.close()
|
||||
command = _build_kingfisher_command(
|
||||
[tmp_dir], temp_output_file.name, confidence, validate, no_dedup=True
|
||||
)
|
||||
process = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=default_secrets_scan_timeout,
|
||||
)
|
||||
if process.returncode not in _kingfisher_success_exit_codes:
|
||||
raise SecretsScanError(
|
||||
f"Kingfisher exited with code {process.returncode}: "
|
||||
f"{process.stderr.strip()[:500]}"
|
||||
)
|
||||
with transient_settings(settings):
|
||||
if file:
|
||||
secrets.scan_file(file)
|
||||
else:
|
||||
secrets.scan_file(temp_data_file.name)
|
||||
|
||||
if not file:
|
||||
os.remove(temp_data_file.name)
|
||||
with open(temp_output_file.name, encoding=encoding_format_utf_8) as f:
|
||||
output = f.read()
|
||||
kingfisher_output = json.loads(output) if output.strip() else {}
|
||||
|
||||
detect_secrets_output = secrets.json()
|
||||
source_lines_cache = {}
|
||||
|
||||
if detect_secrets_output:
|
||||
if file:
|
||||
return detect_secrets_output[file]
|
||||
else:
|
||||
return detect_secrets_output[temp_data_file.name]
|
||||
else:
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error scanning for secrets: {e}")
|
||||
return None
|
||||
def source_lines(file_name: str) -> list:
|
||||
if file_name not in source_lines_cache:
|
||||
with open(
|
||||
os.path.join(tmp_dir, file_name),
|
||||
encoding=encoding_format_utf_8,
|
||||
errors="replace",
|
||||
) as f:
|
||||
source_lines_cache[file_name] = f.read().splitlines()
|
||||
return source_lines_cache[file_name]
|
||||
|
||||
for entry in kingfisher_output.get("findings", []):
|
||||
finding = entry.get("finding", {})
|
||||
name = os.path.basename(finding.get("path", ""))
|
||||
key = index_to_key.get(name)
|
||||
if key is None:
|
||||
continue
|
||||
# Validate the line index before any consumer trusts it. Checks use
|
||||
# ``line_number`` as a 1-based index into their own parallel data
|
||||
# (e.g. CloudWatch does ``events[line_number - 1]``), so a missing,
|
||||
# non-integer, or out-of-range line would crash the check or map the
|
||||
# secret to the wrong resource. Fail closed: surface a malformed
|
||||
# finding as a scan failure so callers report MANUAL instead of a
|
||||
# wrong PASS/FAIL. ``bool`` is rejected explicitly because it is a
|
||||
# subclass of ``int``.
|
||||
line_number = finding.get("line")
|
||||
lines = source_lines(name)
|
||||
if (
|
||||
isinstance(line_number, bool)
|
||||
or not isinstance(line_number, int)
|
||||
or not 1 <= line_number <= len(lines)
|
||||
):
|
||||
raise SecretsScanError(
|
||||
f"Kingfisher returned an invalid line number "
|
||||
f"{line_number!r} for a finding in {name}"
|
||||
)
|
||||
if excluded_secrets and any(
|
||||
re.search(pattern, lines[line_number - 1])
|
||||
for pattern in excluded_secrets
|
||||
):
|
||||
continue
|
||||
results.setdefault(key, []).append(_finding_to_dict(entry, name))
|
||||
except SecretsScanError:
|
||||
# Already a typed scan failure; propagate so callers report MANUAL.
|
||||
raise
|
||||
except subprocess.TimeoutExpired as error:
|
||||
raise SecretsScanError(
|
||||
f"Kingfisher timed out after {default_secrets_scan_timeout}s "
|
||||
"while scanning for secrets"
|
||||
) from error
|
||||
except Exception as error:
|
||||
# Fail closed: a missing/unexecutable binary, unparseable JSON output or
|
||||
# any other runtime failure must NOT be silently treated as "no secrets
|
||||
# found". Surface it so callers can report MANUAL instead of PASS.
|
||||
raise SecretsScanError(f"Secret scan failed: {error}") from error
|
||||
finally:
|
||||
if temp_output_file and os.path.exists(temp_output_file.name):
|
||||
os.remove(temp_output_file.name)
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def detect_secrets_scan_batch(
|
||||
payloads: Union[Mapping[Any, str], Iterable[tuple[Any, str]]],
|
||||
excluded_secrets: Optional[list[str]] = None,
|
||||
confidence: str = default_secrets_confidence,
|
||||
validate: bool = False,
|
||||
chunk_size: int = default_secrets_batch_chunk_size,
|
||||
) -> dict:
|
||||
"""Scan many payloads with Kingfisher in chunked subprocess invocations.
|
||||
|
||||
This is the scan entry point used by every secret check. Each payload is
|
||||
written to its own file and scanned with ``--no-dedup`` so per-payload
|
||||
results match scanning each payload on its own. Payloads are processed in
|
||||
chunks (writing each to disk and releasing it as it is consumed) to bound
|
||||
peak temp-disk and memory use while amortizing the per-process spawn cost
|
||||
across many fragments.
|
||||
|
||||
By default the scan runs fully offline (``--no-validate``,
|
||||
``--no-update-check``): no network calls are made, so the scanned data is
|
||||
never sent anywhere. When ``validate`` is True, Kingfisher additionally
|
||||
checks whether each discovered secret is live by authenticating with it
|
||||
against the provider's API (the secret itself is the credential; no extra
|
||||
permissions are required). That makes outbound network calls, so it must be
|
||||
explicitly opted in.
|
||||
|
||||
Args:
|
||||
payloads: a mapping ``{key: data}`` or any iterable of ``(key, data)``
|
||||
pairs. ``key`` is any hashable the caller uses to map findings back
|
||||
to its source (e.g. a variable name or a ``(resource, stream)``).
|
||||
excluded_secrets (list): regex patterns; a finding whose source line
|
||||
matches one is excluded.
|
||||
confidence (str): minimum Kingfisher confidence ("low"/"medium"/"high").
|
||||
validate (bool): live-validate discovered secrets (outbound calls).
|
||||
chunk_size (int): payloads scanned per Kingfisher invocation.
|
||||
Returns:
|
||||
dict mapping each key that produced findings to its list of finding
|
||||
dicts, each with ``filename``, ``line_number``, ``type``,
|
||||
``hashed_secret`` and ``is_verified`` keys. Keys with no findings are
|
||||
omitted.
|
||||
Raises:
|
||||
SecretsScanError: if the scanner fails for any chunk (non-success exit
|
||||
code, timeout, missing/unexecutable binary or unparseable output).
|
||||
An empty result is therefore always "no secrets found", never a
|
||||
silent scan failure; callers must report MANUAL on this error.
|
||||
"""
|
||||
items = payloads.items() if hasattr(payloads, "items") else payloads
|
||||
results = {}
|
||||
chunk = []
|
||||
for key, data in items:
|
||||
chunk.append((key, data))
|
||||
if len(chunk) >= chunk_size:
|
||||
_scan_batch_chunk(chunk, excluded_secrets, confidence, validate, results)
|
||||
chunk = []
|
||||
_scan_batch_chunk(chunk, excluded_secrets, confidence, validate, results)
|
||||
return results
|
||||
|
||||
|
||||
def annotate_verified_secrets(report, secrets: list) -> None:
|
||||
"""Escalate and annotate a finding when any of its secrets is confirmed live.
|
||||
|
||||
When secret validation (``--scan-secrets-validate`` / ``secrets_validate``)
|
||||
confirms that a discovered secret is live, the finding is more severe than a
|
||||
potential secret: its severity is raised to critical and a note is appended
|
||||
to ``status_extended``. No-op when no secret was validated as live, so the
|
||||
default offline behavior (and existing finding messages) is unchanged.
|
||||
"""
|
||||
if secrets and any(secret.get("is_verified") for secret in secrets):
|
||||
from prowler.lib.check.models import Severity
|
||||
|
||||
report.check_metadata.Severity = Severity.critical
|
||||
report.status_extended += (
|
||||
" One or more of these secrets were confirmed to be live."
|
||||
)
|
||||
|
||||
|
||||
def validate_ip_address(ip_string):
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"Provider": "aws",
|
||||
"CheckID": "apigateway_restapi_no_secrets_in_stage_variables",
|
||||
"CheckTitle": "API Gateway REST API stage variables should not contain secrets",
|
||||
"CheckType": [
|
||||
"Software and Configuration Checks/AWS Security Best Practices",
|
||||
"Software and Configuration Checks/Industry and Regulatory Standards/AWS Foundational Security Best Practices"
|
||||
],
|
||||
"ServiceName": "apigateway",
|
||||
"SubServiceName": "",
|
||||
"ResourceIdTemplate": "arn:aws:apigateway:region::/restapis/api-id/stages/stage-name",
|
||||
"Severity": "high",
|
||||
"ResourceType": "AwsApiGatewayStage",
|
||||
"ResourceGroup": "security",
|
||||
"Description": "Checks API Gateway REST API stage variables for hardcoded secrets such as passwords, API keys, and tokens. Stage variables should reference AWS Secrets Manager or Parameter Store rather than containing plaintext credentials.",
|
||||
"Risk": "Hardcoded secrets in stage variables are stored in plaintext in the AWS control plane and are visible to anyone with read access to the API Gateway configuration. This can lead to unauthorized access, credential theft, and lateral movement across systems.",
|
||||
"RelatedUrl": "",
|
||||
"AdditionalURLs": [
|
||||
"https://docs.aws.amazon.com/secretsmanager/latest/userguide/integrating_how-services-use-secrets_api-gateway.html"
|
||||
],
|
||||
"Remediation": {
|
||||
"Code": {
|
||||
"CLI": "aws apigateway update-stage --rest-api-id <api-id> --stage-name <stage-name> --patch-operations op=remove,path=/variables/<variable-name>",
|
||||
"NativeIaC": "",
|
||||
"Other": "1. Open AWS Console > API Gateway\n2. Select the REST API and stage\n3. Go to Stage Variables tab\n4. Remove any variables containing plaintext secrets\n5. Reference secrets using AWS Secrets Manager integration instead",
|
||||
"Terraform": ""
|
||||
},
|
||||
"Recommendation": {
|
||||
"Text": "Remove hardcoded secrets from API Gateway stage variables. Use AWS Secrets Manager or Parameter Store to manage credentials and retrieve them at runtime using Lambda authorizers or integration request mapping templates.",
|
||||
"Url": "https://hub.prowler.com/check/apigateway_restapi_no_secrets_in_stage_variables"
|
||||
}
|
||||
},
|
||||
"Categories": [
|
||||
"secrets"
|
||||
],
|
||||
"DependsOn": [],
|
||||
"RelatedTo": [],
|
||||
"Notes": "Infrastructure Protection"
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
import json
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.apigateway.apigateway_client import (
|
||||
apigateway_client,
|
||||
)
|
||||
|
||||
|
||||
class apigateway_restapi_no_secrets_in_stage_variables(Check):
|
||||
"""Check that API Gateway REST API stage variables contain no hardcoded secrets."""
|
||||
|
||||
def execute(self) -> list[Check_Report_AWS]:
|
||||
findings = []
|
||||
secrets_ignore_patterns = apigateway_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
validate = apigateway_client.audit_config.get("secrets_validate", False)
|
||||
|
||||
# Collect one payload per stage (its variables) and scan them all in
|
||||
# batched Kingfisher invocations instead of one subprocess per stage.
|
||||
# Findings are keyed by (rest_api index, stage index).
|
||||
def payloads():
|
||||
for api_index, rest_api in enumerate(apigateway_client.rest_apis):
|
||||
for stage_index, stage in enumerate(rest_api.stages):
|
||||
if stage.variables:
|
||||
yield (api_index, stage_index), json.dumps(
|
||||
stage.variables, indent=2
|
||||
)
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
validate=validate,
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for api_index, rest_api in enumerate(apigateway_client.rest_apis):
|
||||
for stage_index, stage in enumerate(rest_api.stages):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=rest_api)
|
||||
report.resource_arn = stage.arn
|
||||
report.resource_id = f"{rest_api.name}/{stage.name}"
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in stage variables of API Gateway "
|
||||
f"REST API {rest_api.name} stage {stage.name}."
|
||||
)
|
||||
|
||||
if stage.variables:
|
||||
if scan_error:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan stage variables of API Gateway REST API "
|
||||
f"{rest_api.name} stage {stage.name} for secrets: "
|
||||
f"{scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
detect_secrets_output = batch_results.get((api_index, stage_index))
|
||||
if detect_secrets_output:
|
||||
variable_names = list(stage.variables.keys())
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} in variable "
|
||||
f"{variable_names[secret['line_number'] - 2]}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = (
|
||||
f"Potential "
|
||||
f"{'secrets' if len(detect_secrets_output) > 1 else 'secret'} "
|
||||
f"found in stage variables of API Gateway REST API "
|
||||
f"{rest_api.name} stage {stage.name} -> {secrets_string}."
|
||||
)
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
@@ -179,6 +179,7 @@ class APIGateway(AWSService):
|
||||
tracing_enabled=tracing_enabled,
|
||||
cache_enabled=cache_enabled,
|
||||
cache_data_encrypted=cache_data_encrypted,
|
||||
variables=stage.get("variables", {}),
|
||||
)
|
||||
)
|
||||
except ClientError as error:
|
||||
@@ -265,6 +266,7 @@ class Stage(BaseModel):
|
||||
tracing_enabled: Optional[bool] = None
|
||||
cache_enabled: Optional[bool] = None
|
||||
cache_data_encrypted: Optional[bool] = None
|
||||
variables: Optional[dict] = {}
|
||||
|
||||
|
||||
class PathResourceMethods(BaseModel):
|
||||
|
||||
+45
-13
@@ -4,7 +4,11 @@ from base64 import b64decode
|
||||
from prowler.config.config import encoding_format_utf_8
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.autoscaling.autoscaling_client import (
|
||||
autoscaling_client,
|
||||
)
|
||||
@@ -16,13 +20,19 @@ class autoscaling_find_secrets_ec2_launch_configuration(Check):
|
||||
secrets_ignore_patterns = autoscaling_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for (
|
||||
configuration_arn,
|
||||
configuration,
|
||||
) in autoscaling_client.launch_configurations.items():
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=configuration)
|
||||
validate = autoscaling_client.audit_config.get("secrets_validate", False)
|
||||
configurations = list(autoscaling_client.launch_configurations.values())
|
||||
|
||||
if configuration.user_data:
|
||||
# Collect the decoded User Data of each launch configuration and scan it
|
||||
# all in batched Kingfisher invocations instead of one subprocess each.
|
||||
# Configurations whose User Data cannot be decoded are undecodable (no report),
|
||||
# matching the original per-resource behavior.
|
||||
undecodable = set()
|
||||
|
||||
def payloads():
|
||||
for index, configuration in enumerate(configurations):
|
||||
if not configuration.user_data:
|
||||
continue
|
||||
user_data = b64decode(configuration.user_data)
|
||||
try:
|
||||
if user_data[0:2] == b"\x1f\x8b": # GZIP magic number
|
||||
@@ -35,24 +45,46 @@ class autoscaling_find_secrets_ec2_launch_configuration(Check):
|
||||
logger.warning(
|
||||
f"{configuration.region} -- Unable to decode user data in autoscaling launch configuration {configuration.name}: {error}"
|
||||
)
|
||||
undecodable.add(index)
|
||||
continue
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{configuration.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
undecodable.add(index)
|
||||
continue
|
||||
yield index, user_data
|
||||
|
||||
has_secrets = detect_secrets_scan(
|
||||
data=user_data,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=autoscaling_client.audit_config.get(
|
||||
"detect_secrets_plugins"
|
||||
),
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for index, configuration in enumerate(configurations):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=configuration)
|
||||
|
||||
if scan_error and configuration.user_data:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan autoscaling {configuration.name} User Data for "
|
||||
f"secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
if index in undecodable:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = f"Could not decode User Data for autoscaling {configuration.name}; manual review is required to scan for secrets."
|
||||
elif configuration.user_data:
|
||||
has_secrets = batch_results.get(index)
|
||||
if has_secrets:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in autoscaling {configuration.name} User Data."
|
||||
annotate_verified_secrets(report, has_secrets)
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in autoscaling {configuration.name} User Data."
|
||||
|
||||
+104
-49
@@ -1,65 +1,120 @@
|
||||
import fnmatch
|
||||
import os
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.awslambda.awslambda_client import awslambda_client
|
||||
|
||||
|
||||
class awslambda_function_no_secrets_in_code(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
if awslambda_client.functions:
|
||||
secrets_ignore_patterns = awslambda_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
if not awslambda_client.functions:
|
||||
return findings
|
||||
|
||||
secrets_ignore_patterns = awslambda_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
# Glob patterns of file names inside the deployment package to skip
|
||||
# when scanning for secrets (e.g. "*.deps.json" for .NET Lambdas).
|
||||
secrets_ignore_files = (
|
||||
awslambda_client.audit_config.get("secrets_ignore_files", []) or []
|
||||
)
|
||||
validate = awslambda_client.audit_config.get("secrets_validate", False)
|
||||
|
||||
# Scan files of every function's package in batched
|
||||
# Kingfisher invocations instead of one subprocess per file per function.
|
||||
# Each package is extracted one at a time and its files are
|
||||
# read (byte-faithfully via latin-1) before the extraction is released,
|
||||
# so only a single package is on disk at a time. Findings are keyed by
|
||||
# (function index, package-relative file name) so they can be grouped
|
||||
# back per function.
|
||||
functions_with_code = []
|
||||
|
||||
def code_payloads():
|
||||
for function, function_code in awslambda_client._get_function_code():
|
||||
if function_code:
|
||||
report = Check_Report_AWS(
|
||||
metadata=self.metadata(), resource=function
|
||||
)
|
||||
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in Lambda function {function.name} code."
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir_name:
|
||||
function_code.code_zip.extractall(tmp_dir_name)
|
||||
# List all files
|
||||
files_in_zip = next(os.walk(tmp_dir_name))[2]
|
||||
secrets_findings = []
|
||||
for file in files_in_zip:
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
file=f"{tmp_dir_name}/{file}",
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=awslambda_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
if not function_code:
|
||||
continue
|
||||
index = len(functions_with_code)
|
||||
functions_with_code.append(function)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir_name:
|
||||
function_code.code_zip.extractall(tmp_dir_name)
|
||||
for root, _, files in os.walk(tmp_dir_name):
|
||||
for file_name in files:
|
||||
file_path = os.path.join(root, file_name)
|
||||
relative_file_path = os.path.relpath(
|
||||
file_path, tmp_dir_name
|
||||
)
|
||||
if detect_secrets_output:
|
||||
for (
|
||||
secret
|
||||
) in (
|
||||
detect_secrets_output
|
||||
): # Appears that only 1 file is being scanned at a time, so could rework this
|
||||
output_file_name = secret["filename"].replace(
|
||||
f"{tmp_dir_name}/", ""
|
||||
)
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} on line {secret['line_number']}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
)
|
||||
secrets_findings.append(
|
||||
f"{output_file_name}: {secrets_string}"
|
||||
)
|
||||
if any(
|
||||
fnmatch.fnmatch(relative_file_path, pattern)
|
||||
for pattern in secrets_ignore_files
|
||||
):
|
||||
continue
|
||||
try:
|
||||
with open(file_path, "rb") as code_file:
|
||||
content = code_file.read().decode("latin-1")
|
||||
except Exception:
|
||||
continue
|
||||
yield (index, relative_file_path), content
|
||||
|
||||
if secrets_findings:
|
||||
final_output_string = "; ".join(secrets_findings)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential {'secrets' if len(secrets_findings) > 1 else 'secret'} found in Lambda function {function.name} code -> {final_output_string}."
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
code_payloads(),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
validate=validate,
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
findings.append(report)
|
||||
if scan_error:
|
||||
# The scan failed before any function's code could be cleared. Report
|
||||
# MANUAL for every function rather than risk a false PASS.
|
||||
for function in awslambda_client.functions.values():
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=function)
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan Lambda function {function.name} code for "
|
||||
f"secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
return findings
|
||||
|
||||
findings_by_function = defaultdict(dict)
|
||||
for (index, file_name), file_findings in batch_results.items():
|
||||
findings_by_function[index][file_name] = file_findings
|
||||
|
||||
for index, function in enumerate(functions_with_code):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=function)
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in Lambda function {function.name} code."
|
||||
)
|
||||
|
||||
files_with_secrets = findings_by_function.get(index)
|
||||
if files_with_secrets:
|
||||
all_secrets = []
|
||||
secrets_findings = []
|
||||
for file_name, file_findings in files_with_secrets.items():
|
||||
all_secrets.extend(file_findings)
|
||||
secrets_string = ", ".join(
|
||||
f"{secret['type']} on line {secret['line_number']}"
|
||||
for secret in file_findings
|
||||
)
|
||||
secrets_findings.append(f"{file_name}: {secrets_string}")
|
||||
|
||||
final_output_string = "; ".join(secrets_findings)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential {'secrets' if len(secrets_findings) > 1 else 'secret'} found in Lambda function {function.name} code -> {final_output_string}."
|
||||
annotate_verified_secrets(report, all_secrets)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
|
||||
+40
-12
@@ -1,7 +1,11 @@
|
||||
import json
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.awslambda.awslambda_client import awslambda_client
|
||||
|
||||
|
||||
@@ -11,7 +15,30 @@ class awslambda_function_no_secrets_in_variables(Check):
|
||||
secrets_ignore_patterns = awslambda_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for function in awslambda_client.functions.values():
|
||||
validate = awslambda_client.audit_config.get("secrets_validate", False)
|
||||
functions = list(awslambda_client.functions.values())
|
||||
|
||||
# Scan every function's environment variables in batched Kingfisher
|
||||
# invocations instead of one subprocess per function. Payloads are
|
||||
# yielded lazily so only a chunk is held/written at a time, which matters
|
||||
# for accounts with very large numbers of Lambda functions.
|
||||
def environment_payloads():
|
||||
for index, function in enumerate(functions):
|
||||
if function.environment:
|
||||
yield index, json.dumps(function.environment, indent=2)
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
environment_payloads(),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
validate=validate,
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for index, function in enumerate(functions):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=function)
|
||||
|
||||
report.status = "PASS"
|
||||
@@ -20,17 +47,17 @@ class awslambda_function_no_secrets_in_variables(Check):
|
||||
)
|
||||
|
||||
if function.environment:
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=json.dumps(function.environment, indent=2),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=awslambda_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
original_env_vars = []
|
||||
for name, value in function.environment.items():
|
||||
original_env_vars.append(name)
|
||||
if scan_error:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan Lambda function {function.name} variables "
|
||||
f"for secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
detect_secrets_output = batch_results.get(index)
|
||||
if detect_secrets_output:
|
||||
original_env_vars = list(function.environment.keys())
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} in variable {original_env_vars[secret['line_number'] - 2]}"
|
||||
@@ -39,6 +66,7 @@ class awslambda_function_no_secrets_in_variables(Check):
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in Lambda function {function.name} variables -> {secrets_string}."
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ from botocore.client import ClientError
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.resource_limit import (
|
||||
get_resource_scan_limit,
|
||||
limit_resources,
|
||||
)
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
@@ -18,8 +22,16 @@ class Lambda(AWSService):
|
||||
def __init__(self, provider):
|
||||
# Call AWSService's __init__
|
||||
super().__init__(__class__.__name__, provider)
|
||||
# Functions are listed first, then trimmed to the subset selected for
|
||||
# analysis before expensive per-function detail is hydrated.
|
||||
self.functions = {}
|
||||
self.security_groups_in_use = set()
|
||||
self.regions_with_functions = set()
|
||||
self.function_limit = get_resource_scan_limit(
|
||||
self.audit_config, "max_lambda_functions"
|
||||
)
|
||||
self.__threading_call__(self._list_functions)
|
||||
self._select_functions_for_analysis()
|
||||
self._list_tags_for_resource()
|
||||
self.__threading_call__(self._get_policy)
|
||||
self.__threading_call__(self._get_function_url_config)
|
||||
@@ -30,24 +42,29 @@ class Lambda(AWSService):
|
||||
try:
|
||||
list_functions_paginator = regional_client.get_paginator("list_functions")
|
||||
for page in list_functions_paginator.paginate():
|
||||
for function in page["Functions"]:
|
||||
if not self.audit_resources or (
|
||||
is_resource_filtered(
|
||||
function["FunctionArn"], self.audit_resources
|
||||
)
|
||||
for function in page.get("Functions", []):
|
||||
if not self.audit_resources or is_resource_filtered(
|
||||
function["FunctionArn"], self.audit_resources
|
||||
):
|
||||
lambda_name = function["FunctionName"]
|
||||
lambda_arn = function["FunctionArn"]
|
||||
vpc_config = function.get("VpcConfig", {})
|
||||
security_groups = vpc_config.get("SecurityGroupIds", [])
|
||||
self.security_groups_in_use.update(security_groups)
|
||||
self.regions_with_functions.add(regional_client.region)
|
||||
# We must use the Lambda ARN as the dict key since we could have Lambdas in different regions with the same name
|
||||
self.functions[lambda_arn] = Function(
|
||||
name=lambda_name,
|
||||
arn=lambda_arn,
|
||||
security_groups=vpc_config.get("SecurityGroupIds", []),
|
||||
security_groups=security_groups,
|
||||
vpc_id=vpc_config.get("VpcId"),
|
||||
subnet_ids=set(vpc_config.get("SubnetIds", [])),
|
||||
region=regional_client.region,
|
||||
)
|
||||
if "LastModified" in function:
|
||||
self.functions[lambda_arn].last_modified = function[
|
||||
"LastModified"
|
||||
]
|
||||
if "Runtime" in function:
|
||||
self.functions[lambda_arn].runtime = function["Runtime"]
|
||||
if "Environment" in function:
|
||||
@@ -76,26 +93,61 @@ class Lambda(AWSService):
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
def _select_functions_for_analysis(self):
|
||||
self.functions = {
|
||||
function.arn: function
|
||||
for function in limit_resources(
|
||||
sorted(
|
||||
self.functions.values(),
|
||||
key=lambda f: f.last_modified or "",
|
||||
reverse=True,
|
||||
),
|
||||
self.function_limit,
|
||||
)
|
||||
}
|
||||
|
||||
def _list_event_source_mappings(self, regional_client):
|
||||
logger.info("Lambda - Listing Event Source Mappings...")
|
||||
try:
|
||||
paginator = regional_client.get_paginator("list_event_source_mappings")
|
||||
for page in paginator.paginate():
|
||||
for mapping in page.get("EventSourceMappings", []):
|
||||
function_arn = mapping.get("FunctionArn", "")
|
||||
# Normalise to unqualified ARN (strip :qualifier suffix if present)
|
||||
base_arn = ":".join(function_arn.split(":")[:7])
|
||||
if base_arn not in self.functions:
|
||||
continue
|
||||
self.functions[base_arn].event_source_mappings.append(
|
||||
EventSourceMapping(
|
||||
uuid=mapping["UUID"],
|
||||
event_source_arn=mapping.get("EventSourceArn", ""),
|
||||
state=mapping.get("State", ""),
|
||||
batch_size=mapping.get("BatchSize"),
|
||||
starting_position=mapping.get("StartingPosition"),
|
||||
if not self.function_limit:
|
||||
for page in paginator.paginate():
|
||||
self._add_event_source_mappings(page.get("EventSourceMappings", []))
|
||||
return
|
||||
|
||||
for function in self.functions.values():
|
||||
if function.region != regional_client.region:
|
||||
continue
|
||||
try:
|
||||
for page in paginator.paginate(FunctionName=function.name):
|
||||
self._add_event_source_mappings(
|
||||
page.get("EventSourceMappings", [])
|
||||
)
|
||||
)
|
||||
except ClientError as error:
|
||||
if (
|
||||
error.response.get("Error", {}).get("Code")
|
||||
== "InvalidParameterValueException"
|
||||
):
|
||||
logger.warning(
|
||||
f"{function.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"{function.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
raise
|
||||
except ClientError as error:
|
||||
if self.function_limit:
|
||||
raise
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
@@ -103,6 +155,23 @@ class Lambda(AWSService):
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
def _add_event_source_mappings(self, event_source_mappings):
|
||||
for mapping in event_source_mappings:
|
||||
function_arn = mapping.get("FunctionArn", "")
|
||||
# Normalise to unqualified ARN (strip :qualifier suffix if present)
|
||||
base_arn = ":".join(function_arn.split(":")[:7])
|
||||
if base_arn not in self.functions:
|
||||
continue
|
||||
self.functions[base_arn].event_source_mappings.append(
|
||||
EventSourceMapping(
|
||||
uuid=mapping["UUID"],
|
||||
event_source_arn=mapping.get("EventSourceArn", ""),
|
||||
state=mapping.get("State", ""),
|
||||
batch_size=mapping.get("BatchSize"),
|
||||
starting_position=mapping.get("StartingPosition"),
|
||||
)
|
||||
)
|
||||
|
||||
def _get_function_code(self):
|
||||
logger.info("Lambda - Getting Function Code...")
|
||||
# Use a thread pool handle the queueing and execution of the _fetch_function_code tasks, up to max_workers tasks concurrently.
|
||||
@@ -158,7 +227,6 @@ class Lambda(AWSService):
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "ResourceNotFoundException":
|
||||
self.functions[function.arn].policy = {}
|
||||
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
@@ -187,7 +255,6 @@ class Lambda(AWSService):
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "ResourceNotFoundException":
|
||||
self.functions[function.arn].url_config = None
|
||||
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
@@ -206,10 +273,9 @@ class Lambda(AWSService):
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "ResourceNotFoundException":
|
||||
function.tags = []
|
||||
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
f"{function.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
|
||||
@@ -259,6 +325,7 @@ class Function(BaseModel):
|
||||
name: str
|
||||
arn: str
|
||||
security_groups: list
|
||||
last_modified: Optional[str] = None
|
||||
runtime: Optional[str] = None
|
||||
environment: Optional[dict] = None
|
||||
region: str
|
||||
|
||||
@@ -5,6 +5,10 @@ from botocore.client import ClientError
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.resource_limit import (
|
||||
get_resource_scan_limit,
|
||||
limit_resources,
|
||||
)
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
@@ -27,8 +31,14 @@ class Backup(AWSService):
|
||||
self.__threading_call__(self._list_backup_report_plans)
|
||||
self.protected_resources = []
|
||||
self.__threading_call__(self._list_backup_selections)
|
||||
# Recovery points are listed first, then only the selected subset is
|
||||
# tagged and exposed for checks.
|
||||
self.recovery_points = []
|
||||
self.__threading_call__(self._list_recovery_points)
|
||||
self.recovery_point_limit = get_resource_scan_limit(
|
||||
self.audit_config, "max_backup_recovery_points"
|
||||
)
|
||||
self.__threading_call__(self._list_recovery_points, self.backup_vaults or [])
|
||||
self._select_recovery_points_for_analysis()
|
||||
self.__threading_call__(self._list_tags, self.recovery_points)
|
||||
|
||||
def _list_backup_vaults(self, regional_client):
|
||||
@@ -183,40 +193,63 @@ class Backup(AWSService):
|
||||
f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _list_recovery_points(self, regional_client):
|
||||
def _list_recovery_points(self, backup_vault=None):
|
||||
logger.info("Backup - Listing Recovery Points...")
|
||||
if backup_vault is None:
|
||||
for vault in self.backup_vaults or []:
|
||||
self._list_recovery_points(vault)
|
||||
return
|
||||
|
||||
try:
|
||||
if self.backup_vaults:
|
||||
for backup_vault in self.backup_vaults:
|
||||
paginator = regional_client.get_paginator(
|
||||
"list_recovery_points_by_backup_vault"
|
||||
)
|
||||
for page in paginator.paginate(BackupVaultName=backup_vault.name):
|
||||
for recovery_point in page.get("RecoveryPoints", []):
|
||||
arn = recovery_point.get("RecoveryPointArn")
|
||||
if arn:
|
||||
self.recovery_points.append(
|
||||
RecoveryPoint(
|
||||
arn=arn,
|
||||
id=arn.split(":")[-1],
|
||||
backup_vault_name=backup_vault.name,
|
||||
encrypted=recovery_point.get(
|
||||
"IsEncrypted", False
|
||||
),
|
||||
backup_vault_region=backup_vault.region,
|
||||
region=regional_client.region,
|
||||
tags=[],
|
||||
)
|
||||
)
|
||||
regional_client = self.regional_clients[backup_vault.region]
|
||||
paginator = regional_client.get_paginator(
|
||||
"list_recovery_points_by_backup_vault"
|
||||
)
|
||||
for page in paginator.paginate(BackupVaultName=backup_vault.name):
|
||||
for recovery_point in page.get("RecoveryPoints", []):
|
||||
arn = recovery_point.get("RecoveryPointArn")
|
||||
if arn:
|
||||
rp = RecoveryPoint(
|
||||
arn=arn,
|
||||
id=arn.split(":")[-1],
|
||||
backup_vault_name=backup_vault.name,
|
||||
encrypted=recovery_point.get("IsEncrypted", False),
|
||||
creation_date=recovery_point.get("CreationDate"),
|
||||
backup_vault_region=backup_vault.region,
|
||||
region=backup_vault.region,
|
||||
tags=[],
|
||||
)
|
||||
self.recovery_points.append(rp)
|
||||
except ClientError as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
f"{backup_vault.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
f"{backup_vault.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _select_recovery_points_for_analysis(self):
|
||||
self.recovery_points = list(
|
||||
limit_resources(
|
||||
sorted(
|
||||
self.recovery_points,
|
||||
key=lambda rp: (
|
||||
(
|
||||
-rp.creation_date.timestamp()
|
||||
if isinstance(rp.creation_date, datetime)
|
||||
else 0.0
|
||||
),
|
||||
rp.region or "",
|
||||
rp.backup_vault_name or "",
|
||||
rp.arn or "",
|
||||
rp.id or "",
|
||||
),
|
||||
),
|
||||
self.recovery_point_limit,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class BackupVault(BaseModel):
|
||||
arn: str
|
||||
@@ -256,4 +289,5 @@ class RecoveryPoint(BaseModel):
|
||||
backup_vault_name: str
|
||||
encrypted: bool
|
||||
backup_vault_region: str
|
||||
creation_date: Optional[datetime] = None
|
||||
tags: Optional[list] = None
|
||||
|
||||
+5
-4
@@ -30,12 +30,13 @@ class bedrock_model_invocation_logs_encryption_enabled(Check):
|
||||
s3_encryption = False
|
||||
if logging.cloudwatch_log_group:
|
||||
log_group_arn = f"arn:{logs_client.audited_partition}:logs:{region}:{logs_client.audited_account}:log-group:{logging.cloudwatch_log_group}"
|
||||
all_log_groups = getattr(logs_client, "all_log_groups", None) or {}
|
||||
if (
|
||||
log_group_arn in logs_client.log_groups
|
||||
and not logs_client.log_groups[log_group_arn].kms_id
|
||||
log_group_arn in all_log_groups
|
||||
and not all_log_groups[log_group_arn].kms_id
|
||||
) or (
|
||||
log_group_arn + ":*" in logs_client.log_groups
|
||||
and not logs_client.log_groups[log_group_arn + ":*"].kms_id
|
||||
log_group_arn + ":*" in all_log_groups
|
||||
and not all_log_groups[log_group_arn + ":*"].kms_id
|
||||
):
|
||||
cloudwatch_encryption = False
|
||||
if not s3_encryption and not cloudwatch_encryption:
|
||||
|
||||
+35
-16
@@ -1,5 +1,9 @@
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.cloudformation.cloudformation_client import (
|
||||
cloudformation_client,
|
||||
)
|
||||
@@ -14,26 +18,41 @@ class cloudformation_stack_outputs_find_secrets(Check):
|
||||
secrets_ignore_patterns = cloudformation_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for stack in cloudformation_client.stacks:
|
||||
validate = cloudformation_client.audit_config.get("secrets_validate", False)
|
||||
stacks = list(cloudformation_client.stacks)
|
||||
|
||||
# Collect one payload per stack (its Outputs) and scan them all in
|
||||
# batched Kingfisher invocations instead of one subprocess per stack.
|
||||
def payloads():
|
||||
for index, stack in enumerate(stacks):
|
||||
if stack.outputs:
|
||||
yield index, "".join(f"{output}\n" for output in stack.outputs)
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for index, stack in enumerate(stacks):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=stack)
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in CloudFormation Stack {stack.name} Outputs."
|
||||
)
|
||||
if stack.outputs:
|
||||
data = ""
|
||||
# Store the CloudFormation Stack Outputs into a file
|
||||
for output in stack.outputs:
|
||||
data += f"{output}\n"
|
||||
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=data,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=cloudformation_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
# If secrets are found, update the report status
|
||||
if scan_error:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan CloudFormation Stack {stack.name} Outputs "
|
||||
f"for secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
detect_secrets_output = batch_results.get(index)
|
||||
if detect_secrets_output:
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
@@ -43,7 +62,7 @@ class cloudformation_stack_outputs_find_secrets(Check):
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in CloudFormation Stack {stack.name} Outputs -> {secrets_string}."
|
||||
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
|
||||
+191
-85
@@ -1,7 +1,11 @@
|
||||
from json import dumps, loads
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.cloudwatch.cloudwatch_service import (
|
||||
convert_to_cloudwatch_timestamp_format,
|
||||
)
|
||||
@@ -11,95 +15,197 @@ from prowler.providers.aws.services.cloudwatch.logs_client import logs_client
|
||||
class cloudwatch_log_group_no_secrets_in_logs(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
if logs_client.log_groups:
|
||||
secrets_ignore_patterns = logs_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
if not logs_client.log_groups:
|
||||
return findings
|
||||
|
||||
secrets_ignore_patterns = logs_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
validate = logs_client.audit_config.get("secrets_validate", False)
|
||||
|
||||
# Phase 1: batch-scan every (log group, log stream). Payloads are yielded
|
||||
# lazily so only a chunk is written/held at a time, which matters for
|
||||
# accounts with very large numbers of log groups/streams. The log group
|
||||
# ARN (not its name) keys every map below, since group and stream names
|
||||
# are not unique across regions and would otherwise collide.
|
||||
def stream_payloads():
|
||||
for log_group in logs_client.log_groups.values():
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=log_group)
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in {log_group.name} log group."
|
||||
if not log_group.log_streams:
|
||||
continue
|
||||
for log_stream_name, events in log_group.log_streams.items():
|
||||
yield (
|
||||
(log_group.arn, log_stream_name),
|
||||
"\n".join(dumps(event["message"]) for event in events),
|
||||
)
|
||||
|
||||
# A scanner failure here must never look like "no secrets": log groups
|
||||
# whose streams could not be scanned are reported MANUAL in Phase 4.
|
||||
stream_scan_error = None
|
||||
try:
|
||||
stream_results = detect_secrets_scan_batch(
|
||||
stream_payloads(),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
validate=validate,
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
stream_results = {}
|
||||
stream_scan_error = error
|
||||
|
||||
# Phase 2: plan the per-event secrets for each flagged stream and collect
|
||||
# the multiline events to rescan. Each multiline event is rescanned once
|
||||
# to resolve per-line detail; the rescans are batched in Phase 3 instead
|
||||
# of one subprocess per event. The event index (``line_number - 1``,
|
||||
# since Phase 1 joins one event per line) is the per-event discriminator:
|
||||
# a CloudWatch stream can hold several events sharing one millisecond
|
||||
# timestamp, so keying only by timestamp would let a later multiline
|
||||
# event overwrite an earlier one's payload and lose secret evidence.
|
||||
# Output still groups/displays by timestamp; only the rescan identity is
|
||||
# per event.
|
||||
# stream_plans: (group arn, stream) ->
|
||||
# {timestamp: {event index: {"multiline", "types"}}}
|
||||
# rescan_payloads: (group arn, stream, timestamp, event index) ->
|
||||
# multiline event data
|
||||
stream_plans = {}
|
||||
rescan_payloads = {}
|
||||
groups_with_rescan = set() # group arns that depend on the Phase 3 rescan
|
||||
for log_group in logs_client.log_groups.values():
|
||||
for log_stream_name in log_group.log_streams or {}:
|
||||
stream_secrets = stream_results.get((log_group.arn, log_stream_name))
|
||||
if not stream_secrets:
|
||||
continue
|
||||
events = log_group.log_streams[log_stream_name]
|
||||
plan = {}
|
||||
for secret in stream_secrets:
|
||||
event_index = secret["line_number"] - 1
|
||||
flagged_event = events[event_index]
|
||||
cloudwatch_timestamp = convert_to_cloudwatch_timestamp_format(
|
||||
flagged_event["timestamp"]
|
||||
)
|
||||
try:
|
||||
log_event_data = dumps(
|
||||
loads(flagged_event["message"]), indent=2
|
||||
)
|
||||
except Exception:
|
||||
log_event_data = dumps(flagged_event["message"], indent=2)
|
||||
multiline = len(log_event_data.split("\n")) > 1
|
||||
events_at_timestamp = plan.setdefault(cloudwatch_timestamp, {})
|
||||
if event_index not in events_at_timestamp:
|
||||
events_at_timestamp[event_index] = {
|
||||
"multiline": multiline,
|
||||
"types": [],
|
||||
}
|
||||
if multiline:
|
||||
# More informative output is possible with more than one
|
||||
# line: the event is rescanned to get the type and line
|
||||
# number of each secret.
|
||||
rescan_payloads[
|
||||
(
|
||||
log_group.arn,
|
||||
log_stream_name,
|
||||
cloudwatch_timestamp,
|
||||
event_index,
|
||||
)
|
||||
] = log_event_data
|
||||
groups_with_rescan.add(log_group.arn)
|
||||
else:
|
||||
events_at_timestamp[event_index]["types"].append(secret["type"])
|
||||
stream_plans[(log_group.arn, log_stream_name)] = plan
|
||||
|
||||
# Phase 3: one batched rescan for all multiline flagged events. Validation
|
||||
# is never enabled here: this rescan only resolves line numbers for
|
||||
# display and must not re-authenticate the secret.
|
||||
# If the rescan fails we know secrets were already found in Phase 1, so
|
||||
# the affected groups must not silently pass; they are reported MANUAL.
|
||||
rescan_scan_error = None
|
||||
rescan_results = {}
|
||||
if rescan_payloads:
|
||||
try:
|
||||
rescan_results = detect_secrets_scan_batch(
|
||||
rescan_payloads, excluded_secrets=secrets_ignore_patterns
|
||||
)
|
||||
log_group_secrets = []
|
||||
if log_group.log_streams:
|
||||
for log_stream_name in log_group.log_streams:
|
||||
log_stream_secrets = {}
|
||||
log_stream_data = "\n".join(
|
||||
[
|
||||
dumps(event["message"])
|
||||
for event in log_group.log_streams[log_stream_name]
|
||||
]
|
||||
)
|
||||
log_stream_secrets_output = detect_secrets_scan(
|
||||
data=log_stream_data,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=logs_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
rescan_scan_error = error
|
||||
|
||||
if log_stream_secrets_output:
|
||||
for secret in log_stream_secrets_output:
|
||||
flagged_event = log_group.log_streams[log_stream_name][
|
||||
secret["line_number"] - 1
|
||||
]
|
||||
cloudwatch_timestamp = (
|
||||
convert_to_cloudwatch_timestamp_format(
|
||||
flagged_event["timestamp"]
|
||||
)
|
||||
)
|
||||
if (
|
||||
cloudwatch_timestamp
|
||||
not in log_stream_secrets.keys()
|
||||
):
|
||||
log_stream_secrets[cloudwatch_timestamp] = (
|
||||
SecretsDict()
|
||||
)
|
||||
# Phase 4: assemble one report per log group.
|
||||
for log_group in logs_client.log_groups.values():
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=log_group)
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in {log_group.name} log group."
|
||||
|
||||
try:
|
||||
log_event_data = dumps(
|
||||
loads(flagged_event["message"]), indent=2
|
||||
)
|
||||
except Exception:
|
||||
log_event_data = dumps(
|
||||
flagged_event["message"], indent=2
|
||||
)
|
||||
if len(log_event_data.split("\n")) > 1:
|
||||
# Can get more informative output if there is more than 1 line.
|
||||
# Will rescan just this event to get the type of secret and the line number
|
||||
event_detect_secrets_output = detect_secrets_scan(
|
||||
data=log_event_data,
|
||||
detect_secrets_plugins=logs_client.audit_config.get(
|
||||
"detect_secrets_plugins"
|
||||
),
|
||||
)
|
||||
if event_detect_secrets_output:
|
||||
for secret in event_detect_secrets_output:
|
||||
log_stream_secrets[
|
||||
cloudwatch_timestamp
|
||||
].add_secret(
|
||||
secret["line_number"], secret["type"]
|
||||
)
|
||||
else:
|
||||
log_stream_secrets[cloudwatch_timestamp].add_secret(
|
||||
1, secret["type"]
|
||||
)
|
||||
if log_stream_secrets:
|
||||
secrets_string = "; ".join(
|
||||
[
|
||||
f"at {timestamp} - {log_stream_secrets[timestamp].to_string()}"
|
||||
for timestamp in log_stream_secrets
|
||||
]
|
||||
)
|
||||
log_group_secrets.append(
|
||||
f"in log stream {log_stream_name} {secrets_string}"
|
||||
)
|
||||
if log_group_secrets:
|
||||
secrets_string = "; ".join(log_group_secrets)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secrets found in log group {log_group.name} {secrets_string}."
|
||||
# The stream scan failed: we cannot conclude this group is clean.
|
||||
if stream_scan_error and log_group.log_streams:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan log group {log_group.name} for secrets: "
|
||||
f"{stream_scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
log_group_secrets = []
|
||||
all_secrets = []
|
||||
for log_stream_name in log_group.log_streams or {}:
|
||||
stream_secrets = stream_results.get((log_group.arn, log_stream_name))
|
||||
if not stream_secrets:
|
||||
continue
|
||||
all_secrets.extend(stream_secrets)
|
||||
log_stream_secrets = {}
|
||||
for cloudwatch_timestamp, events_at_timestamp in stream_plans[
|
||||
(log_group.arn, log_stream_name)
|
||||
].items():
|
||||
secrets_dict = SecretsDict()
|
||||
# Multiple events can share one timestamp; aggregate each
|
||||
# event's secrets into the timestamp's display entry.
|
||||
for event_index, entry in events_at_timestamp.items():
|
||||
if entry["multiline"]:
|
||||
for event_secret in rescan_results.get(
|
||||
(
|
||||
log_group.arn,
|
||||
log_stream_name,
|
||||
cloudwatch_timestamp,
|
||||
event_index,
|
||||
),
|
||||
[],
|
||||
):
|
||||
secrets_dict.add_secret(
|
||||
event_secret["line_number"], event_secret["type"]
|
||||
)
|
||||
else:
|
||||
for secret_type in entry["types"]:
|
||||
secrets_dict.add_secret(1, secret_type)
|
||||
# Only record the event when at least one non-ignored secret
|
||||
# remains after the rescan. A multiline event whose secrets
|
||||
# were all dropped by ``secrets_ignore_patterns`` leaves an
|
||||
# empty SecretsDict, which must not produce a FAIL with no
|
||||
# actual secret evidence.
|
||||
if secrets_dict:
|
||||
log_stream_secrets[cloudwatch_timestamp] = secrets_dict
|
||||
if log_stream_secrets:
|
||||
secrets_string = "; ".join(
|
||||
[
|
||||
f"at {timestamp} - {log_stream_secrets[timestamp].to_string()}"
|
||||
for timestamp in log_stream_secrets
|
||||
]
|
||||
)
|
||||
log_group_secrets.append(
|
||||
f"in log stream {log_stream_name} {secrets_string}"
|
||||
)
|
||||
# The multiline rescan failed for a group that had flagged secrets:
|
||||
# detail is unavailable, so report MANUAL rather than risk a false
|
||||
# PASS when every flagged event was multiline.
|
||||
if rescan_scan_error and log_group.arn in groups_with_rescan:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Secrets were detected in log group {log_group.name} but the "
|
||||
f"detailed rescan failed: {rescan_scan_error}; manual review "
|
||||
"is required."
|
||||
)
|
||||
elif log_group_secrets:
|
||||
secrets_string = "; ".join(log_group_secrets)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secrets found in log group {log_group.name} {secrets_string}."
|
||||
annotate_verified_secrets(report, all_secrets)
|
||||
findings.append(report)
|
||||
return findings
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,10 @@ from botocore.exceptions import ClientError
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.resource_limit import (
|
||||
get_resource_scan_limit,
|
||||
limit_resources,
|
||||
)
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
@@ -83,8 +87,19 @@ class Logs(AWSService):
|
||||
# Call AWSService's __init__
|
||||
super().__init__(__class__.__name__, provider)
|
||||
self.log_group_arn_template = f"arn:{self.audited_partition}:logs:{self.region}:{self.audited_account}:log-group"
|
||||
# Log groups are listed first, then only the selected subset is enriched
|
||||
# and exposed for primary log group checks. Keep a complete lightweight
|
||||
# index for cross-service evidence lookups.
|
||||
self.all_log_groups = {}
|
||||
self.log_groups = {}
|
||||
self._log_groups_hydrated = set()
|
||||
self.log_group_limit = get_resource_scan_limit(
|
||||
self.audit_config, "max_cloudwatch_log_groups"
|
||||
)
|
||||
# The threshold for number of events to return per log group.
|
||||
self.events_per_log_group_threshold = 1000
|
||||
self.__threading_call__(self._describe_log_groups)
|
||||
self._select_log_groups_for_analysis()
|
||||
self.resource_policies = {}
|
||||
self.__threading_call__(self._describe_resource_policies)
|
||||
self.metric_filters = []
|
||||
@@ -94,14 +109,27 @@ class Logs(AWSService):
|
||||
"cloudwatch_log_group_no_secrets_in_logs"
|
||||
in provider.audit_metadata.expected_checks
|
||||
):
|
||||
self.events_per_log_group_threshold = (
|
||||
1000 # The threshold for number of events to return per log group.
|
||||
)
|
||||
self.__threading_call__(self._get_log_events)
|
||||
self.__threading_call__(self._get_log_events, self.log_groups.values())
|
||||
self.__threading_call__(
|
||||
self._list_tags_for_resource, self.log_groups.values()
|
||||
)
|
||||
|
||||
def _select_log_groups_for_analysis(self):
|
||||
"""Select the newest log groups for bounded analysis."""
|
||||
if not self.log_groups:
|
||||
return
|
||||
self.log_groups = {
|
||||
log_group.arn: log_group
|
||||
for log_group in limit_resources(
|
||||
sorted(
|
||||
self.log_groups.values(),
|
||||
key=lambda lg: lg.creation_time or 0,
|
||||
reverse=True,
|
||||
),
|
||||
self.log_group_limit,
|
||||
)
|
||||
}
|
||||
|
||||
def _describe_metric_filters(self, regional_client):
|
||||
logger.info("CloudWatch Logs - Describing metric filters...")
|
||||
try:
|
||||
@@ -118,11 +146,21 @@ class Logs(AWSService):
|
||||
self.metric_filters = []
|
||||
|
||||
log_group = None
|
||||
for lg in self.log_groups.values():
|
||||
if lg.name == filter["logGroupName"]:
|
||||
for lg in (self.all_log_groups or {}).values():
|
||||
if (
|
||||
lg.name == filter["logGroupName"]
|
||||
and lg.region == regional_client.region
|
||||
):
|
||||
log_group = lg
|
||||
break
|
||||
|
||||
if (
|
||||
log_group
|
||||
and log_group.arn in (self.log_groups or {})
|
||||
and log_group.arn not in self._log_groups_hydrated
|
||||
):
|
||||
self._list_tags_for_resource(log_group)
|
||||
|
||||
self.metric_filters.append(
|
||||
MetricFilter(
|
||||
arn=arn,
|
||||
@@ -156,9 +194,9 @@ class Logs(AWSService):
|
||||
"describe_log_groups"
|
||||
)
|
||||
for page in describe_log_groups_paginator.paginate():
|
||||
for log_group in page["logGroups"]:
|
||||
if not self.audit_resources or (
|
||||
is_resource_filtered(log_group["arn"], self.audit_resources)
|
||||
for log_group in page.get("logGroups", []):
|
||||
if not self.audit_resources or is_resource_filtered(
|
||||
log_group["arn"], self.audit_resources
|
||||
):
|
||||
never_expire = False
|
||||
kms = log_group.get("kmsKeyId")
|
||||
@@ -168,20 +206,26 @@ class Logs(AWSService):
|
||||
retention_days = 9999
|
||||
if self.log_groups is None:
|
||||
self.log_groups = {}
|
||||
self.log_groups[log_group["arn"]] = LogGroup(
|
||||
if self.all_log_groups is None:
|
||||
self.all_log_groups = {}
|
||||
log_group_object = LogGroup(
|
||||
arn=log_group["arn"],
|
||||
name=log_group["logGroupName"],
|
||||
retention_days=retention_days,
|
||||
never_expire=never_expire,
|
||||
kms_id=kms,
|
||||
creation_time=log_group.get("creationTime"),
|
||||
region=regional_client.region,
|
||||
)
|
||||
self.all_log_groups[log_group_object.arn] = log_group_object
|
||||
self.log_groups[log_group_object.arn] = log_group_object
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] == "AccessDeniedException":
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
if not self.log_groups:
|
||||
self.all_log_groups = None
|
||||
self.log_groups = None
|
||||
else:
|
||||
logger.error(
|
||||
@@ -192,37 +236,29 @@ class Logs(AWSService):
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _get_log_events(self, regional_client):
|
||||
regional_log_groups = [
|
||||
log_group
|
||||
for log_group in self.log_groups.values()
|
||||
if log_group.region == regional_client.region
|
||||
]
|
||||
total_log_groups = len(regional_log_groups)
|
||||
def _get_log_events(self, log_group):
|
||||
"""Retrieve recent log events for a selected log group.
|
||||
|
||||
Args:
|
||||
log_group: Log group selected for bounded analysis.
|
||||
"""
|
||||
logger.info(
|
||||
f"CloudWatch Logs - Retrieving log events for {total_log_groups} log groups in {regional_client.region}..."
|
||||
f"CloudWatch Logs - Retrieving log events for log group {log_group.name}..."
|
||||
)
|
||||
try:
|
||||
for count, log_group in enumerate(regional_log_groups, start=1):
|
||||
events = regional_client.filter_log_events(
|
||||
logGroupName=log_group.name,
|
||||
limit=self.events_per_log_group_threshold,
|
||||
)["events"]
|
||||
for event in events:
|
||||
if event["logStreamName"] not in log_group.log_streams:
|
||||
log_group.log_streams[event["logStreamName"]] = []
|
||||
log_group.log_streams[event["logStreamName"]].append(event)
|
||||
if count % 10 == 0:
|
||||
logger.info(
|
||||
f"CloudWatch Logs - Retrieved log events for {count}/{total_log_groups} log groups in {regional_client.region}..."
|
||||
)
|
||||
regional_client = self.regional_clients[log_group.region]
|
||||
events = regional_client.filter_log_events(
|
||||
logGroupName=log_group.name,
|
||||
limit=self.events_per_log_group_threshold,
|
||||
)["events"]
|
||||
for event in events:
|
||||
if event["logStreamName"] not in log_group.log_streams:
|
||||
log_group.log_streams[event["logStreamName"]] = []
|
||||
log_group.log_streams[event["logStreamName"]].append(event)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
f"{log_group.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
logger.info(
|
||||
f"CloudWatch Logs - Finished retrieving log events in {regional_client.region}..."
|
||||
)
|
||||
|
||||
def _describe_resource_policies(self, regional_client):
|
||||
logger.info("CloudWatch Logs - Describing resource policies...")
|
||||
@@ -257,6 +293,13 @@ class Logs(AWSService):
|
||||
)
|
||||
|
||||
def _list_tags_for_resource(self, log_group):
|
||||
"""Hydrate tags for a selected log group once.
|
||||
|
||||
Args:
|
||||
log_group: Log group selected for tag hydration.
|
||||
"""
|
||||
if log_group.arn in self._log_groups_hydrated:
|
||||
return
|
||||
logger.info(f"CloudWatch Logs - List Tags for Log Group {log_group.name}...")
|
||||
try:
|
||||
regional_client = self.regional_clients[log_group.region]
|
||||
@@ -264,6 +307,7 @@ class Logs(AWSService):
|
||||
resourceArn=log_group.arn
|
||||
)["tags"]
|
||||
log_group.tags = [response]
|
||||
self._log_groups_hydrated.add(log_group.arn)
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] == "ResourceNotFoundException":
|
||||
logger.warning(
|
||||
@@ -292,6 +336,7 @@ class LogGroup(BaseModel):
|
||||
retention_days: int
|
||||
never_expire: bool
|
||||
kms_id: Optional[str]
|
||||
creation_time: Optional[int] = None
|
||||
region: str
|
||||
log_streams: dict[str, list[str]] = (
|
||||
{}
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
from typing import Iterator, Optional, Tuple
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.resource_limit import (
|
||||
get_resource_scan_limit,
|
||||
iter_limited_paginator_items,
|
||||
)
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
@@ -15,9 +19,18 @@ class CodeArtifact(AWSService):
|
||||
super().__init__(__class__.__name__, provider)
|
||||
# repositories is a dictionary containing all the codeartifact service information
|
||||
self.repositories = {}
|
||||
# repository ARNs whose selected packages have been listed and memoized
|
||||
# into repository.packages.
|
||||
self._packages_listed = set()
|
||||
self.package_limit = get_resource_scan_limit(
|
||||
self.audit_config, "max_codeartifact_packages"
|
||||
)
|
||||
self.__threading_call__(self._list_repositories)
|
||||
self.__threading_call__(self._list_packages)
|
||||
self._list_tags_for_resource()
|
||||
for _ in self._load_packages_for_analysis():
|
||||
pass
|
||||
self.__threading_call__(
|
||||
self._list_tags_for_resource, self.repositories.values()
|
||||
)
|
||||
|
||||
def _list_repositories(self, regional_client):
|
||||
logger.info("CodeArtifact - Listing Repositories...")
|
||||
@@ -51,134 +64,146 @@ class CodeArtifact(AWSService):
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
def _list_packages(self, regional_client):
|
||||
logger.info("CodeArtifact - Listing Packages and retrieving information...")
|
||||
for repository in self.repositories:
|
||||
try:
|
||||
if self.repositories[repository].region == regional_client.region:
|
||||
list_packages_paginator = regional_client.get_paginator(
|
||||
"list_packages"
|
||||
def _iter_repository_packages(
|
||||
self, repository, limit: Optional[int] = None
|
||||
) -> Iterator["Package"]:
|
||||
"""Yield packages for a single repository, hydrating each one lazily.
|
||||
|
||||
Each package requires an extra ``list_package_versions`` call to
|
||||
resolve its latest version, so producing them lazily lets the resource
|
||||
limit stop before extra package version calls.
|
||||
"""
|
||||
regional_client = self.regional_clients[repository.region]
|
||||
try:
|
||||
list_packages_paginator = regional_client.get_paginator("list_packages")
|
||||
list_packages_parameters = {
|
||||
"domain": repository.domain_name,
|
||||
"domainOwner": repository.domain_owner,
|
||||
"repository": repository.name,
|
||||
}
|
||||
for package in iter_limited_paginator_items(
|
||||
list_packages_paginator,
|
||||
"packages",
|
||||
limit,
|
||||
**list_packages_parameters,
|
||||
):
|
||||
# Package information
|
||||
package_format = package["format"]
|
||||
package_namespace = package.get("namespace")
|
||||
package_name = package["package"]
|
||||
package_origin_configuration_restrictions_publish = package[
|
||||
"originConfiguration"
|
||||
]["restrictions"]["publish"]
|
||||
package_origin_configuration_restrictions_upstream = package[
|
||||
"originConfiguration"
|
||||
]["restrictions"]["upstream"]
|
||||
# Get Latest Package Version
|
||||
list_package_versions_parameters = {
|
||||
"domain": repository.domain_name,
|
||||
"domainOwner": repository.domain_owner,
|
||||
"repository": repository.name,
|
||||
"format": package_format,
|
||||
"package": package_name,
|
||||
"sortBy": "PUBLISHED_TIME",
|
||||
"maxResults": 1,
|
||||
}
|
||||
if package_namespace:
|
||||
list_package_versions_parameters["namespace"] = package_namespace
|
||||
latest_version_information = regional_client.list_package_versions(
|
||||
**list_package_versions_parameters
|
||||
)
|
||||
latest_version = ""
|
||||
latest_origin_type = "UNKNOWN"
|
||||
latest_status = "Published"
|
||||
if latest_version_information.get("versions"):
|
||||
latest_version = latest_version_information["versions"][0].get(
|
||||
"version"
|
||||
)
|
||||
list_packages_parameters = {
|
||||
"domain": self.repositories[repository].domain_name,
|
||||
"domainOwner": self.repositories[repository].domain_owner,
|
||||
"repository": self.repositories[repository].name,
|
||||
}
|
||||
packages = []
|
||||
for page in list_packages_paginator.paginate(
|
||||
**list_packages_parameters
|
||||
):
|
||||
for package in page["packages"]:
|
||||
# Package information
|
||||
package_format = package["format"]
|
||||
package_namespace = package.get("namespace")
|
||||
package_name = package["package"]
|
||||
package_origin_configuration_restrictions_publish = package[
|
||||
"originConfiguration"
|
||||
]["restrictions"]["publish"]
|
||||
package_origin_configuration_restrictions_upstream = (
|
||||
package["originConfiguration"]["restrictions"][
|
||||
"upstream"
|
||||
]
|
||||
)
|
||||
# Get Latest Package Version
|
||||
if package_namespace:
|
||||
latest_version_information = (
|
||||
regional_client.list_package_versions(
|
||||
domain=self.repositories[
|
||||
repository
|
||||
].domain_name,
|
||||
domainOwner=self.repositories[
|
||||
repository
|
||||
].domain_owner,
|
||||
repository=self.repositories[repository].name,
|
||||
format=package_format,
|
||||
namespace=package_namespace,
|
||||
package=package_name,
|
||||
sortBy="PUBLISHED_TIME",
|
||||
maxResults=1,
|
||||
)
|
||||
)
|
||||
else:
|
||||
latest_version_information = (
|
||||
regional_client.list_package_versions(
|
||||
domain=self.repositories[
|
||||
repository
|
||||
].domain_name,
|
||||
domainOwner=self.repositories[
|
||||
repository
|
||||
].domain_owner,
|
||||
repository=self.repositories[repository].name,
|
||||
format=package_format,
|
||||
package=package_name,
|
||||
sortBy="PUBLISHED_TIME",
|
||||
maxResults=1,
|
||||
)
|
||||
)
|
||||
latest_version = ""
|
||||
latest_origin_type = "UNKNOWN"
|
||||
latest_status = "Published"
|
||||
if latest_version_information.get("versions"):
|
||||
latest_version = latest_version_information["versions"][
|
||||
0
|
||||
].get("version")
|
||||
latest_origin_type = (
|
||||
latest_version_information["versions"][0]
|
||||
.get("origin", {})
|
||||
.get("originType", "UNKNOWN")
|
||||
)
|
||||
latest_status = latest_version_information["versions"][
|
||||
0
|
||||
].get("status", "Published")
|
||||
|
||||
packages.append(
|
||||
Package(
|
||||
name=package_name,
|
||||
namespace=package_namespace,
|
||||
format=package_format,
|
||||
origin_configuration=OriginConfiguration(
|
||||
restrictions=Restrictions(
|
||||
publish=package_origin_configuration_restrictions_publish,
|
||||
upstream=package_origin_configuration_restrictions_upstream,
|
||||
)
|
||||
),
|
||||
latest_version=LatestPackageVersion(
|
||||
version=latest_version,
|
||||
status=latest_status,
|
||||
origin=OriginInformation(
|
||||
origin_type=latest_origin_type
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
# Save all the packages information
|
||||
self.repositories[repository].packages = packages
|
||||
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] == "ResourceNotFoundException":
|
||||
logger.warning(
|
||||
f"{regional_client.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
latest_origin_type = (
|
||||
latest_version_information["versions"][0]
|
||||
.get("origin", {})
|
||||
.get("originType", "UNKNOWN")
|
||||
)
|
||||
latest_status = latest_version_information["versions"][0].get(
|
||||
"status", "Published"
|
||||
)
|
||||
continue
|
||||
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
yield Package(
|
||||
name=package_name,
|
||||
namespace=package_namespace,
|
||||
format=package_format,
|
||||
origin_configuration=OriginConfiguration(
|
||||
restrictions=Restrictions(
|
||||
publish=package_origin_configuration_restrictions_publish,
|
||||
upstream=package_origin_configuration_restrictions_upstream,
|
||||
)
|
||||
),
|
||||
latest_version=LatestPackageVersion(
|
||||
version=latest_version,
|
||||
status=latest_status,
|
||||
origin=OriginInformation(origin_type=latest_origin_type),
|
||||
),
|
||||
)
|
||||
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] == "ResourceNotFoundException":
|
||||
logger.warning(
|
||||
f"{repository.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"{repository.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{repository.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
def _list_tags_for_resource(self):
|
||||
def _load_packages_for_analysis(self) -> Iterator[Tuple["Repository", "Package"]]:
|
||||
"""Yield the ``(repository, package)`` pairs selected for analysis.
|
||||
|
||||
Package listing stays in the service layer so checks receive only the
|
||||
selected packages and remain unaware of resource-analysis limits.
|
||||
"""
|
||||
yielded = 0
|
||||
for repository in list(self.repositories.values()):
|
||||
if repository.arn in self._packages_listed:
|
||||
for package in repository.packages:
|
||||
yield repository, package
|
||||
yielded += 1
|
||||
if self.package_limit and yielded >= self.package_limit:
|
||||
return
|
||||
continue
|
||||
collected = []
|
||||
remaining_limit = None
|
||||
if self.package_limit:
|
||||
remaining_limit = self.package_limit - yielded
|
||||
if remaining_limit <= 0:
|
||||
return
|
||||
for package in self._iter_repository_packages(repository, remaining_limit):
|
||||
collected.append(package)
|
||||
repository.packages = collected
|
||||
yield repository, package
|
||||
yielded += 1
|
||||
if self.package_limit and yielded >= self.package_limit:
|
||||
self._packages_listed.add(repository.arn)
|
||||
return
|
||||
self._packages_listed.add(repository.arn)
|
||||
|
||||
def _list_tags_for_resource(self, repository):
|
||||
logger.info("CodeArtifact - List Tags...")
|
||||
try:
|
||||
for repository in self.repositories.values():
|
||||
regional_client = self.regional_clients[repository.region]
|
||||
response = regional_client.list_tags_for_resource(
|
||||
resourceArn=repository.arn
|
||||
)["tags"]
|
||||
repository.tags = response
|
||||
regional_client = self.regional_clients[repository.region]
|
||||
response = regional_client.list_tags_for_resource(
|
||||
resourceArn=repository.arn
|
||||
)["tags"]
|
||||
repository.tags = response
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
|
||||
+57
-17
@@ -1,7 +1,11 @@
|
||||
import json
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.codebuild.codebuild_client import codebuild_client
|
||||
|
||||
|
||||
@@ -14,35 +18,71 @@ class codebuild_project_no_secrets_in_variables(Check):
|
||||
secrets_ignore_patterns = codebuild_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for project in codebuild_client.projects.values():
|
||||
validate = codebuild_client.audit_config.get("secrets_validate", False)
|
||||
projects = list(codebuild_client.projects.values())
|
||||
|
||||
# Collect every scannable plaintext variable across all projects and scan
|
||||
# them in batched Kingfisher invocations instead of one subprocess per
|
||||
# variable. Findings are keyed by (project index, variable index).
|
||||
def payloads():
|
||||
for project_index, project in enumerate(projects):
|
||||
if project.environment_variables:
|
||||
for var_index, env_var in enumerate(project.environment_variables):
|
||||
if (
|
||||
env_var.type == "PLAINTEXT"
|
||||
and env_var.name not in sensitive_vars_excluded
|
||||
):
|
||||
yield (project_index, var_index), json.dumps(
|
||||
{env_var.name: env_var.value}
|
||||
)
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for project_index, project in enumerate(projects):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=project)
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"CodeBuild project {project.name} does not have sensitive environment plaintext credentials."
|
||||
secrets_found = []
|
||||
all_secrets = []
|
||||
|
||||
if scan_error and any(
|
||||
env_var.type == "PLAINTEXT"
|
||||
and env_var.name not in sensitive_vars_excluded
|
||||
for env_var in project.environment_variables or []
|
||||
):
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan CodeBuild project {project.name} environment "
|
||||
f"variables for secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
if project.environment_variables:
|
||||
for env_var in project.environment_variables:
|
||||
if (
|
||||
env_var.type == "PLAINTEXT"
|
||||
and env_var.name not in sensitive_vars_excluded
|
||||
):
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=json.dumps({env_var.name: env_var.value}),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=codebuild_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
if detect_secrets_output:
|
||||
secrets_info = [
|
||||
for var_index, env_var in enumerate(project.environment_variables):
|
||||
detect_secrets_output = batch_results.get(
|
||||
(project_index, var_index)
|
||||
)
|
||||
if detect_secrets_output:
|
||||
all_secrets.extend(detect_secrets_output)
|
||||
secrets_found.extend(
|
||||
[
|
||||
f"{secret['type']} in variable {env_var.name}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
secrets_found.extend(secrets_info)
|
||||
)
|
||||
|
||||
if secrets_found:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"CodeBuild project {project.name} has sensitive environment plaintext credentials in variables: {', '.join(secrets_found)}."
|
||||
annotate_verified_secrets(report, all_secrets)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
|
||||
+81
-45
@@ -4,7 +4,11 @@ from base64 import b64decode
|
||||
from prowler.config.config import encoding_format_utf_8
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.ec2.ec2_client import ec2_client
|
||||
|
||||
|
||||
@@ -14,54 +18,86 @@ class ec2_instance_secrets_user_data(Check):
|
||||
secrets_ignore_patterns = ec2_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for instance in ec2_client.instances:
|
||||
if instance.state != "terminated":
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=instance)
|
||||
if instance.user_data:
|
||||
user_data = b64decode(instance.user_data)
|
||||
try:
|
||||
if user_data[0:2] == b"\x1f\x8b": # GZIP magic number
|
||||
user_data = zlib.decompress(
|
||||
user_data, zlib.MAX_WBITS | 32
|
||||
).decode(encoding_format_utf_8)
|
||||
else:
|
||||
user_data = user_data.decode(encoding_format_utf_8)
|
||||
except UnicodeDecodeError as error:
|
||||
logger.warning(
|
||||
f"{instance.region} -- Unable to decode user data in EC2 instance {instance.id}: {error}"
|
||||
)
|
||||
continue
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{instance.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
continue
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=user_data,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=ec2_client.audit_config.get(
|
||||
"detect_secrets_plugins"
|
||||
),
|
||||
)
|
||||
if detect_secrets_output:
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} on line {secret['line_number']}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in EC2 instance {instance.id} User Data -> {secrets_string}."
|
||||
validate = ec2_client.audit_config.get("secrets_validate", False)
|
||||
instances = list(ec2_client.instances)
|
||||
|
||||
# Collect the decoded User Data of each non-terminated instance and scan
|
||||
# it all in batched Kingfisher invocations instead of one subprocess each.
|
||||
# Instances whose User Data cannot be decoded are undecodable (no report),
|
||||
# matching the original per-resource behavior.
|
||||
undecodable = set()
|
||||
|
||||
def payloads():
|
||||
for index, instance in enumerate(instances):
|
||||
if instance.state == "terminated" or not instance.user_data:
|
||||
continue
|
||||
user_data = b64decode(instance.user_data)
|
||||
try:
|
||||
if user_data[0:2] == b"\x1f\x8b": # GZIP magic number
|
||||
user_data = zlib.decompress(
|
||||
user_data, zlib.MAX_WBITS | 32
|
||||
).decode(encoding_format_utf_8)
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in EC2 instance {instance.id} User Data."
|
||||
)
|
||||
user_data = user_data.decode(encoding_format_utf_8)
|
||||
except UnicodeDecodeError as error:
|
||||
logger.warning(
|
||||
f"{instance.region} -- Unable to decode user data in EC2 instance {instance.id}: {error}"
|
||||
)
|
||||
undecodable.add(index)
|
||||
continue
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{instance.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
undecodable.add(index)
|
||||
continue
|
||||
yield index, user_data
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for index, instance in enumerate(instances):
|
||||
if instance.state == "terminated":
|
||||
continue
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=instance)
|
||||
if scan_error and instance.user_data:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan EC2 instance {instance.id} User Data for "
|
||||
f"secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
if index in undecodable:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = f"Could not decode User Data for EC2 instance {instance.id}; manual review is required to scan for secrets."
|
||||
elif instance.user_data:
|
||||
detect_secrets_output = batch_results.get(index)
|
||||
if detect_secrets_output:
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} on line {secret['line_number']}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in EC2 instance {instance.id} User Data -> {secrets_string}."
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in EC2 instance {instance.id} since User Data is empty."
|
||||
report.status_extended = (
|
||||
f"No secrets found in EC2 instance {instance.id} User Data."
|
||||
)
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in EC2 instance {instance.id} since User Data is empty."
|
||||
|
||||
findings.append(report)
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
|
||||
+76
-33
@@ -4,7 +4,11 @@ from base64 import b64decode
|
||||
from prowler.config.config import encoding_format_utf_8
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.ec2.ec2_client import ec2_client
|
||||
|
||||
|
||||
@@ -14,43 +18,77 @@ class ec2_launch_template_no_secrets(Check):
|
||||
secrets_ignore_patterns = ec2_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for template in ec2_client.launch_templates:
|
||||
validate = ec2_client.audit_config.get("secrets_validate", False)
|
||||
templates = list(ec2_client.launch_templates)
|
||||
|
||||
# Track versions whose User Data cannot be decoded so the template is
|
||||
# surfaced (MANUAL) instead of silently claiming no secrets were found.
|
||||
undecodable_versions = {}
|
||||
|
||||
# Collect the decoded User Data of every (template, version) and scan it
|
||||
# all in batched Kingfisher invocations instead of one subprocess per
|
||||
# version. Versions whose User Data cannot be decoded are recorded above.
|
||||
def payloads():
|
||||
for template_index, template in enumerate(templates):
|
||||
for version_index, version in enumerate(template.versions):
|
||||
if not version.template_data.user_data:
|
||||
continue
|
||||
user_data = b64decode(version.template_data.user_data)
|
||||
try:
|
||||
if user_data[0:2] == b"\x1f\x8b": # GZIP magic number
|
||||
user_data = zlib.decompress(
|
||||
user_data, zlib.MAX_WBITS | 32
|
||||
).decode(encoding_format_utf_8)
|
||||
else:
|
||||
user_data = user_data.decode(encoding_format_utf_8)
|
||||
except UnicodeDecodeError as error:
|
||||
logger.warning(
|
||||
f"{template.region} -- Unable to decode User Data in EC2 Launch Template {template.name} version {version.version_number}: {error}"
|
||||
)
|
||||
undecodable_versions.setdefault(template_index, []).append(
|
||||
version.version_number
|
||||
)
|
||||
continue
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{template.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
undecodable_versions.setdefault(template_index, []).append(
|
||||
version.version_number
|
||||
)
|
||||
continue
|
||||
yield (template_index, version_index), user_data
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for template_index, template in enumerate(templates):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=template)
|
||||
|
||||
versions_with_secrets = []
|
||||
|
||||
for version in template.versions:
|
||||
if not version.template_data.user_data:
|
||||
continue
|
||||
user_data = b64decode(version.template_data.user_data)
|
||||
|
||||
try:
|
||||
if user_data[0:2] == b"\x1f\x8b": # GZIP magic number
|
||||
user_data = zlib.decompress(
|
||||
user_data, zlib.MAX_WBITS | 32
|
||||
).decode(encoding_format_utf_8)
|
||||
else:
|
||||
user_data = user_data.decode(encoding_format_utf_8)
|
||||
except UnicodeDecodeError as error:
|
||||
logger.warning(
|
||||
f"{template.region} -- Unable to decode User Data in EC2 Launch Template {template.name} version {version.version_number}: {error}"
|
||||
)
|
||||
continue
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{template.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
continue
|
||||
|
||||
version_secrets = detect_secrets_scan(
|
||||
data=user_data,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=ec2_client.audit_config.get(
|
||||
"detect_secrets_plugins"
|
||||
),
|
||||
if scan_error and any(
|
||||
version.template_data.user_data for version in template.versions
|
||||
):
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan EC2 Launch Template {template.name} User Data "
|
||||
f"for secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
versions_with_secrets = []
|
||||
all_secrets = []
|
||||
|
||||
for version_index, version in enumerate(template.versions):
|
||||
version_secrets = batch_results.get((template_index, version_index))
|
||||
if version_secrets:
|
||||
all_secrets.extend(version_secrets)
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} on line {secret['line_number']}"
|
||||
@@ -61,9 +99,14 @@ class ec2_launch_template_no_secrets(Check):
|
||||
f"Version {version.version_number}: {secrets_string}"
|
||||
)
|
||||
|
||||
undecodable = undecodable_versions.get(template_index, [])
|
||||
if len(versions_with_secrets) > 0:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in User Data for EC2 Launch Template {template.name} in template versions: {', '.join(versions_with_secrets)}."
|
||||
annotate_verified_secrets(report, all_secrets)
|
||||
elif undecodable:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = f"Could not decode User Data for EC2 Launch Template {template.name} versions: {', '.join(str(version_number) for version_number in undecodable)}; manual review is required to scan for secrets."
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in User Data of any version for EC2 Launch Template {template.name}."
|
||||
|
||||
+3
-4
@@ -15,11 +15,10 @@ class ec2_securitygroup_not_used(Check):
|
||||
report.resource_details = security_group.name
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"Security group {security_group.name} ({security_group.id}) it is being used."
|
||||
sg_in_lambda = False
|
||||
sg_in_lambda = (
|
||||
security_group.id in awslambda_client.security_groups_in_use
|
||||
)
|
||||
sg_associated = False
|
||||
for function in awslambda_client.functions.values():
|
||||
if security_group.id in function.security_groups:
|
||||
sg_in_lambda = True
|
||||
for sg in ec2_client.security_groups.values():
|
||||
if security_group.id in sg.associated_sgs:
|
||||
sg_associated = True
|
||||
|
||||
@@ -6,6 +6,10 @@ from botocore.client import ClientError
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.resource_limit import (
|
||||
get_resource_scan_limit,
|
||||
limit_resources,
|
||||
)
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
@@ -26,8 +30,12 @@ class EC2(AWSService):
|
||||
self.snapshots = []
|
||||
self.volumes_with_snapshots = {}
|
||||
self.regions_with_snapshots = {}
|
||||
# Snapshots are listed first, then limited after per-region snapshot
|
||||
# presence is derived and before public status is hydrated.
|
||||
self.snapshot_limit = get_resource_scan_limit(
|
||||
self.audit_config, "max_ebs_snapshots"
|
||||
)
|
||||
self.__threading_call__(self._describe_snapshots)
|
||||
self.__threading_call__(self._determine_public_snapshots, self.snapshots)
|
||||
self.network_interfaces = {}
|
||||
self.__threading_call__(self._describe_network_interfaces)
|
||||
self.images = []
|
||||
@@ -36,6 +44,8 @@ class EC2(AWSService):
|
||||
self.__threading_call__(self._describe_volumes)
|
||||
self.attributes_for_regions = {}
|
||||
self.__threading_call__(self._get_resources_for_regions)
|
||||
self._select_snapshots_for_analysis()
|
||||
self.__threading_call__(self._determine_public_snapshots, self.snapshots)
|
||||
self.ebs_encryption_by_default = []
|
||||
self.__threading_call__(self._get_ebs_encryption_settings)
|
||||
self.elastic_ips = []
|
||||
@@ -207,6 +217,7 @@ class EC2(AWSService):
|
||||
arn=arn,
|
||||
region=regional_client.region,
|
||||
encrypted=snapshot.get("Encrypted", False),
|
||||
start_time=snapshot.get("StartTime"),
|
||||
tags=snapshot.get("Tags"),
|
||||
volume=snapshot["VolumeId"],
|
||||
)
|
||||
@@ -243,6 +254,18 @@ class EC2(AWSService):
|
||||
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _select_snapshots_for_analysis(self):
|
||||
self.snapshots = list(
|
||||
limit_resources(
|
||||
sorted(
|
||||
self.snapshots,
|
||||
key=lambda s: (s.start_time.timestamp() if s.start_time else 0.0),
|
||||
reverse=True,
|
||||
),
|
||||
self.snapshot_limit,
|
||||
)
|
||||
)
|
||||
|
||||
def _describe_network_interfaces(self, regional_client):
|
||||
try:
|
||||
# Get Network Interfaces with Public IPs
|
||||
@@ -686,6 +709,7 @@ class Snapshot(BaseModel):
|
||||
region: str
|
||||
encrypted: bool
|
||||
public: bool = False
|
||||
start_time: Optional[datetime] = None
|
||||
tags: Optional[list] = []
|
||||
volume: Optional[str]
|
||||
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
from datetime import datetime
|
||||
from itertools import zip_longest
|
||||
from re import sub
|
||||
from typing import Optional
|
||||
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from prowler.lib.resource_limit import (
|
||||
get_resource_scan_limit,
|
||||
iter_limited_paginator_items,
|
||||
limit_resources,
|
||||
)
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
@@ -12,40 +19,95 @@ class ECS(AWSService):
|
||||
def __init__(self, provider):
|
||||
# Call AWSService's __init__
|
||||
super().__init__(__class__.__name__, provider)
|
||||
# Task definition ARNs are listed first, then only the selected subset
|
||||
# is described and exposed for checks.
|
||||
self.task_definitions = {}
|
||||
self._task_definition_arns = None
|
||||
self._task_definition_arns_by_region = {}
|
||||
self.task_definition_limit = get_resource_scan_limit(
|
||||
self.audit_config, "max_ecs_task_definitions"
|
||||
)
|
||||
self.services = {}
|
||||
self.clusters = {}
|
||||
self.task_sets = {}
|
||||
self.__threading_call__(self._list_task_definitions)
|
||||
self.__threading_call__(
|
||||
self._describe_task_definition, self.task_definitions.values()
|
||||
)
|
||||
for _ in self._load_task_definitions_for_analysis():
|
||||
pass
|
||||
self.__threading_call__(self._list_clusters)
|
||||
self.__threading_call__(self._describe_clusters, self.clusters.values())
|
||||
self.__threading_call__(self._describe_services, self.clusters.values())
|
||||
|
||||
def _list_task_definitions(self, regional_client):
|
||||
def _list_task_definition_arns(self) -> list:
|
||||
"""List task definition ARNs newest-first, memoized.
|
||||
|
||||
AWS returns ``list_task_definitions(sort=DESC)`` results per region.
|
||||
Prowler limits the task definitions it describes and exposes to checks.
|
||||
"""
|
||||
if self._task_definition_arns is not None:
|
||||
return self._task_definition_arns
|
||||
logger.info("ECS - Listing Task Definitions...")
|
||||
self.__threading_call__(self._list_task_definition_arns_by_region)
|
||||
arns_by_region = []
|
||||
for region in self.regional_clients:
|
||||
arns_by_region.append(self._task_definition_arns_by_region.get(region, []))
|
||||
arns = []
|
||||
for task_definition_batch in zip_longest(*arns_by_region):
|
||||
for task_definition in task_definition_batch:
|
||||
if task_definition:
|
||||
arns.append(task_definition)
|
||||
self._task_definition_arns = arns
|
||||
return arns
|
||||
|
||||
def _list_task_definition_arns_by_region(self, regional_client):
|
||||
try:
|
||||
list_ecs_paginator = regional_client.get_paginator("list_task_definitions")
|
||||
for page in list_ecs_paginator.paginate():
|
||||
for task_definition in page["taskDefinitionArns"]:
|
||||
if not self.audit_resources or (
|
||||
is_resource_filtered(task_definition, self.audit_resources)
|
||||
):
|
||||
self.task_definitions[task_definition] = TaskDefinition(
|
||||
# we want the family name without the revision
|
||||
name=sub(":.*", "", task_definition.split("/")[-1]),
|
||||
arn=task_definition,
|
||||
revision=task_definition.split(":")[-1],
|
||||
region=regional_client.region,
|
||||
environment_variables=[],
|
||||
)
|
||||
regional_arns = []
|
||||
for task_definition in iter_limited_paginator_items(
|
||||
list_ecs_paginator,
|
||||
"taskDefinitionArns",
|
||||
None,
|
||||
item_filter=lambda task_definition: not self.audit_resources
|
||||
or is_resource_filtered(task_definition, self.audit_resources),
|
||||
sort="DESC",
|
||||
):
|
||||
regional_arns.append((task_definition, regional_client.region))
|
||||
self._task_definition_arns_by_region[regional_client.region] = regional_arns
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _load_task_definitions_for_analysis(self):
|
||||
"""Yield task definitions lazily, describing each one on demand.
|
||||
|
||||
Resources already fetched are memoized in ``self.task_definitions`` and
|
||||
reused across checks (checks run sequentially, so no locking is needed).
|
||||
The configured resource limit bounds ``describe_task_definition`` calls.
|
||||
"""
|
||||
task_definitions = []
|
||||
for arn, region in limit_resources(
|
||||
self._list_task_definition_arns(), self.task_definition_limit
|
||||
):
|
||||
task_definition = self.task_definitions.get(arn)
|
||||
if task_definition is None:
|
||||
task_definition = TaskDefinition(
|
||||
# we want the family name without the revision
|
||||
name=sub(":.*", "", arn.split("/")[-1]),
|
||||
arn=arn,
|
||||
revision=arn.split(":")[-1],
|
||||
region=region,
|
||||
environment_variables=[],
|
||||
)
|
||||
self.task_definitions[arn] = task_definition
|
||||
task_definitions.append(task_definition)
|
||||
|
||||
self.__threading_call__(self._describe_task_definition, task_definitions)
|
||||
|
||||
for arn, _ in limit_resources(
|
||||
self._list_task_definition_arns(), self.task_definition_limit
|
||||
):
|
||||
task_definition = self.task_definitions[arn]
|
||||
yield task_definition
|
||||
|
||||
def _describe_task_definition(self, task_definition):
|
||||
logger.info("ECS - Describing Task Definition...")
|
||||
try:
|
||||
@@ -84,6 +146,9 @@ class ECS(AWSService):
|
||||
)
|
||||
)
|
||||
task_definition.pid_mode = response["taskDefinition"].get("pidMode", "")
|
||||
task_definition.registered_at = response["taskDefinition"].get(
|
||||
"registeredAt"
|
||||
)
|
||||
task_definition.tags = response.get("tags")
|
||||
task_definition.network_mode = response["taskDefinition"].get(
|
||||
"networkMode", "bridge"
|
||||
@@ -208,6 +273,7 @@ class TaskDefinition(BaseModel):
|
||||
region: str
|
||||
container_definitions: list[ContainerDefinition] = []
|
||||
pid_mode: Optional[str]
|
||||
registered_at: Optional[datetime] = None
|
||||
tags: Optional[list] = []
|
||||
network_mode: Optional[str]
|
||||
|
||||
|
||||
+56
-17
@@ -1,7 +1,11 @@
|
||||
from json import dumps
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.ecs.ecs_client import ecs_client
|
||||
|
||||
|
||||
@@ -11,33 +15,67 @@ class ecs_task_definitions_no_environment_secrets(Check):
|
||||
secrets_ignore_patterns = ecs_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for task_definition in ecs_client.task_definitions.values():
|
||||
validate = ecs_client.audit_config.get("secrets_validate", False)
|
||||
task_definitions = list(ecs_client.task_definitions.values())
|
||||
|
||||
# Scan every (task definition, container) environment in batched
|
||||
# Kingfisher invocations instead of one subprocess per container.
|
||||
# Payloads are yielded lazily so only a chunk is held/written at a time.
|
||||
def environment_payloads():
|
||||
for td_index, task_definition in enumerate(task_definitions):
|
||||
for c_index, container in enumerate(
|
||||
task_definition.container_definitions
|
||||
):
|
||||
if container.environment:
|
||||
dump_env_vars = {
|
||||
env_var.name: env_var.value
|
||||
for env_var in container.environment
|
||||
}
|
||||
yield (td_index, c_index), dumps(dump_env_vars, indent=2)
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
environment_payloads(),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
validate=validate,
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for td_index, task_definition in enumerate(task_definitions):
|
||||
report = Check_Report_AWS(
|
||||
metadata=self.metadata(), resource=task_definition
|
||||
)
|
||||
report.resource_id = f"{task_definition.name}:{task_definition.revision}"
|
||||
report.status = "PASS"
|
||||
extended_status_parts = []
|
||||
all_secrets = []
|
||||
|
||||
for container in task_definition.container_definitions:
|
||||
if scan_error and any(
|
||||
container.environment
|
||||
for container in task_definition.container_definitions
|
||||
):
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan ECS task definition {task_definition.name} with "
|
||||
f"revision {task_definition.revision} for secrets: {scan_error}; "
|
||||
"manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
for c_index, container in enumerate(task_definition.container_definitions):
|
||||
container_secrets_found = []
|
||||
|
||||
if container.environment:
|
||||
dump_env_vars = {}
|
||||
original_env_vars = []
|
||||
for env_var in container.environment:
|
||||
dump_env_vars.update({env_var.name: env_var.value})
|
||||
original_env_vars.append(env_var.name)
|
||||
|
||||
env_data = dumps(dump_env_vars, indent=2)
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=env_data,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=ecs_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
original_env_vars = [
|
||||
env_var.name for env_var in container.environment
|
||||
]
|
||||
detect_secrets_output = batch_results.get((td_index, c_index))
|
||||
if detect_secrets_output:
|
||||
all_secrets.extend(detect_secrets_output)
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
f"{secret['type']} on the environment variable {original_env_vars[secret['line_number'] - 2]}"
|
||||
@@ -56,6 +94,7 @@ class ecs_task_definitions_no_environment_secrets(Check):
|
||||
+ "; ".join(extended_status_parts)
|
||||
+ "."
|
||||
)
|
||||
annotate_verified_secrets(report, all_secrets)
|
||||
else:
|
||||
report.status_extended = f"No secrets found in variables of ECS task definition {task_definition.name} with revision {task_definition.revision}."
|
||||
findings.append(report)
|
||||
|
||||
+83
-52
@@ -1,52 +1,83 @@
|
||||
import json
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.providers.aws.services.glue.glue_client import glue_client
|
||||
|
||||
|
||||
class glue_etl_jobs_no_secrets_in_arguments(Check):
|
||||
"""Check if Glue ETL jobs have secrets in their default arguments.
|
||||
|
||||
Scans the DefaultArguments of each Glue job for hardcoded credentials,
|
||||
tokens, passwords, and other sensitive values that should be stored in
|
||||
Secrets Manager or Parameter Store instead.
|
||||
"""
|
||||
|
||||
def execute(self):
|
||||
findings = []
|
||||
secrets_ignore_patterns = glue_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for job in glue_client.jobs:
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=job)
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in Glue job {job.name} default arguments."
|
||||
)
|
||||
|
||||
if job.arguments:
|
||||
secrets_found = []
|
||||
for arg_name, arg_value in job.arguments.items():
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=json.dumps({arg_name: arg_value}),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=glue_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
if detect_secrets_output:
|
||||
secrets_found.extend(
|
||||
[
|
||||
f"{secret['type']} in argument {arg_name}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
)
|
||||
|
||||
if secrets_found:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secrets found in Glue job {job.name} default arguments: {', '.join(secrets_found)}."
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
import json
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.glue.glue_client import glue_client
|
||||
|
||||
|
||||
class glue_etl_jobs_no_secrets_in_arguments(Check):
|
||||
"""Check if Glue ETL jobs have secrets in their default arguments.
|
||||
|
||||
Scans the DefaultArguments of each Glue job for hardcoded credentials,
|
||||
tokens, passwords, and other sensitive values that should be stored in
|
||||
Secrets Manager or Parameter Store instead.
|
||||
"""
|
||||
|
||||
def execute(self):
|
||||
findings = []
|
||||
secrets_ignore_patterns = glue_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
validate = glue_client.audit_config.get("secrets_validate", False)
|
||||
jobs = list(glue_client.jobs)
|
||||
|
||||
# Collect every default argument across all jobs and scan them in batched
|
||||
# Kingfisher invocations instead of one subprocess per argument. Findings
|
||||
# are keyed by (job index, argument name).
|
||||
def payloads():
|
||||
for job_index, job in enumerate(jobs):
|
||||
if job.arguments:
|
||||
for arg_name, arg_value in job.arguments.items():
|
||||
yield (job_index, arg_name), json.dumps({arg_name: arg_value})
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for job_index, job in enumerate(jobs):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=job)
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No secrets found in Glue job {job.name} default arguments."
|
||||
)
|
||||
|
||||
if job.arguments and scan_error:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan Glue job {job.name} default arguments for "
|
||||
f"secrets: {scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
|
||||
if job.arguments:
|
||||
secrets_found = []
|
||||
all_secrets = []
|
||||
for arg_name in job.arguments:
|
||||
detect_secrets_output = batch_results.get((job_index, arg_name))
|
||||
if detect_secrets_output:
|
||||
all_secrets.extend(detect_secrets_output)
|
||||
secrets_found.extend(
|
||||
[
|
||||
f"{secret['type']} in argument {arg_name}"
|
||||
for secret in detect_secrets_output
|
||||
]
|
||||
)
|
||||
|
||||
if secrets_found:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secrets found in Glue job {job.name} default arguments: {', '.join(secrets_found)}."
|
||||
annotate_verified_secrets(report, all_secrets)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
|
||||
@@ -19,6 +19,7 @@ from prowler.providers.aws.services.iam.lib.policy import get_effective_actions
|
||||
# - https://github.com/RhinoSecurityLabs/Security-Research/blob/master/tools/aws-pentest-tools/aws_escalate.py
|
||||
# - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation/
|
||||
# - https://github.com/DataDog/pathfinding.cloud (AWS IAM Privilege Escalation Path Library)
|
||||
# - https://www.beyondtrust.com/blog/entry/aws-agentcore-privilege-escalation (AWS Bedrock AgentCore)
|
||||
|
||||
privilege_escalation_policies_combination = {
|
||||
# IAM self-escalation and policy manipulation
|
||||
@@ -299,6 +300,7 @@ privilege_escalation_policies_combination = {
|
||||
"PassRole+AgentCoreCreateInterpreter+InvokeInterpreter": {
|
||||
"iam:PassRole",
|
||||
"bedrock-agentcore:CreateCodeInterpreter",
|
||||
"bedrock-agentcore:StartCodeInterpreterSession",
|
||||
"bedrock-agentcore:InvokeCodeInterpreter",
|
||||
},
|
||||
# Prerequisite: Existing Bedrock code interpreter with admin role
|
||||
@@ -306,6 +308,40 @@ privilege_escalation_policies_combination = {
|
||||
"bedrock-agentcore:StartCodeInterpreterSession",
|
||||
"bedrock-agentcore:InvokeCodeInterpreter",
|
||||
},
|
||||
# Prerequisite: Existing AgentCore Runtime or Harness with admin execution role.
|
||||
# InvokeAgentRuntimeCommand runs shell commands as root inside the microVM and
|
||||
# reads the execution role credentials from MMDS, bypassing the agent and guardrails.
|
||||
"AgentCoreInvokeRuntimeCommand": {
|
||||
"bedrock-agentcore:InvokeAgentRuntimeCommand",
|
||||
},
|
||||
"PassRole+AgentCoreCreateRuntime+InvokeRuntimeCommand": {
|
||||
"iam:PassRole",
|
||||
"bedrock-agentcore:CreateAgentRuntime",
|
||||
"bedrock-agentcore:CreateAgentRuntimeEndpoint",
|
||||
"bedrock-agentcore:CreateWorkloadIdentity",
|
||||
"bedrock-agentcore:InvokeAgentRuntimeCommand",
|
||||
},
|
||||
"PassRole+AgentCoreCreateHarness+InvokeRuntimeCommand": {
|
||||
"iam:PassRole",
|
||||
"bedrock-agentcore:CreateHarness",
|
||||
"bedrock-agentcore:CreateAgentRuntime",
|
||||
"bedrock-agentcore:CreateAgentRuntimeEndpoint",
|
||||
"bedrock-agentcore:CreateWorkloadIdentity",
|
||||
"bedrock-agentcore:GetAgentRuntime",
|
||||
"bedrock-agentcore:InvokeAgentRuntimeCommand",
|
||||
},
|
||||
# Prerequisite: Existing AgentCore Custom Browser with admin execution role.
|
||||
# A remote CDP driver on the browser session reads the role credentials from MMDS.
|
||||
"AgentCoreBrowserSessionConnect": {
|
||||
"bedrock-agentcore:StartBrowserSession",
|
||||
"bedrock-agentcore:ConnectBrowserAutomationStream",
|
||||
},
|
||||
"PassRole+AgentCoreCreateBrowser+ConnectBrowser": {
|
||||
"iam:PassRole",
|
||||
"bedrock-agentcore:CreateBrowser",
|
||||
"bedrock-agentcore:StartBrowserSession",
|
||||
"bedrock-agentcore:ConnectBrowserAutomationStream",
|
||||
},
|
||||
# TO-DO: We have to handle AssumeRole just if the resource is * and without conditions
|
||||
# "sts:AssumeRole": {"sts:AssumeRole"},
|
||||
}
|
||||
|
||||
+5
-6
@@ -15,11 +15,10 @@ class inspector2_is_enabled(Check):
|
||||
if inspector.status == "ENABLED":
|
||||
report.status = "PASS"
|
||||
report.status_extended = "Inspector2 is enabled for EC2 instances, ECR container images, Lambda functions and code."
|
||||
funtions_in_region = False
|
||||
functions_in_region = (
|
||||
inspector.region in awslambda_client.regions_with_functions
|
||||
)
|
||||
ec2_in_region = False
|
||||
for function in awslambda_client.functions.values():
|
||||
if function.region == inspector.region:
|
||||
funtions_in_region = True
|
||||
for instance in ec2_client.instances:
|
||||
if instance == inspector.region:
|
||||
ec2_in_region = True
|
||||
@@ -36,12 +35,12 @@ class inspector2_is_enabled(Check):
|
||||
failed_services.append("ECR")
|
||||
if inspector.lambda_status != "ENABLED" and (
|
||||
inspector2_client.provider.scan_unused_services
|
||||
or funtions_in_region
|
||||
or functions_in_region
|
||||
):
|
||||
failed_services.append("Lambda")
|
||||
if inspector.lambda_code_status != "ENABLED" and (
|
||||
inspector2_client.provider.scan_unused_services
|
||||
or funtions_in_region
|
||||
or functions_in_region
|
||||
):
|
||||
failed_services.append("Lambda Code")
|
||||
|
||||
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"Provider": "aws",
|
||||
"CheckID": "s3_bucket_object_public",
|
||||
"CheckTitle": "Spot-check S3 bucket objects for public ACLs",
|
||||
"CheckType": [
|
||||
"Software and Configuration Checks/AWS Security Best Practices",
|
||||
"Effects/Data Exposure"
|
||||
],
|
||||
"ServiceName": "s3",
|
||||
"SubServiceName": "",
|
||||
"ResourceIdTemplate": "arn:partition:s3:::resource",
|
||||
"Severity": "low",
|
||||
"ResourceType": "AwsS3Bucket",
|
||||
"Description": "Spot-checks a configurable sample of objects in each S3 bucket and flags any whose ACL grants access to the AllUsers or AuthenticatedUsers groups. This is a sampling-based check, not a comprehensive audit, so public objects outside the sample can be missed. It is disabled by default and must be enabled via the s3_bucket_object_public_enabled configuration flag.",
|
||||
"Risk": "Public objects can be accessed by anyone on the internet, potentially leaking sensitive data. A bucket can appear private at the bucket-policy level while still containing individual objects with public ACL grants.",
|
||||
"RelatedUrl": "",
|
||||
"Remediation": {
|
||||
"Code": {
|
||||
"CLI": "aws s3api put-object-acl --bucket <bucket_name> --key <object_key> --acl private",
|
||||
"NativeIaC": "",
|
||||
"Other": "",
|
||||
"Terraform": ""
|
||||
},
|
||||
"Recommendation": {
|
||||
"Text": "For complete coverage, enable the s3_bucket_acl_prohibited check, which enforces the BucketOwnerEnforced Object Ownership setting (AWS's recommended approach since April 2023) and prevents public object ACLs entirely. Use this spot-check as a supplementary tool for manual assessments.",
|
||||
"Url": "https://hub.prowler.com/check/s3_bucket_object_public"
|
||||
}
|
||||
},
|
||||
"Categories": [
|
||||
"internet-exposed"
|
||||
],
|
||||
"DependsOn": [],
|
||||
"RelatedTo": [
|
||||
"s3_bucket_acl_prohibited"
|
||||
],
|
||||
"Notes": "Disabled by default. Configure s3_bucket_object_public_enabled, s3_bucket_object_public_max_objects, and s3_bucket_object_public_sample_size in the Prowler configuration. Because only a sample of objects is inspected, a PASS does not guarantee the bucket is free of public objects; use s3_bucket_acl_prohibited for full assurance."
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
from typing import List
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.providers.aws.services.s3.s3_client import s3_client
|
||||
|
||||
# ACL grantee groups that make an object effectively public. AllUsers is anyone on
|
||||
# the internet; AuthenticatedUsers is any authenticated AWS principal (any account).
|
||||
PUBLIC_ACL_URIS = {
|
||||
"http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
"http://acs.amazonaws.com/groups/global/AuthenticatedUsers",
|
||||
}
|
||||
|
||||
|
||||
class s3_bucket_object_public(Check):
|
||||
"""Spot-check a sample of S3 bucket objects for public ACL grants."""
|
||||
|
||||
def execute(self) -> List[Check_Report_AWS]:
|
||||
"""Evaluate sampled object ACLs for AllUsers/AuthenticatedUsers grants.
|
||||
|
||||
Returns:
|
||||
List[Check_Report_AWS]: One report per sampled bucket (empty when the
|
||||
check is disabled via configuration).
|
||||
"""
|
||||
findings = []
|
||||
|
||||
if not s3_client.audit_config.get("s3_bucket_object_public_enabled", False):
|
||||
return findings
|
||||
|
||||
for bucket in s3_client.buckets.values():
|
||||
sampling = bucket.object_sampling
|
||||
# Sampling is populated by the service layer only when the check is
|
||||
# enabled; skip any bucket that was not sampled.
|
||||
if sampling is None or not sampling.performed:
|
||||
continue
|
||||
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=bucket)
|
||||
|
||||
if sampling.error_code is not None:
|
||||
report.status = "MANUAL"
|
||||
if sampling.error_code == "AccessDenied":
|
||||
report.status_extended = (
|
||||
f"Access Denied when spot-checking objects in bucket "
|
||||
f"{bucket.name}."
|
||||
)
|
||||
else:
|
||||
report.status_extended = (
|
||||
f"Could not spot-check objects in bucket {bucket.name}: "
|
||||
f"{sampling.error_message}."
|
||||
)
|
||||
elif sampling.is_empty:
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"S3 Bucket {bucket.name} is empty."
|
||||
else:
|
||||
public_objects = [
|
||||
obj.key
|
||||
for obj in sampling.objects
|
||||
if any(
|
||||
grantee.type == "Group" and grantee.URI in PUBLIC_ACL_URIS
|
||||
for grantee in obj.grantees
|
||||
)
|
||||
]
|
||||
sampled = len(sampling.objects)
|
||||
|
||||
if public_objects:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = (
|
||||
f"S3 Bucket {bucket.name} has public objects detected in "
|
||||
f"spot-check sample of {sampled} objects: "
|
||||
f"{', '.join(public_objects)}."
|
||||
)
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"No public objects detected in spot-check sample of "
|
||||
f"{sampled} objects in bucket {bucket.name}. For complete "
|
||||
f"assurance, ensure ACLs are disabled via Object Ownership "
|
||||
f"settings."
|
||||
)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
@@ -36,6 +36,10 @@ class S3(AWSService):
|
||||
self.__threading_call__(
|
||||
self._get_bucket_notification_configuration, self.buckets.values()
|
||||
)
|
||||
# Object-level ACL sampling is expensive and opt-in, so only run it when
|
||||
# the s3_bucket_object_public check is explicitly enabled in the config.
|
||||
if self.audit_config.get("s3_bucket_object_public_enabled", False):
|
||||
self.__threading_call__(self._get_public_objects, self.buckets.values())
|
||||
|
||||
def _list_buckets(self, provider):
|
||||
logger.info("S3 - Listing buckets...")
|
||||
@@ -487,6 +491,69 @@ class S3(AWSService):
|
||||
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _get_public_objects(self, bucket):
|
||||
logger.info("S3 - Spot-checking bucket objects for public ACLs...")
|
||||
max_objects = self.audit_config.get("s3_bucket_object_public_max_objects", 100)
|
||||
sample_size = self.audit_config.get("s3_bucket_object_public_sample_size", 3)
|
||||
# Guard against misconfigured non-positive values: a zero sample size would
|
||||
# raise ZeroDivisionError and a negative one would silently sample nothing.
|
||||
if not isinstance(max_objects, int) or max_objects <= 0:
|
||||
max_objects = 100
|
||||
if not isinstance(sample_size, int) or sample_size <= 0:
|
||||
sample_size = 3
|
||||
sampling = BucketObjectSampling(performed=True)
|
||||
regional_client = None
|
||||
try:
|
||||
regional_client = self.regional_clients[bucket.region]
|
||||
contents = regional_client.list_objects_v2(
|
||||
Bucket=bucket.name, MaxKeys=max_objects
|
||||
).get("Contents", [])
|
||||
|
||||
if not contents:
|
||||
sampling.is_empty = True
|
||||
bucket.object_sampling = sampling
|
||||
return
|
||||
|
||||
all_keys = [obj["Key"] for obj in contents]
|
||||
# Deterministic, evenly-spaced sampling so findings are reproducible
|
||||
# across scans instead of flipping between PASS/FAIL with a random sample.
|
||||
if len(all_keys) <= sample_size:
|
||||
sample_keys = all_keys
|
||||
else:
|
||||
step = len(all_keys) // sample_size
|
||||
sample_keys = [all_keys[i * step] for i in range(sample_size)]
|
||||
|
||||
for key in sample_keys:
|
||||
acl = regional_client.get_object_acl(Bucket=bucket.name, Key=key)
|
||||
grantees = []
|
||||
for grant in acl.get("Grants", []):
|
||||
grant_grantee = grant.get("Grantee", {})
|
||||
grantee = ACL_Grantee(type=grant_grantee.get("Type", ""))
|
||||
grantee.display_name = grant_grantee.get("DisplayName")
|
||||
grantee.ID = grant_grantee.get("ID")
|
||||
grantee.URI = grant_grantee.get("URI")
|
||||
grantee.permission = grant.get("Permission")
|
||||
grantees.append(grantee)
|
||||
sampling.objects.append(ObjectACL(key=key, grantees=grantees))
|
||||
|
||||
bucket.object_sampling = sampling
|
||||
except ClientError as error:
|
||||
sampling.error_code = error.response["Error"]["Code"]
|
||||
sampling.error_message = str(error)
|
||||
bucket.object_sampling = sampling
|
||||
region = regional_client.region if regional_client else bucket.region
|
||||
logger.warning(
|
||||
f"{region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
except Exception as error:
|
||||
sampling.error_code = error.__class__.__name__
|
||||
sampling.error_message = str(error)
|
||||
bucket.object_sampling = sampling
|
||||
region = regional_client.region if regional_client else bucket.region
|
||||
logger.error(
|
||||
f"{region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _head_bucket(self, bucket_name):
|
||||
logger.info("S3 - Checking if bucket exists...")
|
||||
try:
|
||||
@@ -654,6 +721,19 @@ class PublicAccessBlock(BaseModel):
|
||||
restrict_public_buckets: bool
|
||||
|
||||
|
||||
class ObjectACL(BaseModel):
|
||||
key: str
|
||||
grantees: List[ACL_Grantee] = Field(default_factory=list)
|
||||
|
||||
|
||||
class BucketObjectSampling(BaseModel):
|
||||
performed: bool = False
|
||||
is_empty: bool = False
|
||||
objects: List[ObjectACL] = Field(default_factory=list)
|
||||
error_code: Optional[str] = None
|
||||
error_message: Optional[str] = None
|
||||
|
||||
|
||||
class AccessPoint(BaseModel):
|
||||
arn: str
|
||||
account_id: str
|
||||
@@ -703,3 +783,4 @@ class Bucket(BaseModel):
|
||||
lifecycle: List[LifeCycleRule] = Field(default_factory=list)
|
||||
replication_rules: List[ReplicationRule] = Field(default_factory=list)
|
||||
notification_config: Dict = Field(default_factory=dict)
|
||||
object_sampling: Optional[BucketObjectSampling] = None
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import json
|
||||
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.ssm.ssm_client import ssm_client
|
||||
|
||||
|
||||
@@ -11,7 +15,26 @@ class ssm_document_secrets(Check):
|
||||
secrets_ignore_patterns = ssm_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for document in ssm_client.documents.values():
|
||||
validate = ssm_client.audit_config.get("secrets_validate", False)
|
||||
documents = list(ssm_client.documents.values())
|
||||
|
||||
# Collect one payload per document (its content) and scan them all in
|
||||
# batched Kingfisher invocations instead of one subprocess per document.
|
||||
def payloads():
|
||||
for index, document in enumerate(documents):
|
||||
if document.content:
|
||||
yield index, json.dumps(document.content, indent=2)
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for index, document in enumerate(documents):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=document)
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
@@ -19,13 +42,15 @@ class ssm_document_secrets(Check):
|
||||
)
|
||||
|
||||
if document.content:
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=json.dumps(document.content, indent=2),
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=ssm_client.audit_config.get(
|
||||
"detect_secrets_plugins"
|
||||
),
|
||||
)
|
||||
if scan_error:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan SSM Document {document.name} for secrets: "
|
||||
f"{scan_error}; manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
detect_secrets_output = batch_results.get(index)
|
||||
if detect_secrets_output:
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
@@ -35,6 +60,7 @@ class ssm_document_secrets(Check):
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Potential secret found in SSM Document {document.name} -> {secrets_string}."
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
|
||||
+36
-10
@@ -1,5 +1,9 @@
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.lib.utils.utils import detect_secrets_scan
|
||||
from prowler.lib.utils.utils import (
|
||||
SecretsScanError,
|
||||
annotate_verified_secrets,
|
||||
detect_secrets_scan_batch,
|
||||
)
|
||||
from prowler.providers.aws.services.stepfunctions.stepfunctions_client import (
|
||||
stepfunctions_client,
|
||||
)
|
||||
@@ -13,20 +17,41 @@ class stepfunctions_statemachine_no_secrets_in_definition(Check):
|
||||
secrets_ignore_patterns = stepfunctions_client.audit_config.get(
|
||||
"secrets_ignore_patterns", []
|
||||
)
|
||||
for state_machine in stepfunctions_client.state_machines.values():
|
||||
validate = stepfunctions_client.audit_config.get("secrets_validate", False)
|
||||
state_machines = list(stepfunctions_client.state_machines.values())
|
||||
|
||||
# Collect one payload per state machine (its definition) and scan them
|
||||
# all in batched Kingfisher invocations instead of one subprocess each.
|
||||
def payloads():
|
||||
for index, state_machine in enumerate(state_machines):
|
||||
if state_machine.definition:
|
||||
yield index, state_machine.definition
|
||||
|
||||
scan_error = None
|
||||
try:
|
||||
batch_results = detect_secrets_scan_batch(
|
||||
payloads(), excluded_secrets=secrets_ignore_patterns, validate=validate
|
||||
)
|
||||
except SecretsScanError as error:
|
||||
batch_results = {}
|
||||
scan_error = error
|
||||
|
||||
for index, state_machine in enumerate(state_machines):
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=state_machine)
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"No secrets found in Step Functions state machine {state_machine.name} definition."
|
||||
|
||||
if state_machine.definition:
|
||||
detect_secrets_output = detect_secrets_scan(
|
||||
data=state_machine.definition,
|
||||
excluded_secrets=secrets_ignore_patterns,
|
||||
detect_secrets_plugins=stepfunctions_client.audit_config.get(
|
||||
"detect_secrets_plugins",
|
||||
),
|
||||
)
|
||||
|
||||
if scan_error:
|
||||
report.status = "MANUAL"
|
||||
report.status_extended = (
|
||||
f"Could not scan Step Functions state machine "
|
||||
f"{state_machine.name} definition for secrets: {scan_error}; "
|
||||
"manual review is required."
|
||||
)
|
||||
findings.append(report)
|
||||
continue
|
||||
detect_secrets_output = batch_results.get(index)
|
||||
if detect_secrets_output:
|
||||
secrets_string = ", ".join(
|
||||
[
|
||||
@@ -40,6 +65,7 @@ class stepfunctions_statemachine_no_secrets_in_definition(Check):
|
||||
f"found in Step Functions state machine {state_machine.name} definition "
|
||||
f"-> {secrets_string}."
|
||||
)
|
||||
annotate_verified_secrets(report, detect_secrets_output)
|
||||
|
||||
findings.append(report)
|
||||
return findings
|
||||
|
||||
@@ -16,6 +16,7 @@ from azure.identity import (
|
||||
DefaultAzureCredential,
|
||||
InteractiveBrowserCredential,
|
||||
)
|
||||
from azure.mgmt.resource import ResourceManagementClient
|
||||
from azure.mgmt.subscription import SubscriptionClient
|
||||
from colorama import Fore, Style
|
||||
from msgraph import GraphServiceClient
|
||||
@@ -104,6 +105,7 @@ class AzureProvider(Provider):
|
||||
_region_config: AzureRegionConfig
|
||||
_locations: dict
|
||||
_mutelist: AzureMutelist
|
||||
_resource_groups: dict[str, list[str]]
|
||||
# TODO: this is not optional, enforce for all providers
|
||||
audit_metadata: Audit_Metadata
|
||||
|
||||
@@ -123,6 +125,7 @@ class AzureProvider(Provider):
|
||||
mutelist_content: dict = None,
|
||||
client_id: str = None,
|
||||
client_secret: str = None,
|
||||
resource_groups: list = [],
|
||||
):
|
||||
"""
|
||||
Initializes the Azure provider.
|
||||
@@ -142,6 +145,7 @@ class AzureProvider(Provider):
|
||||
mutelist_content (dict): The mutelist content.
|
||||
client_id (str): The Azure client ID.
|
||||
client_secret (str): The Azure client secret.
|
||||
resource_groups (list): List of resource group names.
|
||||
|
||||
Returns:
|
||||
None
|
||||
@@ -206,7 +210,7 @@ class AzureProvider(Provider):
|
||||
... managed_identity_auth=False,
|
||||
... region="AzureUSGovernment",
|
||||
... )
|
||||
- Subscriptions: rowler is multisubscription, which means that is going to scan all the subscriptions is able to list. If you only assign permissions to one subscription, it is going to scan a single one.
|
||||
- Subscriptions: Prowler is multisubscription, which means that is going to scan all the subscriptions is able to list. If you only assign permissions to one subscription, it is going to scan a single one.
|
||||
Prowler also allows you to specify the subscriptions you want to scan by passing a list of subscription IDs.
|
||||
>>> AzureProvider(
|
||||
... az_cli_auth=False,
|
||||
@@ -215,6 +219,11 @@ class AzureProvider(Provider):
|
||||
... managed_identity_auth=False,
|
||||
... subscription_ids=["XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"],
|
||||
... )
|
||||
- Resource Groups: Prowler allows you to narrow the scan to specific resource groups.
|
||||
>>> AzureProvider(
|
||||
... az_cli_auth=True,
|
||||
... resource_groups=["rg-production", "rg-staging"],
|
||||
... )
|
||||
|
||||
"""
|
||||
logger.info("Setting Azure provider ...")
|
||||
@@ -272,6 +281,8 @@ class AzureProvider(Provider):
|
||||
# TODO: should we keep this here or within the identity?
|
||||
self._locations = self.get_locations()
|
||||
|
||||
self._resource_groups = self.validate_resource_groups(resource_groups)
|
||||
|
||||
# Audit Config
|
||||
if config_content:
|
||||
self._audit_config = config_content
|
||||
@@ -337,6 +348,11 @@ class AzureProvider(Provider):
|
||||
"""Mutelist object associated with this Azure provider."""
|
||||
return self._mutelist
|
||||
|
||||
@property
|
||||
def resource_groups(self) -> dict[str, list[str]]:
|
||||
"""Mapping of subscription name to the list of resource groups to scan within it."""
|
||||
return self._resource_groups
|
||||
|
||||
# TODO: this should be moved to the argparse, if not we need to enforce it from the Provider
|
||||
# previously was using the AzureException
|
||||
@staticmethod
|
||||
@@ -439,7 +455,7 @@ class AzureProvider(Provider):
|
||||
"""Azure credentials information.
|
||||
|
||||
This method prints the Azure Tenant Domain, Azure Tenant ID, Azure Region,
|
||||
Azure Subscriptions, Azure Identity Type, and Azure Identity ID.
|
||||
Azure Subscriptions, Azure Resource Groups, Azure Identity Type, and Azure Identity ID.
|
||||
|
||||
Args:
|
||||
None
|
||||
@@ -455,6 +471,7 @@ class AzureProvider(Provider):
|
||||
f"Azure Tenant Domain: {Fore.YELLOW}{self._identity.tenant_domain}{Style.RESET_ALL} Azure Tenant ID: {Fore.YELLOW}{self._identity.tenant_ids[0]}{Style.RESET_ALL}",
|
||||
f"Azure Region: {Fore.YELLOW}{self.region_config.name}{Style.RESET_ALL}",
|
||||
f"Azure Subscriptions: {Fore.YELLOW}{printed_subscriptions}{Style.RESET_ALL}",
|
||||
f"Azure Resource Groups: {Fore.YELLOW}{sorted({rg for rgs in self._resource_groups.values() for rg in rgs}) if any(self._resource_groups.values()) else ('NONE (no matching resource groups found)' if self._resource_groups else 'ALL')}{Style.RESET_ALL}",
|
||||
f"Azure Identity Type: {Fore.YELLOW}{self._identity.identity_type}{Style.RESET_ALL} Azure Identity ID: {Fore.YELLOW}{self._identity.identity_id}{Style.RESET_ALL}",
|
||||
]
|
||||
report_title = (
|
||||
@@ -1102,6 +1119,54 @@ class AzureProvider(Provider):
|
||||
|
||||
return set(chain.from_iterable(locations.values()))
|
||||
|
||||
def validate_resource_groups(self, resource_groups: list) -> dict[str, list[str]]:
|
||||
resource_groups = [r.strip() for r in resource_groups if r and r.strip()]
|
||||
if not resource_groups:
|
||||
return {}
|
||||
|
||||
rg_map = {
|
||||
subscription_id: [] for subscription_id in self._identity.subscriptions
|
||||
}
|
||||
credentials = self.session
|
||||
|
||||
for subscription_id, display_name in self._identity.subscriptions.items():
|
||||
try:
|
||||
rg_client = ResourceManagementClient(
|
||||
credentials,
|
||||
subscription_id,
|
||||
base_url=self._region_config.base_url,
|
||||
credential_scopes=self._region_config.credential_scopes,
|
||||
)
|
||||
existing_rgs = {
|
||||
rg.name.lower(): rg.name for rg in rg_client.resource_groups.list()
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Could not list resource groups for subscription '{display_name}' "
|
||||
f"({subscription_id}): {e}. Skipping resource group filtering for this subscription."
|
||||
)
|
||||
continue
|
||||
|
||||
for rg in resource_groups:
|
||||
real_name = existing_rgs.get(rg.lower())
|
||||
if real_name:
|
||||
rg_map[subscription_id].append(real_name)
|
||||
|
||||
for rg in resource_groups:
|
||||
if not any(rg.lower() == r.lower() for rgs in rg_map.values() for r in rgs):
|
||||
logger.warning(
|
||||
f"Resource group '{rg}' was not found in any subscription. "
|
||||
"Please check the resource group name and try again."
|
||||
)
|
||||
|
||||
if not any(rgs for rgs in rg_map.values()):
|
||||
logger.warning(
|
||||
f"None of the provided resource groups {resource_groups} were found "
|
||||
"in any subscription. Please check the resource group names and try again."
|
||||
)
|
||||
|
||||
return rg_map
|
||||
|
||||
@staticmethod
|
||||
def validate_static_credentials(
|
||||
tenant_id: str = None,
|
||||
|
||||
@@ -53,6 +53,16 @@ def init_parser(self):
|
||||
type=validate_azure_region,
|
||||
help="Azure region from `az cloud list --output table`, by default AzureCloud",
|
||||
)
|
||||
# Resource Groups
|
||||
azure_rg_subparser = azure_parser.add_argument_group("Resource Groups")
|
||||
azure_rg_subparser.add_argument(
|
||||
"--azure-resource-group",
|
||||
"--azure-resource-groups",
|
||||
nargs="+",
|
||||
default=[],
|
||||
dest="resource_groups",
|
||||
help="Azure Resource Group names to scope the scan to specific groups.",
|
||||
)
|
||||
|
||||
|
||||
def validate_azure_region(region):
|
||||
|
||||
@@ -26,6 +26,7 @@ class AzureService:
|
||||
)
|
||||
|
||||
self.subscriptions = provider.identity.subscriptions
|
||||
self.resource_groups = provider.resource_groups
|
||||
self.locations = provider.locations
|
||||
self.audit_config = provider.audit_config
|
||||
self.fixer_config = provider.fixer_config
|
||||
@@ -49,6 +50,26 @@ class AzureService:
|
||||
|
||||
return results
|
||||
|
||||
def list_with_rg_scope(self, subscription_id, list_all_fn, list_by_rg_fn):
|
||||
if not self.resource_groups:
|
||||
return list(list_all_fn())
|
||||
resource_groups = self.resource_groups.get(subscription_id, [])
|
||||
if not resource_groups:
|
||||
logger.info(
|
||||
f"No valid resource groups for subscription {subscription_id}, skipping."
|
||||
)
|
||||
return []
|
||||
output = []
|
||||
for resource_group in resource_groups:
|
||||
try:
|
||||
output += list(list_by_rg_fn(resource_group_name=resource_group))
|
||||
except Exception as error:
|
||||
logger.warning(
|
||||
f"Subscription ID: {subscription_id} -- Resource Group: {resource_group} -- "
|
||||
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
return output
|
||||
|
||||
def __set_clients__(self, identity, session, service, region_config):
|
||||
clients = {}
|
||||
try:
|
||||
|
||||
@@ -17,7 +17,11 @@ class AISearch(AzureService):
|
||||
for subscription, client in self.clients.items():
|
||||
try:
|
||||
aisearch_services.update({subscription: {}})
|
||||
aisearch_services_list = client.services.list_by_subscription()
|
||||
aisearch_services_list = self.list_with_rg_scope(
|
||||
subscription,
|
||||
client.services.list_by_subscription,
|
||||
client.services.list_by_resource_group,
|
||||
)
|
||||
for aisearch_service in aisearch_services_list:
|
||||
aisearch_services[subscription].update(
|
||||
{
|
||||
|
||||
@@ -19,8 +19,12 @@ class AKS(AzureService):
|
||||
|
||||
for subscription_id, client in self.clients.items():
|
||||
try:
|
||||
clusters_list = client.managed_clusters.list()
|
||||
clusters.update({subscription_id: {}})
|
||||
clusters_list = self.list_with_rg_scope(
|
||||
subscription_id,
|
||||
client.managed_clusters.list,
|
||||
client.managed_clusters.list_by_resource_group,
|
||||
)
|
||||
|
||||
for cluster in clusters_list:
|
||||
if getattr(cluster, "kubernetes_version", None):
|
||||
|
||||
@@ -131,7 +131,11 @@ class APIM(AzureService):
|
||||
for subscription, client in self.clients.items():
|
||||
try:
|
||||
instances.update({subscription: []})
|
||||
apim_instances = client.api_management_service.list()
|
||||
apim_instances = self.list_with_rg_scope(
|
||||
subscription,
|
||||
client.api_management_service.list,
|
||||
client.api_management_service.list_by_resource_group,
|
||||
)
|
||||
|
||||
for instance in apim_instances:
|
||||
workspace_id = self._get_log_analytics_workspace_id(
|
||||
|
||||
@@ -22,8 +22,12 @@ class App(AzureService):
|
||||
|
||||
for subscription_id, client in self.clients.items():
|
||||
try:
|
||||
apps_list = client.web_apps.list()
|
||||
apps.update({subscription_id: {}})
|
||||
apps_list = self.list_with_rg_scope(
|
||||
subscription_id,
|
||||
client.web_apps.list,
|
||||
client.web_apps.list_by_resource_group,
|
||||
)
|
||||
|
||||
for app in apps_list:
|
||||
# Filter function apps
|
||||
@@ -117,8 +121,12 @@ class App(AzureService):
|
||||
|
||||
for subscription_id, client in self.clients.items():
|
||||
try:
|
||||
functions_list = client.web_apps.list()
|
||||
functions.update({subscription_id: {}})
|
||||
functions_list = self.list_with_rg_scope(
|
||||
subscription_id,
|
||||
client.web_apps.list,
|
||||
client.web_apps.list_by_resource_group,
|
||||
)
|
||||
|
||||
for function in functions_list:
|
||||
# Filter function apps
|
||||
|
||||
@@ -17,8 +17,12 @@ class AppInsights(AzureService):
|
||||
|
||||
for subscription_id, client in self.clients.items():
|
||||
try:
|
||||
components_list = client.components.list()
|
||||
components.update({subscription_id: {}})
|
||||
components_list = self.list_with_rg_scope(
|
||||
subscription_id,
|
||||
client.components.list,
|
||||
client.components.list_by_resource_group,
|
||||
)
|
||||
|
||||
for component in components_list:
|
||||
components[subscription_id].update(
|
||||
|
||||
@@ -19,8 +19,12 @@ class ContainerRegistry(AzureService):
|
||||
registries = {}
|
||||
for subscription, client in self.clients.items():
|
||||
try:
|
||||
registries_list = client.registries.list()
|
||||
registries.update({subscription: {}})
|
||||
registries_list = self.list_with_rg_scope(
|
||||
subscription,
|
||||
client.registries.list,
|
||||
client.registries.list_by_resource_group,
|
||||
)
|
||||
|
||||
for registry in registries_list:
|
||||
resource_group = self._get_resource_group(registry.id)
|
||||
|
||||
@@ -18,8 +18,13 @@ class CosmosDB(AzureService):
|
||||
accounts = {}
|
||||
for subscription, client in self.clients.items():
|
||||
try:
|
||||
accounts_list = client.database_accounts.list()
|
||||
accounts.update({subscription: []})
|
||||
accounts_list = self.list_with_rg_scope(
|
||||
subscription,
|
||||
client.database_accounts.list,
|
||||
client.database_accounts.list_by_resource_group,
|
||||
)
|
||||
|
||||
for account in accounts_list:
|
||||
accounts[subscription].append(
|
||||
Account(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user