mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-05-14 16:25:13 +00:00
Compare commits
133 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5198b0251c | |||
| 9894ac7bc3 | |||
| 8ae076f22c | |||
| 27fb8518cd | |||
| 375de5dc87 | |||
| 3410fc927a | |||
| dcf91ef252 | |||
| bf4fd8fabd | |||
| 3d65208fd3 | |||
| 7d3ed62e90 | |||
| 5f92989492 | |||
| 6befa78978 | |||
| 78af0c24fe | |||
| 1bb547e5e1 | |||
| 1f39b01fb2 | |||
| fb0ef391f2 | |||
| f2e6a3264d | |||
| 9bd4e4b65c | |||
| 68ffb2b219 | |||
| 739be07077 | |||
| 0abbb7fc59 | |||
| 0b4393776c | |||
| 4dd5baadf6 | |||
| 934d995661 | |||
| ccdc01ed7b | |||
| d84099e87a | |||
| cf55f7eb43 | |||
| 9293c7b58d | |||
| a883bb30d4 | |||
| e476bbde2d | |||
| 7f3dcdf02f | |||
| 132e79df89 | |||
| b2ed9ee221 | |||
| def2d3d188 | |||
| 1090ed59b7 | |||
| 67e4b1a082 | |||
| 7478ec9420 | |||
| a30b6623ed | |||
| 15bc317ec4 | |||
| 1536102784 | |||
| 1b99550572 | |||
| 6dfa135755 | |||
| 80482da1cb | |||
| 9cedbd3582 | |||
| c3d1c5c5f7 | |||
| 1fd6c51af6 | |||
| adbfc0bcd1 | |||
| 8f041f6f52 | |||
| 1b0e12ec51 | |||
| 759f7b84d6 | |||
| 0b26c1a39c | |||
| fc7fbddfe7 | |||
| 500b395125 | |||
| a1961d6d5f | |||
| a7e988c361 | |||
| 02cdcb29db | |||
| 6e0d7866cd | |||
| 4b71f37c91 | |||
| cdfbe5b2e3 | |||
| 1b6a459df4 | |||
| 73c0305dc4 | |||
| 0e01e67257 | |||
| 1ad329f9cf | |||
| d03d1d2393 | |||
| 832516be2a | |||
| 34727a7237 | |||
| 4216a3e23a | |||
| a59192e6f5 | |||
| 592bc6f6a8 | |||
| 962ebac8e4 | |||
| 2c5d47a8cd | |||
| bcaa6ac488 | |||
| 71683f3093 | |||
| 2357af912d | |||
| 7971b40f49 | |||
| e585ae45bd | |||
| 4d9921a9b7 | |||
| 19b602c381 | |||
| 4c3e741af7 | |||
| 8affbf44ce | |||
| 16798e293d | |||
| 1194d34396 | |||
| 98277689f5 | |||
| 0ddd7fbd69 | |||
| 22b233f206 | |||
| aa759ab6b7 | |||
| 369d6cecc1 | |||
| d23c2f3b53 | |||
| 786059bfb2 | |||
| 703a33108c | |||
| 7c6d658154 | |||
| 21d7d08b4b | |||
| f314725f4d | |||
| 02f43a7ad6 | |||
| 0dd8981ee4 | |||
| 269e51259d | |||
| f4afdf0541 | |||
| 652cb69216 | |||
| 921f49a0de | |||
| 6cb770fcc8 | |||
| 86449fb99d | |||
| 40dd0e640b | |||
| 8db3a89669 | |||
| c802dc8a36 | |||
| 3ab9a4efa5 | |||
| 36b8aa1b79 | |||
| e821e07d7d | |||
| 228fe6d579 | |||
| 578186aa40 | |||
| 4608e45c8a | |||
| 5987651aee | |||
| 85800f2ddd | |||
| 4fb5272362 | |||
| 85d38b5f71 | |||
| 59dcdb87c4 | |||
| 9297453b8a | |||
| dd37f4ee1f | |||
| 20f36f7c84 | |||
| ec4d27746f | |||
| 7076900fb1 | |||
| 5d90352a0f | |||
| a981dc64a7 | |||
| d2086cad3f | |||
| 380b89cfb6 | |||
| 13b04d339b | |||
| be3c5fb3c1 | |||
| 1de01bcb78 | |||
| 13d983450c | |||
| 8b368e1343 | |||
| c76a9baa20 | |||
| 30e2813e02 | |||
| 0f874c6ffd | |||
| 2242689295 |
+5
-6
@@ -2,20 +2,19 @@
|
||||
# Runs automatically on `wt switch --create`.
|
||||
|
||||
# Block 1: setup + copy gitignored env files (.envrc, ui/.env.local)
|
||||
# from the primary worktree — patterns selected via .worktreeinclude.
|
||||
# from the primary worktree - patterns selected via .worktreeinclude.
|
||||
[[pre-start]]
|
||||
skills = "./skills/setup.sh --claude"
|
||||
python = "poetry env use python3.12"
|
||||
envs = "wt step copy-ignored"
|
||||
|
||||
# Block 2: install Python deps (requires `poetry env use` from block 1).
|
||||
# Block 2: install Python deps (uv manages the venv on `uv sync`).
|
||||
[[pre-start]]
|
||||
deps = "poetry install --with dev"
|
||||
deps = "uv sync"
|
||||
|
||||
# Block 3: reminder — last visible output before `wt switch` returns.
|
||||
# Block 3: reminder - last visible output before `wt switch` returns.
|
||||
# Hooks can't mutate the parent shell, so venv activation is manual.
|
||||
[[pre-start]]
|
||||
reminder = "echo '>> Reminder: activate the venv in this shell with: eval $(poetry env activate)'"
|
||||
reminder = "echo '>> Reminder: activate the venv in this shell with: source .venv/bin/activate'"
|
||||
|
||||
# Background: pnpm install runs while you start working.
|
||||
# Tail logs via `wt config state logs`.
|
||||
|
||||
@@ -145,7 +145,7 @@ SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.25.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.27.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: [prowler-cloud]
|
||||
# patreon: # Replace with a single Patreon username
|
||||
# open_collective: # Replace with a single Open Collective username
|
||||
# ko_fi: # Replace with a single Ko-fi username
|
||||
# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
# liberapay: # Replace with a single Liberapay username
|
||||
# issuehunt: # Replace with a single IssueHunt username
|
||||
# lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
||||
# polar: # Replace with a single Polar username
|
||||
# buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
|
||||
# thanks_dev: # Replace with a single thanks.dev username
|
||||
# custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
||||
@@ -0,0 +1,143 @@
|
||||
name: "🔎 New Check Request"
|
||||
description: Request a new Prowler security check
|
||||
title: "[New Check]: "
|
||||
labels: ["feature-request", "status/needs-triage"]
|
||||
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: search
|
||||
attributes:
|
||||
label: Existing check search
|
||||
description: Confirm this check does not already exist before opening a new request.
|
||||
options:
|
||||
- label: I have searched existing issues, Prowler Hub, and the public roadmap, and this check does not already exist.
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Use this form to describe the security condition that Prowler should evaluate.
|
||||
|
||||
The most useful inputs for [Prowler Studio](https://github.com/prowler-cloud/prowler-studio) are:
|
||||
- What should be detected
|
||||
- What PASS and FAIL mean
|
||||
- Vendor docs, API references, SDK methods, CLI commands, or reference code
|
||||
|
||||
- type: dropdown
|
||||
id: provider
|
||||
attributes:
|
||||
label: Provider
|
||||
description: Cloud or platform this check targets.
|
||||
options:
|
||||
- AWS
|
||||
- Azure
|
||||
- GCP
|
||||
- Kubernetes
|
||||
- GitHub
|
||||
- Microsoft 365
|
||||
- OCI
|
||||
- Alibaba Cloud
|
||||
- Cloudflare
|
||||
- MongoDB Atlas
|
||||
- Google Workspace
|
||||
- OpenStack
|
||||
- Vercel
|
||||
- NHN
|
||||
- Other / New provider
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: other_provider_name
|
||||
attributes:
|
||||
label: New provider name
|
||||
description: Only fill this if you selected "Other / New provider" above.
|
||||
placeholder: "NewProviderName"
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: service_name
|
||||
attributes:
|
||||
label: Service or product area
|
||||
description: Optional. Main service, product, or feature to audit.
|
||||
placeholder: "s3, bedrock, entra, repository, apiserver"
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: suggested_check_name
|
||||
attributes:
|
||||
label: Suggested check name
|
||||
description: Optional. Use `snake_case` following `<service>_<resource>_<best_practice>`, with lowercase letters and underscores only.
|
||||
placeholder: "bedrock_guardrail_sensitive_information_filter_enabled"
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: context
|
||||
attributes:
|
||||
label: Context and goal
|
||||
description: Describe the security problem, why it matters, and what this new check should help detect.
|
||||
placeholder: |-
|
||||
- Security condition to validate:
|
||||
- Why it matters:
|
||||
- Resource, feature, or configuration involved:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected_behavior
|
||||
attributes:
|
||||
label: Expected behavior
|
||||
description: Explain what the check should evaluate and what PASS, FAIL, or MANUAL should mean.
|
||||
placeholder: |-
|
||||
- Resource or scope to evaluate:
|
||||
- PASS when:
|
||||
- FAIL when:
|
||||
- MANUAL when (if applicable):
|
||||
- Exclusions, thresholds, or edge cases:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: references
|
||||
attributes:
|
||||
label: References
|
||||
description: Add vendor docs, API references, SDK methods, CLI commands, endpoint docs, sample payloads, or similar reference material.
|
||||
placeholder: |-
|
||||
- Product or service documentation:
|
||||
- API or SDK reference:
|
||||
- CLI command or endpoint documentation:
|
||||
- Sample payload or response:
|
||||
- Security advisory or benchmark:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: severity
|
||||
attributes:
|
||||
label: Suggested severity
|
||||
description: Your best estimate. Reviewers will confirm during triage.
|
||||
options:
|
||||
- Critical
|
||||
- High
|
||||
- Medium
|
||||
- Low
|
||||
- Informational
|
||||
- Not sure
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: implementation_notes
|
||||
attributes:
|
||||
label: Additional implementation notes
|
||||
description: Optional. Add permissions, unsupported regions, config knobs, product limitations, or anything else that may affect implementation.
|
||||
placeholder: |-
|
||||
- Required permissions or scopes:
|
||||
- Region, tenant, or subscription limitations:
|
||||
- Configurable behavior or thresholds:
|
||||
- Other constraints:
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,169 @@
|
||||
name: 'OSV-Scanner'
|
||||
description: 'Install osv-scanner and scan a lockfile, failing on HIGH/CRITICAL/UNKNOWN severity findings. Posts/updates a PR comment with findings on pull_request events (requires pull-requests: write).'
|
||||
author: 'Prowler'
|
||||
|
||||
inputs:
|
||||
lockfile:
|
||||
description: 'Path to the lockfile to scan, relative to the repository root (e.g. uv.lock, api/uv.lock, ui/pnpm-lock.yaml).'
|
||||
required: true
|
||||
severity-levels:
|
||||
description: 'Comma-separated severity levels that fail the scan. Default: HIGH,CRITICAL,UNKNOWN.'
|
||||
required: false
|
||||
default: 'HIGH,CRITICAL,UNKNOWN'
|
||||
version:
|
||||
description: 'osv-scanner release tag to install. When overriding, you MUST also override binary-sha256.'
|
||||
required: false
|
||||
default: 'v2.3.8'
|
||||
binary-sha256:
|
||||
description: 'Expected SHA256 of osv-scanner_linux_amd64 for the given version. Default tracks v2.3.8. See https://github.com/google/osv-scanner/releases/download/<version>/osv-scanner_SHA256SUMS.'
|
||||
required: false
|
||||
default: 'bc98e15319ed0d515e3f9235287ba53cdc5535d576d24fd573978ecfe9ab92dc'
|
||||
post-pr-comment:
|
||||
description: 'Post or update a PR comment with the scan report. Only effective on pull_request events. Requires pull-requests: write permission on the caller job.'
|
||||
required: false
|
||||
default: 'true'
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Install osv-scanner
|
||||
shell: bash
|
||||
env:
|
||||
OSV_SCANNER_VERSION: ${{ inputs.version }}
|
||||
# Download the binary AND the published SHA256SUMS file, then verify the
|
||||
# binary checksum against the upstream-signed manifest. Aborts on mismatch.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if command -v osv-scanner >/dev/null 2>&1; then
|
||||
INSTALLED="$(osv-scanner --version 2>&1 | awk '/scanner version/ {print $NF; exit}')"
|
||||
if [ "v${INSTALLED}" = "${OSV_SCANNER_VERSION}" ]; then
|
||||
echo "osv-scanner ${OSV_SCANNER_VERSION} already installed."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
BASE="https://github.com/google/osv-scanner/releases/download/${OSV_SCANNER_VERSION}"
|
||||
BIN_NAME="osv-scanner_linux_amd64"
|
||||
curl -fSL --retry 3 "${BASE}/${BIN_NAME}" -o "${RUNNER_TEMP}/${BIN_NAME}"
|
||||
curl -fSL --retry 3 "${BASE}/osv-scanner_SHA256SUMS" -o "${RUNNER_TEMP}/osv-scanner_SHA256SUMS"
|
||||
(cd "${RUNNER_TEMP}" && sha256sum --check --ignore-missing osv-scanner_SHA256SUMS)
|
||||
chmod +x "${RUNNER_TEMP}/${BIN_NAME}"
|
||||
sudo mv "${RUNNER_TEMP}/${BIN_NAME}" /usr/local/bin/osv-scanner
|
||||
rm -f "${RUNNER_TEMP}/osv-scanner_SHA256SUMS"
|
||||
osv-scanner --version
|
||||
|
||||
- name: Run osv-scanner
|
||||
id: scan
|
||||
shell: bash
|
||||
working-directory: ${{ github.workspace }}
|
||||
env:
|
||||
OSV_LOCKFILE: ${{ inputs.lockfile }}
|
||||
OSV_SEVERITY_LEVELS: ${{ inputs.severity-levels }}
|
||||
OSV_REPORT_FILE: ${{ runner.temp }}/osv-scanner-findings.json
|
||||
# Per-vulnerability ignores (reason + expiry) live in osv-scanner.toml at the repo root, if present.
|
||||
# Severity filter is enforced in the wrapper via OSV_SEVERITY_LEVELS.
|
||||
# `continue-on-error: true` lets the PR-comment step run even when findings exist;
|
||||
# the gate step below re-fails the job from the wrapper exit code.
|
||||
continue-on-error: true
|
||||
run: ./.github/scripts/osv-scan.sh --lockfile="${OSV_LOCKFILE}"
|
||||
|
||||
- name: Post osv-scanner report on PR
|
||||
if: >-
|
||||
always()
|
||||
&& inputs.post-pr-comment == 'true'
|
||||
&& github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.head.repo.full_name == github.repository
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
||||
env:
|
||||
OSV_REPORT_FILE: ${{ runner.temp }}/osv-scanner-findings.json
|
||||
OSV_LOCKFILE: ${{ inputs.lockfile }}
|
||||
OSV_SEVERITY_LEVELS: ${{ inputs.severity-levels }}
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const lockfile = process.env.OSV_LOCKFILE;
|
||||
const severityLevels = process.env.OSV_SEVERITY_LEVELS;
|
||||
const reportFile = process.env.OSV_REPORT_FILE;
|
||||
const marker = `<!-- osv-scanner-report:${lockfile} -->`;
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
let findings = [];
|
||||
if (fs.existsSync(reportFile)) {
|
||||
try {
|
||||
findings = JSON.parse(fs.readFileSync(reportFile, 'utf8'));
|
||||
} catch (err) {
|
||||
core.warning(`Could not parse ${reportFile}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body?.includes(marker));
|
||||
|
||||
if (findings.length === 0) {
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
core.info(`Deleted stale osv-scanner comment for ${lockfile}.`);
|
||||
} else {
|
||||
core.info(`No findings and no stale comment for ${lockfile}.`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const sevIcon = (s) => ({
|
||||
CRITICAL: '🔴', HIGH: '🟠', MEDIUM: '🟡', LOW: '🟢', UNKNOWN: '⚪',
|
||||
}[s] || '⚪');
|
||||
const escape = (s) => String(s ?? '').replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
||||
const rows = findings.map(f =>
|
||||
`| ${sevIcon(f.severity)} ${f.severity}${f.score ? ` (${f.score})` : ''} | \`${escape(f.id)}\` | \`${escape(f.ecosystem)}/${escape(f.package)}\` | \`${escape(f.version)}\` | ${escape(f.summary || '(no summary)')} |`
|
||||
);
|
||||
|
||||
const body = [
|
||||
marker,
|
||||
`## 🔒 osv-scanner: ${findings.length} finding(s) in \`${lockfile}\``,
|
||||
'',
|
||||
`Severity gate: \`${severityLevels}\``,
|
||||
'',
|
||||
'| Severity | ID | Package | Version | Summary |',
|
||||
'|----------|----|---------|---------|---------|',
|
||||
...rows,
|
||||
'',
|
||||
`To accept a finding, add an \`[[IgnoredVulns]]\` entry to \`osv-scanner.toml\` at the repo root with a reason and \`ignoreUntil\`.`,
|
||||
'',
|
||||
`<sub>[View run](${runUrl})</sub>`,
|
||||
].join('\n');
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
core.info(`Updated osv-scanner comment for ${lockfile}.`);
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
core.info(`Posted new osv-scanner comment for ${lockfile}.`);
|
||||
}
|
||||
|
||||
- name: Enforce osv-scanner severity gate
|
||||
shell: bash
|
||||
env:
|
||||
SCAN_OUTCOME: ${{ steps.scan.outcome }}
|
||||
run: |
|
||||
if [ "${SCAN_OUTCOME}" != "success" ]; then
|
||||
echo "osv-scanner gate: scan reported findings (outcome=${SCAN_OUTCOME})" >&2
|
||||
exit 1
|
||||
fi
|
||||
+30
-40
@@ -1,5 +1,5 @@
|
||||
name: 'Setup Python with Poetry'
|
||||
description: 'Setup Python environment with Poetry and install dependencies'
|
||||
name: 'Setup Python with uv'
|
||||
description: 'Setup Python environment with uv and install dependencies'
|
||||
author: 'Prowler'
|
||||
|
||||
inputs:
|
||||
@@ -7,23 +7,15 @@ inputs:
|
||||
description: 'Python version to use'
|
||||
required: true
|
||||
working-directory:
|
||||
description: 'Working directory for Poetry'
|
||||
description: 'Working directory for uv'
|
||||
required: false
|
||||
default: '.'
|
||||
poetry-version:
|
||||
description: 'Poetry version to install'
|
||||
uv-version:
|
||||
description: 'uv version to install'
|
||||
required: false
|
||||
default: '2.3.4'
|
||||
default: '0.11.14'
|
||||
install-dependencies:
|
||||
description: 'Install Python dependencies with Poetry'
|
||||
required: false
|
||||
default: 'true'
|
||||
update-lock:
|
||||
description: 'Run `poetry lock` during setup. Only enable when a prior step mutates pyproject.toml (e.g. API `@master` VCS rewrite). Default: false.'
|
||||
required: false
|
||||
default: 'false'
|
||||
enable-cache:
|
||||
description: 'Whether to enable Poetry dependency caching via actions/setup-python'
|
||||
description: 'Install Python dependencies with uv'
|
||||
required: false
|
||||
default: 'true'
|
||||
|
||||
@@ -47,54 +39,52 @@ runs:
|
||||
sed -i "s|\(git+https://github.com/prowler-cloud/prowler[^@]*\)@master|\1@$BRANCH_NAME|g" pyproject.toml
|
||||
fi
|
||||
|
||||
- name: Install poetry
|
||||
shell: bash
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pipx install poetry==${INPUTS_POETRY_VERSION}
|
||||
env:
|
||||
INPUTS_POETRY_VERSION: ${{ inputs.poetry-version }}
|
||||
|
||||
- name: Update poetry.lock with latest Prowler commit
|
||||
- name: Update uv.lock with latest Prowler commit
|
||||
if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / {
|
||||
s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/
|
||||
}' poetry.lock
|
||||
echo "Updated resolved_reference:"
|
||||
grep -A2 -B2 "resolved_reference" poetry.lock
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
grep "prowler-cloud/prowler" uv.lock
|
||||
|
||||
- name: Update poetry.lock (prowler repo only)
|
||||
if: github.repository == 'prowler-cloud/prowler' && inputs.update-lock == 'true'
|
||||
- name: Update uv.lock SDK commit (prowler repo on push)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: poetry lock
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
grep "prowler-cloud/prowler" uv.lock
|
||||
|
||||
- name: Install uv
|
||||
shell: bash
|
||||
env:
|
||||
UV_VERSION: ${{ inputs.uv-version }}
|
||||
run: pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir "uv==${UV_VERSION}"
|
||||
|
||||
- name: Set up Python ${{ inputs.python-version }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
# Disable cache when callers skip dependency install: Poetry 2.3.4 creates
|
||||
# the venv in a path setup-python can't hash, breaking the post-step save-cache.
|
||||
cache: ${{ inputs.enable-cache == 'true' && 'poetry' || '' }}
|
||||
cache-dependency-path: ${{ inputs.enable-cache == 'true' && format('{0}/poetry.lock', inputs.working-directory) || '' }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Python dependencies
|
||||
if: inputs.install-dependencies == 'true'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry install --no-root
|
||||
poetry run pip list
|
||||
uv sync --no-install-project
|
||||
uv run pip list
|
||||
|
||||
- name: Update Prowler Cloud API Client
|
||||
if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry remove prowler-cloud-api-client
|
||||
poetry add ./prowler-cloud-api-client
|
||||
uv remove prowler-cloud-api-client
|
||||
uv add ./prowler-cloud-api-client
|
||||
@@ -72,6 +72,11 @@ provider/vercel:
|
||||
- any-glob-to-any-file: "prowler/providers/vercel/**"
|
||||
- any-glob-to-any-file: "tests/providers/vercel/**"
|
||||
|
||||
provider/okta:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/okta/**"
|
||||
- any-glob-to-any-file: "tests/providers/okta/**"
|
||||
|
||||
github_actions:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: ".github/workflows/*"
|
||||
@@ -109,6 +114,8 @@ mutelist:
|
||||
- any-glob-to-any-file: "tests/providers/googleworkspace/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/vercel/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/vercel/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/okta/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/okta/lib/mutelist/**"
|
||||
|
||||
integration/s3:
|
||||
- changed-files:
|
||||
|
||||
@@ -36,6 +36,7 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
#### UI
|
||||
- [ ] All issue/task requirements work as expected on the UI
|
||||
- [ ] If this PR adds or updates npm dependencies, include package-health evidence (maintenance, popularity, known vulnerabilities, license, release age) and explain why existing/native alternatives are insufficient.
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
|
||||
@@ -48,7 +49,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Performance test results (if applicable)
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required (e.g., specs, Poetry, etc.).
|
||||
- [ ] Check if version updates are required (e.g., specs, uv, etc.).
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
|
||||
|
||||
### License
|
||||
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run osv-scanner and fail when findings match the configured severity levels.
|
||||
#
|
||||
# Replaces `safety check --policy-file .safety-policy.yml`. Used by:
|
||||
# - .github/actions/osv-scanner/action.yml (composite action)
|
||||
# - .github/workflows/api-security.yml, sdk-security.yml, ui-security.yml
|
||||
#
|
||||
# Severity levels (comma-separated) are read from OSV_SEVERITY_LEVELS.
|
||||
# Default: HIGH,CRITICAL,UNKNOWN — preserves prior .safety-policy.yml policy
|
||||
# (ignore-cvss-severity-below: 7 + ignore-cvss-unknown-severity: False).
|
||||
# osv-scanner has no native CVSS threshold (google/osv-scanner#1400, closed
|
||||
# not-planned). Severity is derived from $group.max_severity (numeric CVSS
|
||||
# score string) which osv-scanner emits per group.
|
||||
#
|
||||
# CVSS v3 score → categorical label:
|
||||
# CRITICAL >= 9.0
|
||||
# HIGH >= 7.0
|
||||
# MEDIUM >= 4.0
|
||||
# LOW > 0.0
|
||||
# UNKNOWN no score available
|
||||
#
|
||||
# Per-vulnerability ignores (with reason + expiry) live in osv-scanner.toml at
|
||||
# the repo root, if it exists. Without that file, osv-scanner uses defaults.
|
||||
#
|
||||
# Usage:
|
||||
# osv-scan.sh [osv-scanner pass-through args...]
|
||||
# Examples:
|
||||
# osv-scan.sh --lockfile=uv.lock
|
||||
# osv-scan.sh --recursive .
|
||||
# OSV_SEVERITY_LEVELS=CRITICAL osv-scan.sh --lockfile=uv.lock
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel)"
|
||||
CONFIG="${ROOT}/osv-scanner.toml"
|
||||
SEVERITY_LEVELS="${OSV_SEVERITY_LEVELS:-HIGH,CRITICAL,UNKNOWN}"
|
||||
|
||||
for bin in osv-scanner jq; do
|
||||
if ! command -v "${bin}" >/dev/null 2>&1; then
|
||||
echo "error: ${bin} not found in PATH" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
SCAN_ARGS=()
|
||||
if [ -f "${CONFIG}" ]; then
|
||||
SCAN_ARGS+=(--config="${CONFIG}")
|
||||
fi
|
||||
|
||||
# Exit codes: 0=clean, 1=findings, 127=no supported files, 128=internal error.
|
||||
STDERR="$(mktemp)"
|
||||
trap 'rm -f "${STDERR}"' EXIT
|
||||
|
||||
set +e
|
||||
OUTPUT="$(osv-scanner scan source "${SCAN_ARGS[@]}" --format=json "$@" 2>"${STDERR}")"
|
||||
RC=$?
|
||||
set -e
|
||||
|
||||
case "${RC}" in
|
||||
0|1) ;;
|
||||
127) echo "osv-scanner: no supported lockfiles in scan target"; exit 0 ;;
|
||||
*)
|
||||
echo "osv-scanner: exited with code ${RC}" >&2
|
||||
[ -s "${STDERR}" ] && cat "${STDERR}" >&2
|
||||
exit "${RC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Build a JSON array of normalized severity levels for jq.
|
||||
SEVERITY_JSON="$(printf '%s' "${SEVERITY_LEVELS}" | jq -Rc '
|
||||
split(",") | map(ascii_upcase | sub("^\\s+"; "") | sub("\\s+$"; ""))
|
||||
')"
|
||||
|
||||
# Walk each vulnerability, look up its group's max_severity (numeric CVSS),
|
||||
# map to a categorical label, then filter by OSV_SEVERITY_LEVELS.
|
||||
FINDINGS="$(printf '%s' "${OUTPUT}" | jq --argjson sevs "${SEVERITY_JSON}" '
|
||||
[ .results[]?.packages[]?
|
||||
| . as $pkg
|
||||
| ($pkg.groups // []) as $groups
|
||||
| ($pkg.vulnerabilities // [])[]
|
||||
| . as $vuln
|
||||
| ([ $groups[] | select((.ids // []) | index($vuln.id)) ][0] // {}) as $group
|
||||
| (($group.max_severity // "") | tonumber? // null) as $score
|
||||
| (if $score == null then "UNKNOWN"
|
||||
elif $score >= 9.0 then "CRITICAL"
|
||||
elif $score >= 7.0 then "HIGH"
|
||||
elif $score >= 4.0 then "MEDIUM"
|
||||
elif $score > 0 then "LOW"
|
||||
else "UNKNOWN"
|
||||
end) as $label
|
||||
| {
|
||||
id: $vuln.id,
|
||||
severity: $label,
|
||||
score: $score,
|
||||
summary: ($vuln.summary // null),
|
||||
package: $pkg.package.name,
|
||||
version: $pkg.package.version,
|
||||
ecosystem: $pkg.package.ecosystem
|
||||
}
|
||||
| select(.severity as $s | $sevs | any(. == $s))
|
||||
]
|
||||
')"
|
||||
|
||||
COUNT="$(printf '%s' "${FINDINGS}" | jq 'length')"
|
||||
|
||||
# Write the findings JSON to OSV_REPORT_FILE so callers (e.g. the composite
|
||||
# action's PR-comment step) can consume the same data the gate decision uses.
|
||||
if [ -n "${OSV_REPORT_FILE:-}" ]; then
|
||||
printf '%s' "${FINDINGS}" > "${OSV_REPORT_FILE}"
|
||||
fi
|
||||
|
||||
if [ "${COUNT}" -gt 0 ]; then
|
||||
echo "osv-scanner: ${COUNT} finding(s) at severity ${SEVERITY_LEVELS}"
|
||||
printf '%s' "${FINDINGS}" | jq -r '
|
||||
.[] | " [\(.severity)\(if .score then " \(.score)" else "" end)] \(.id) \(.ecosystem)/\(.package)@\(.version) — \(.summary // "(no summary)")"
|
||||
'
|
||||
echo
|
||||
echo "To accept a finding, create osv-scanner.toml at the repo root with a reason and ignoreUntil."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "osv-scanner: no findings at severity levels: ${SEVERITY_LEVELS}"
|
||||
@@ -43,6 +43,7 @@ jobs:
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
api.github.com:443
|
||||
raw.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -63,26 +64,25 @@ jobs:
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Poetry check
|
||||
- name: uv lock check
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry check --lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Ruff lint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run ruff check . --exclude contrib
|
||||
run: uv run ruff check . --exclude contrib
|
||||
|
||||
- name: Ruff format
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run ruff format --check . --exclude contrib
|
||||
run: uv run ruff format --check . --exclude contrib
|
||||
|
||||
- name: Pylint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/
|
||||
run: uv run pylint --disable=W,C,R,E -j 0 -rn -sn src/
|
||||
|
||||
@@ -158,7 +158,7 @@ jobs:
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
|
||||
@@ -5,10 +5,16 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -57,16 +63,7 @@ jobs:
|
||||
|
||||
api-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -119,23 +116,22 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build container for ${{ matrix.arch }}
|
||||
- name: Build container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: ${{ env.API_WORKING_DIR }}
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
|
||||
- name: Scan container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -5,10 +5,24 @@ on:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-tests.yml'
|
||||
- '.github/workflows/api-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-tests.yml'
|
||||
- '.github/workflows/api-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -25,6 +39,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
@@ -42,10 +57,12 @@ jobs:
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
github.com:443
|
||||
auth.safetycli.com:443
|
||||
pyup.io:443
|
||||
data.safetycli.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -60,30 +77,34 @@ jobs:
|
||||
files: |
|
||||
api/**
|
||||
.github/workflows/api-security.yml
|
||||
.safety-policy.yml
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Bandit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r .
|
||||
# Exclude .venv because uv places the project venv inside ./api; otherwise
|
||||
# bandit would recurse into installed third-party packages.
|
||||
run: uv run bandit -q -lll -x '*_test.py,./contrib/,./.venv/' -r .
|
||||
|
||||
- name: Safety
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
# Accepted CVEs, severity threshold, and ignore expirations live in ../.safety-policy.yml
|
||||
run: poetry run safety check --policy-file ../.safety-policy.yml
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: api/uv.lock
|
||||
|
||||
- name: Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 .
|
||||
# Run even when osv-scanner reports findings so dead-code signal isn't masked by SCA failures.
|
||||
if: ${{ !cancelled() && steps.check-changes.outputs.any_changed == 'true' }}
|
||||
run: uv run vulture --exclude "contrib,tests,conftest.py,.venv" --min-confidence 100 .
|
||||
|
||||
@@ -87,6 +87,7 @@ jobs:
|
||||
files.pythonhosted.org:443
|
||||
cli.codecov.io:443
|
||||
keybase.io:443
|
||||
raw.githubusercontent.com:443
|
||||
ingest.codecov.io:443
|
||||
storage.googleapis.com:443
|
||||
o26192.ingest.us.sentry.io:443
|
||||
@@ -112,17 +113,16 @@ jobs:
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Run tests with pytest
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend
|
||||
run: uv run pytest --cov=./src/backend --cov-report=xml src/backend
|
||||
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -4,8 +4,6 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v3'
|
||||
- 'v4.*'
|
||||
- 'v5.*'
|
||||
types:
|
||||
- 'opened'
|
||||
|
||||
@@ -43,14 +43,11 @@ jobs:
|
||||
|
||||
echo "Processing release tag: $RELEASE_TAG"
|
||||
|
||||
# Remove 'v' prefix if present (e.g., v3.2.0 -> 3.2.0)
|
||||
VERSION_ONLY="${RELEASE_TAG#v}"
|
||||
|
||||
# Check if it's a minor version (X.Y.0)
|
||||
if [[ "$VERSION_ONLY" =~ ^([0-9]+)\.([0-9]+)\.0$ ]]; then
|
||||
echo "Release $RELEASE_TAG (version $VERSION_ONLY) is a minor version. Proceeding to create backport label."
|
||||
|
||||
# Extract X.Y from X.Y.0 (e.g., 5.6 from 5.6.0)
|
||||
MAJOR="${BASH_REMATCH[1]}"
|
||||
MINOR="${BASH_REMATCH[2]}"
|
||||
TWO_DIGIT_VERSION="${MAJOR}.${MINOR}"
|
||||
@@ -62,7 +59,6 @@ jobs:
|
||||
echo "Label name: $LABEL_NAME"
|
||||
echo "Label description: $LABEL_DESC"
|
||||
|
||||
# Check if label already exists
|
||||
if gh label list --repo ${{ github.repository }} --limit 1000 | grep -q "^${LABEL_NAME}[[:space:]]"; then
|
||||
echo "Label '$LABEL_NAME' already exists."
|
||||
else
|
||||
|
||||
@@ -37,10 +37,13 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# PRs only need the diff range; push to master/release walks the new range from event.before.
|
||||
# 50 is enough headroom for the longest realistic PR/push chain without paying for a full clone.
|
||||
fetch-depth: 50
|
||||
persist-credentials: false
|
||||
|
||||
- name: Scan for secrets with TruffleHog
|
||||
- name: Scan diff for secrets with TruffleHog
|
||||
# Action auto-injects --since-commit/--branch from event payload; passing them in extra_args produces duplicate flags.
|
||||
uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4
|
||||
with:
|
||||
extra_args: '--results=verified,unknown'
|
||||
extra_args: --results=verified,unknown
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
"Alan-TheGentleman"
|
||||
"alejandrobailo"
|
||||
"amitsharm"
|
||||
"andoniaf"
|
||||
# "andoniaf"
|
||||
"cesararroba"
|
||||
"danibarranqueroo"
|
||||
"HugoPBrito"
|
||||
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
org.opencontainers.image.created=${{ github.event_name == 'release' && github.event.release.published_at || github.event.head_commit.timestamp }}
|
||||
${{ github.event_name == 'release' && format('org.opencontainers.image.version={0}', env.RELEASE_TAG) || '' }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
|
||||
@@ -5,10 +5,16 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/**'
|
||||
- '.github/workflows/mcp-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/**'
|
||||
- '.github/workflows/mcp-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -56,16 +62,7 @@ jobs:
|
||||
|
||||
mcp-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -112,23 +109,22 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build MCP container for ${{ matrix.arch }}
|
||||
- name: Build MCP container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: ${{ env.MCP_WORKING_DIR }}
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
|
||||
- name: Scan MCP container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan MCP container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -86,11 +86,32 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
# The MCP server version (mcp_server/pyproject.toml) is decoupled from the Prowler release
|
||||
# version: it only changes when MCP code changes. mcp-bump-version.yml normally keeps it in
|
||||
# sync with mcp_server/CHANGELOG.md, but this publish workflow still runs on every release.
|
||||
# Pre-flight PyPI check covers the legitimate "no MCP changes for this release" case (and any
|
||||
# workflow_dispatch re-runs) without failing with HTTP 400 (version exists).
|
||||
- name: Check if prowler-mcp version already exists on PyPI
|
||||
id: pypi-check
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: |
|
||||
MCP_VERSION=$(grep '^version' pyproject.toml | head -1 | sed -E 's/^version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/')
|
||||
echo "mcp_version=${MCP_VERSION}" >> "$GITHUB_OUTPUT"
|
||||
if curl -fsS "https://pypi.org/pypi/prowler-mcp/${MCP_VERSION}/json" >/dev/null 2>&1; then
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice title=Skipping prowler-mcp publish::Version ${MCP_VERSION} already exists on PyPI; bump mcp_server/pyproject.toml to publish a new release."
|
||||
else
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice title=Publishing prowler-mcp::Version ${MCP_VERSION} not on PyPI yet; proceeding."
|
||||
fi
|
||||
|
||||
- name: Build prowler-mcp package
|
||||
if: steps.pypi-check.outputs.skip != 'true'
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-mcp package to PyPI
|
||||
if: steps.pypi-check.outputs.skip != 'true'
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
with:
|
||||
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
name: 'Nightly: ARM64 Container Builds'
|
||||
|
||||
# Mitigation for amd64-only PR container-checks: build amd64+arm64 nightly against
|
||||
# master to keep arm-specific Dockerfile regressions caught quickly. Build only —
|
||||
# no push, no Trivy (weekly checks already cover that).
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 4 * * *'
|
||||
workflow_dispatch: {}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build-arm64:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- component: sdk
|
||||
context: .
|
||||
dockerfile: ./Dockerfile
|
||||
image_name: prowler
|
||||
- component: api
|
||||
context: ./api
|
||||
dockerfile: ./api/Dockerfile
|
||||
image_name: prowler-api
|
||||
- component: ui
|
||||
context: ./ui
|
||||
dockerfile: ./ui/Dockerfile
|
||||
image_name: prowler-ui
|
||||
target: prod
|
||||
build_args: |
|
||||
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
|
||||
- component: mcp
|
||||
context: ./mcp_server
|
||||
dockerfile: ./mcp_server/Dockerfile
|
||||
image_name: prowler-mcp
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build ${{ matrix.component }} container (linux/arm64)
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: ${{ matrix.context }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
target: ${{ matrix.target }}
|
||||
push: false
|
||||
load: false
|
||||
platforms: linux/arm64
|
||||
tags: ${{ matrix.image_name }}:nightly-arm64
|
||||
build-args: ${{ matrix.build_args }}
|
||||
cache-from: type=gha,scope=arm64
|
||||
cache-to: type=gha,mode=min,scope=arm64
|
||||
|
||||
notify-failure:
|
||||
needs: build-arm64
|
||||
if: failure() && github.event_name == 'schedule'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Notify Slack on failure
|
||||
uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
|
||||
with:
|
||||
method: chat.postMessage
|
||||
token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload: |
|
||||
channel: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
text: ":rotating_light: Nightly arm64 container build failed for prowler — <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|view run>"
|
||||
errors: true
|
||||
@@ -41,10 +41,15 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
@@ -54,7 +59,7 @@ jobs:
|
||||
ui/**
|
||||
prowler/**
|
||||
mcp_server/**
|
||||
poetry.lock
|
||||
uv.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
@@ -79,9 +84,9 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (poetry.lock, pyproject.toml)
|
||||
# Check root-level dependency files (uv.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(poetry\.lock|pyproject\.toml)$" || true)
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
|
||||
@@ -45,10 +45,15 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
|
||||
@@ -36,8 +36,14 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
|
||||
@@ -40,12 +40,11 @@ jobs:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Setup Python with uv
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: '3.12'
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
@@ -339,10 +338,11 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Update poetry lock file
|
||||
echo "Updating poetry.lock file..."
|
||||
# Update uv lock file
|
||||
echo "Updating uv.lock file..."
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
cd api
|
||||
poetry lock
|
||||
uv lock
|
||||
cd ..
|
||||
|
||||
echo "✓ Prepared prowler dependency update to: $UPDATED_PROWLER_REF"
|
||||
@@ -357,7 +357,7 @@ jobs:
|
||||
base: ${{ env.BRANCH_NAME }}
|
||||
add-paths: |
|
||||
api/pyproject.toml
|
||||
api/poetry.lock
|
||||
api/uv.lock
|
||||
title: "chore(api): Update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}"
|
||||
body: |
|
||||
### Description
|
||||
@@ -366,7 +366,7 @@ jobs:
|
||||
|
||||
**Changes:**
|
||||
- Updates `api/pyproject.toml` prowler dependency from `@master` to `@${{ env.BRANCH_NAME }}`
|
||||
- Updates `api/poetry.lock` file with resolved dependencies
|
||||
- Updates `api/uv.lock` file with resolved dependencies
|
||||
|
||||
This PR should be merged into the `${{ env.BRANCH_NAME }}` release branch.
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'tests/providers/**/*_test.py'
|
||||
- '.github/workflows/sdk-check-duplicate-test-names.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
@@ -71,24 +71,26 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Check Poetry lock file
|
||||
- name: Check uv lock file
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry check --lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Lint with flake8
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api,skills
|
||||
run: uv run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude .venv,contrib,ui,api,skills,mcp_server
|
||||
|
||||
- name: Check format with black
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run black --exclude "api|ui|skills" --check .
|
||||
# mcp_server has its own pyproject and uses ruff format, exclude it so SDK black
|
||||
# does not fight ruff over rules it never formatted.
|
||||
run: uv run black --exclude "\.venv|api|ui|skills|mcp_server" --check .
|
||||
|
||||
- name: Lint with pylint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
|
||||
run: uv run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
|
||||
|
||||
@@ -3,9 +3,7 @@ name: 'SDK: Container Build and Push'
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'v3' # For v3-latest
|
||||
- 'v4.6' # For v4-latest
|
||||
- 'master' # For latest
|
||||
- 'master'
|
||||
paths-ignore:
|
||||
- '.github/**'
|
||||
- '!.github/workflows/sdk-container-build-push.yml'
|
||||
@@ -56,7 +54,6 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }}
|
||||
prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }}
|
||||
latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }}
|
||||
stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }}
|
||||
permissions:
|
||||
@@ -76,48 +73,19 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Inject poetry-bumpversion plugin
|
||||
run: pipx inject poetry poetry-bumpversion
|
||||
|
||||
- name: Get Prowler version and set tags
|
||||
id: get-prowler-version
|
||||
run: |
|
||||
PROWLER_VERSION="$(poetry version -s 2>/dev/null)"
|
||||
PROWLER_VERSION="$(grep -E '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]')"
|
||||
echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Extract major version
|
||||
PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}"
|
||||
echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Set version-specific tags
|
||||
case ${PROWLER_VERSION_MAJOR} in
|
||||
3)
|
||||
echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable"
|
||||
;;
|
||||
4)
|
||||
echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable"
|
||||
;;
|
||||
5)
|
||||
echo "latest_tag=latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v5 detected - tags: latest, stable"
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [[ "${PROWLER_VERSION_MAJOR}" != "5" ]]; then
|
||||
echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}"
|
||||
exit 1
|
||||
fi
|
||||
echo "latest_tag=latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=stable" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
notify-release-started:
|
||||
if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
@@ -228,7 +196,7 @@ jobs:
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
@@ -386,39 +354,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
dispatch-v3-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && needs.setup.outputs.prowler_version_major == '3' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Calculate short SHA
|
||||
id: short-sha
|
||||
run: echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Dispatch v3 deployment (latest)
|
||||
if: github.event_name == 'push'
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
event-type: dispatch
|
||||
client-payload: '{"version":"v3-latest","tag":"${{ steps.short-sha.outputs.short_sha }}"}'
|
||||
|
||||
- name: Dispatch v3 deployment (release)
|
||||
if: github.event_name == 'release'
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
event-type: dispatch
|
||||
client-payload: '{"version":"release","tag":"${{ needs.setup.outputs.prowler_version }}"}'
|
||||
|
||||
@@ -5,10 +5,22 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'Dockerfile*'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'Dockerfile*'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -56,16 +68,7 @@ jobs:
|
||||
|
||||
sdk-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -132,23 +135,22 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build SDK container for ${{ matrix.arch }}
|
||||
- name: Build SDK container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
|
||||
- name: Scan SDK container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan SDK container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -75,15 +75,14 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Setup Python with uv
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Build Prowler package
|
||||
run: poetry build
|
||||
run: uv build
|
||||
|
||||
- name: Publish Prowler package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
@@ -112,12 +111,11 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Setup Python with uv
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Install toml package
|
||||
run: pip install toml
|
||||
@@ -128,7 +126,7 @@ jobs:
|
||||
python util/replicate_pypi_package.py
|
||||
|
||||
- name: Build prowler-cloud package
|
||||
run: poetry build
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-cloud package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
|
||||
@@ -5,10 +5,30 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'tests/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-tests.yml'
|
||||
- '.github/workflows/sdk-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'tests/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-tests.yml'
|
||||
- '.github/workflows/sdk-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -23,6 +43,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -33,10 +54,12 @@ jobs:
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
github.com:443
|
||||
auth.safetycli.com:443
|
||||
pyup.io:443
|
||||
data.safetycli.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -71,21 +94,23 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Security scan with Bandit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run bandit -q -lll -x '*_test.py,./contrib/,./api/,./ui' -r .
|
||||
run: uv run bandit -q -lll -x '*_test.py,./.venv/,./contrib/,./api/,./ui' -r .
|
||||
|
||||
- name: Security scan with Safety
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
# Accepted CVEs, severity threshold, and ignore expirations live in .safety-policy.yml
|
||||
run: poetry run safety check -r pyproject.toml --policy-file .safety-policy.yml
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: uv.lock
|
||||
|
||||
- name: Dead code detection with Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run vulture --exclude "contrib,api,ui" --min-confidence 100 .
|
||||
# Run even when osv-scanner reports findings so dead-code signal isn't masked by SCA failures.
|
||||
if: ${{ !cancelled() && steps.check-changes.outputs.any_changed == 'true' }}
|
||||
run: uv run vulture --exclude ".venv,contrib,api,ui" --min-confidence 100 .
|
||||
|
||||
@@ -92,9 +92,9 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/aws/**
|
||||
./tests/**/aws/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Resolve AWS services under test
|
||||
if: steps.changed-aws.outputs.any_changed == 'true'
|
||||
@@ -209,11 +209,11 @@ jobs:
|
||||
echo "AWS service_paths='${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}'"
|
||||
|
||||
if [ "${STEPS_AWS_SERVICES_OUTPUTS_RUN_ALL}" = "true" ]; then
|
||||
poetry run pytest -p no:randomly -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws
|
||||
uv run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws
|
||||
elif [ -z "${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}" ]; then
|
||||
echo "No AWS service paths detected; skipping AWS tests."
|
||||
else
|
||||
poetry run pytest -p no:randomly -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml ${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}
|
||||
uv run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml ${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}
|
||||
fi
|
||||
env:
|
||||
STEPS_AWS_SERVICES_OUTPUTS_RUN_ALL: ${{ steps.aws-services.outputs.run_all }}
|
||||
@@ -237,11 +237,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/azure/**
|
||||
./tests/**/azure/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Azure tests
|
||||
if: steps.changed-azure.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure
|
||||
|
||||
- name: Upload Azure coverage to Codecov
|
||||
if: steps.changed-azure.outputs.any_changed == 'true'
|
||||
@@ -261,11 +261,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/gcp/**
|
||||
./tests/**/gcp/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run GCP tests
|
||||
if: steps.changed-gcp.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp
|
||||
|
||||
- name: Upload GCP coverage to Codecov
|
||||
if: steps.changed-gcp.outputs.any_changed == 'true'
|
||||
@@ -285,11 +285,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/kubernetes/**
|
||||
./tests/**/kubernetes/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Kubernetes tests
|
||||
if: steps.changed-kubernetes.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes
|
||||
|
||||
- name: Upload Kubernetes coverage to Codecov
|
||||
if: steps.changed-kubernetes.outputs.any_changed == 'true'
|
||||
@@ -309,11 +309,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/github/**
|
||||
./tests/**/github/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run GitHub tests
|
||||
if: steps.changed-github.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github
|
||||
|
||||
- name: Upload GitHub coverage to Codecov
|
||||
if: steps.changed-github.outputs.any_changed == 'true'
|
||||
@@ -324,6 +324,30 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-github
|
||||
files: ./github_coverage.xml
|
||||
|
||||
# Okta Provider
|
||||
- name: Check if Okta files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-okta
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/okta/**
|
||||
./tests/**/okta/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run Okta tests
|
||||
if: steps.changed-okta.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/okta --cov-report=xml:okta_coverage.xml tests/providers/okta
|
||||
|
||||
- name: Upload Okta coverage to Codecov
|
||||
if: steps.changed-okta.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-okta
|
||||
files: ./okta_coverage.xml
|
||||
|
||||
# NHN Provider
|
||||
- name: Check if NHN files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -333,11 +357,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/nhn/**
|
||||
./tests/**/nhn/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run NHN tests
|
||||
if: steps.changed-nhn.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn
|
||||
|
||||
- name: Upload NHN coverage to Codecov
|
||||
if: steps.changed-nhn.outputs.any_changed == 'true'
|
||||
@@ -357,11 +381,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/m365/**
|
||||
./tests/**/m365/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run M365 tests
|
||||
if: steps.changed-m365.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365
|
||||
|
||||
- name: Upload M365 coverage to Codecov
|
||||
if: steps.changed-m365.outputs.any_changed == 'true'
|
||||
@@ -381,11 +405,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/iac/**
|
||||
./tests/**/iac/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run IaC tests
|
||||
if: steps.changed-iac.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac
|
||||
|
||||
- name: Upload IaC coverage to Codecov
|
||||
if: steps.changed-iac.outputs.any_changed == 'true'
|
||||
@@ -405,11 +429,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/mongodbatlas/**
|
||||
./tests/**/mongodbatlas/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run MongoDB Atlas tests
|
||||
if: steps.changed-mongodbatlas.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodbatlas_coverage.xml tests/providers/mongodbatlas
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodbatlas_coverage.xml tests/providers/mongodbatlas
|
||||
|
||||
- name: Upload MongoDB Atlas coverage to Codecov
|
||||
if: steps.changed-mongodbatlas.outputs.any_changed == 'true'
|
||||
@@ -429,11 +453,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/oraclecloud/**
|
||||
./tests/**/oraclecloud/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run OCI tests
|
||||
if: steps.changed-oraclecloud.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oraclecloud_coverage.xml tests/providers/oraclecloud
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oraclecloud_coverage.xml tests/providers/oraclecloud
|
||||
|
||||
- name: Upload OCI coverage to Codecov
|
||||
if: steps.changed-oraclecloud.outputs.any_changed == 'true'
|
||||
@@ -453,11 +477,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/openstack/**
|
||||
./tests/**/openstack/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run OpenStack tests
|
||||
if: steps.changed-openstack.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/openstack --cov-report=xml:openstack_coverage.xml tests/providers/openstack
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/openstack --cov-report=xml:openstack_coverage.xml tests/providers/openstack
|
||||
|
||||
- name: Upload OpenStack coverage to Codecov
|
||||
if: steps.changed-openstack.outputs.any_changed == 'true'
|
||||
@@ -477,11 +501,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/googleworkspace/**
|
||||
./tests/**/googleworkspace/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Google Workspace tests
|
||||
if: steps.changed-googleworkspace.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/googleworkspace --cov-report=xml:googleworkspace_coverage.xml tests/providers/googleworkspace
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/googleworkspace --cov-report=xml:googleworkspace_coverage.xml tests/providers/googleworkspace
|
||||
|
||||
- name: Upload Google Workspace coverage to Codecov
|
||||
if: steps.changed-googleworkspace.outputs.any_changed == 'true'
|
||||
@@ -501,11 +525,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/vercel/**
|
||||
./tests/**/vercel/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Vercel tests
|
||||
if: steps.changed-vercel.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/vercel --cov-report=xml:vercel_coverage.xml tests/providers/vercel
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/vercel --cov-report=xml:vercel_coverage.xml tests/providers/vercel
|
||||
|
||||
- name: Upload Vercel coverage to Codecov
|
||||
if: steps.changed-vercel.outputs.any_changed == 'true'
|
||||
@@ -525,11 +549,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/lib/**
|
||||
./tests/lib/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Lib tests
|
||||
if: steps.changed-lib.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib
|
||||
run: uv run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib
|
||||
|
||||
- name: Upload Lib coverage to Codecov
|
||||
if: steps.changed-lib.outputs.any_changed == 'true'
|
||||
@@ -549,11 +573,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/config/**
|
||||
./tests/config/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Config tests
|
||||
if: steps.changed-config.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config
|
||||
run: uv run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config
|
||||
|
||||
- name: Upload Config coverage to Codecov
|
||||
if: steps.changed-config.outputs.any_changed == 'true'
|
||||
|
||||
@@ -151,7 +151,7 @@ jobs:
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
|
||||
@@ -5,10 +5,16 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/**'
|
||||
- '.github/workflows/ui-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/**'
|
||||
- '.github/workflows/ui-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -57,16 +63,7 @@ jobs:
|
||||
|
||||
ui-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -114,7 +111,7 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build UI container for ${{ matrix.arch }}
|
||||
- name: Build UI container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
@@ -122,18 +119,17 @@ jobs:
|
||||
target: prod
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
|
||||
|
||||
- name: Scan UI container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan UI container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -15,6 +15,10 @@ on:
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
@@ -126,6 +130,12 @@ jobs:
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
|
||||
- name: Build API image from current code
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
# which lags behind PR changes; build locally so E2E exercises the API image
|
||||
# produced by this PR.
|
||||
run: docker build -t prowlercloud/prowler-api:latest ./api
|
||||
|
||||
- name: Start API services
|
||||
run: |
|
||||
export PROWLER_API_VERSION=latest
|
||||
@@ -154,7 +164,7 @@ jobs:
|
||||
for fixture in api/fixtures/dev/*.json; do
|
||||
if [ -f "$fixture" ]; then
|
||||
echo "Loading $fixture"
|
||||
poetry run python manage.py loaddata "$fixture" --database admin
|
||||
uv run python manage.py loaddata "$fixture" --database admin
|
||||
fi
|
||||
done
|
||||
'
|
||||
@@ -266,7 +276,7 @@ jobs:
|
||||
with:
|
||||
name: playwright-report
|
||||
path: ui/playwright-report/
|
||||
retention-days: 30
|
||||
retention-days: 7
|
||||
|
||||
- name: Cleanup services
|
||||
if: always()
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
name: 'UI: Security'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/package.json'
|
||||
- 'ui/pnpm-lock.yaml'
|
||||
- '.github/workflows/ui-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/package.json'
|
||||
- 'ui/pnpm-lock.yaml'
|
||||
- '.github/workflows/ui-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Check for UI dependency changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
ui/package.json
|
||||
ui/pnpm-lock.yaml
|
||||
.github/workflows/ui-security.yml
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: ui/pnpm-lock.yaml
|
||||
@@ -1,14 +1,14 @@
|
||||
name: 'UI: Tests'
|
||||
name: "UI: Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
- "master"
|
||||
- "v5.*"
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
- "master"
|
||||
- "v5.*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -16,7 +16,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: '24.13.0'
|
||||
NODE_VERSION: "24.13.0"
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -42,6 +42,9 @@ jobs:
|
||||
fonts.gstatic.com:443
|
||||
api.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
cdn.playwright.dev:443
|
||||
objects.githubusercontent.com:443
|
||||
playwright.download.prss.microsoft.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -129,11 +132,15 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run healthcheck
|
||||
|
||||
- name: Run pnpm audit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run audit
|
||||
|
||||
- name: Run unit tests (all - critical paths changed)
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.critical-changes.outputs.any_changed == 'true'
|
||||
run: |
|
||||
echo "Critical paths changed - running ALL unit tests"
|
||||
pnpm run test:run
|
||||
pnpm run test:unit
|
||||
|
||||
- name: Run unit tests (related to changes only)
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.critical-changes.outputs.any_changed != 'true' && steps.changed-source.outputs.all_changed_files != ''
|
||||
@@ -142,7 +149,7 @@ jobs:
|
||||
echo "${STEPS_CHANGED_SOURCE_OUTPUTS_ALL_CHANGED_FILES}"
|
||||
# Convert space-separated to vitest related format (remove ui/ prefix for relative paths)
|
||||
CHANGED_FILES=$(echo "${STEPS_CHANGED_SOURCE_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | sed 's|^ui/||' | tr '\n' ' ')
|
||||
pnpm exec vitest related $CHANGED_FILES --run
|
||||
pnpm exec vitest related $CHANGED_FILES --run --project unit
|
||||
env:
|
||||
STEPS_CHANGED_SOURCE_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-source.outputs.all_changed_files }}
|
||||
|
||||
@@ -150,7 +157,25 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.critical-changes.outputs.any_changed != 'true' && steps.changed-source.outputs.all_changed_files == ''
|
||||
run: |
|
||||
echo "Only test files changed - running ALL unit tests"
|
||||
pnpm run test:run
|
||||
pnpm run test:unit
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: playwright-cache
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-playwright-chromium-
|
||||
|
||||
- name: Install Playwright Chromium browser
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.playwright-cache.outputs.cache-hit != 'true'
|
||||
run: pnpm exec playwright install chromium
|
||||
|
||||
- name: Run browser tests
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run test:browser
|
||||
|
||||
- name: Build application
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -8,6 +8,7 @@ rules:
|
||||
- docs-bump-version.yml
|
||||
- issue-triage.lock.yml
|
||||
- mcp-container-build-push.yml
|
||||
- nightly-arm64-container-builds.yml
|
||||
- pr-merged.yml
|
||||
- prepare-release.yml
|
||||
- sdk-bump-version.yml
|
||||
|
||||
+49
-50
@@ -1,17 +1,34 @@
|
||||
# Priority tiers (lower = runs first, same priority = concurrent):
|
||||
# P0 — fast file fixers
|
||||
# P10 — validators and guards
|
||||
# P20 — auto-formatters
|
||||
# P30 — linters
|
||||
# P40 — security scanners
|
||||
# P50 — dependency validation
|
||||
|
||||
default_install_hook_types: [pre-commit]
|
||||
|
||||
repos:
|
||||
## GENERAL (prek built-in — no external repo needed)
|
||||
- repo: builtin
|
||||
hooks:
|
||||
- id: check-merge-conflict
|
||||
priority: 10
|
||||
- id: check-yaml
|
||||
args: ["--allow-multiple-documents"]
|
||||
exclude: (prowler/config/llm_config.yaml|contrib/)
|
||||
priority: 10
|
||||
- id: check-json
|
||||
priority: 10
|
||||
- id: end-of-file-fixer
|
||||
priority: 0
|
||||
- id: trailing-whitespace
|
||||
priority: 0
|
||||
- id: no-commit-to-branch
|
||||
priority: 10
|
||||
- id: pretty-format-json
|
||||
args: ["--autofix", --no-sort-keys, --no-ensure-ascii]
|
||||
priority: 10
|
||||
|
||||
## TOML
|
||||
- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
|
||||
@@ -20,13 +37,17 @@ repos:
|
||||
- id: pretty-format-toml
|
||||
args: [--autofix]
|
||||
files: pyproject.toml
|
||||
priority: 20
|
||||
|
||||
## GITHUB ACTIONS
|
||||
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
||||
rev: v1.24.1
|
||||
hooks:
|
||||
- id: zizmor
|
||||
files: ^\.github/
|
||||
# zizmor only audits workflows, composite actions and dependabot
|
||||
# config; broader paths trip exit 3 ("no audit was performed").
|
||||
files: ^\.github/(workflows|actions)/.+\.ya?ml$|^\.github/dependabot\.ya?ml$
|
||||
priority: 30
|
||||
|
||||
## BASH
|
||||
- repo: https://github.com/koalaman/shellcheck-precommit
|
||||
@@ -34,6 +55,7 @@ repos:
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
exclude: contrib
|
||||
priority: 30
|
||||
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/)
|
||||
- repo: https://github.com/myint/autoflake
|
||||
@@ -42,12 +64,8 @@ repos:
|
||||
- id: autoflake
|
||||
name: "SDK - autoflake"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args:
|
||||
[
|
||||
"--in-place",
|
||||
"--remove-all-unused-imports",
|
||||
"--remove-unused-variable",
|
||||
]
|
||||
args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variable"]
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 8.0.1
|
||||
@@ -56,6 +74,7 @@ repos:
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 26.3.1
|
||||
@@ -63,6 +82,7 @@ repos:
|
||||
- id: black
|
||||
name: "SDK - black"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 7.3.0
|
||||
@@ -71,6 +91,7 @@ repos:
|
||||
name: "SDK - flake8"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
priority: 30
|
||||
|
||||
## PYTHON — API + MCP Server (ruff)
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
@@ -80,37 +101,29 @@ repos:
|
||||
name: "API + MCP - ruff check"
|
||||
files: { glob: ["{api,mcp_server}/**/*.py"] }
|
||||
args: ["--fix"]
|
||||
priority: 30
|
||||
- id: ruff-format
|
||||
name: "API + MCP - ruff format"
|
||||
files: { glob: ["{api,mcp_server}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
## PYTHON — Poetry
|
||||
- repo: https://github.com/python-poetry/poetry
|
||||
rev: 2.3.4
|
||||
## PYTHON — uv (API + SDK)
|
||||
- repo: https://github.com/astral-sh/uv-pre-commit
|
||||
rev: 0.11.14
|
||||
hooks:
|
||||
- id: poetry-check
|
||||
name: API - poetry-check
|
||||
args: ["--directory=./api"]
|
||||
files: { glob: ["api/{pyproject.toml,poetry.lock}"] }
|
||||
- id: uv-lock
|
||||
name: API - uv-lock
|
||||
args: ["--check", "--project=./api"]
|
||||
files: { glob: ["api/{pyproject.toml,uv.lock}"] }
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
- id: poetry-lock
|
||||
name: API - poetry-lock
|
||||
args: ["--directory=./api"]
|
||||
files: { glob: ["api/{pyproject.toml,poetry.lock}"] }
|
||||
pass_filenames: false
|
||||
|
||||
- id: poetry-check
|
||||
name: SDK - poetry-check
|
||||
args: ["--directory=./"]
|
||||
files: { glob: ["{pyproject.toml,poetry.lock}"] }
|
||||
pass_filenames: false
|
||||
|
||||
- id: poetry-lock
|
||||
name: SDK - poetry-lock
|
||||
args: ["--directory=./"]
|
||||
files: { glob: ["{pyproject.toml,poetry.lock}"] }
|
||||
- id: uv-lock
|
||||
name: SDK - uv-lock
|
||||
args: ["--check", "--project=./"]
|
||||
files: { glob: ["{pyproject.toml,uv.lock}"] }
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
## CONTAINERS
|
||||
- repo: https://github.com/hadolint/hadolint
|
||||
@@ -118,6 +131,7 @@ repos:
|
||||
hooks:
|
||||
- id: hadolint
|
||||
args: ["--ignore=DL3013"]
|
||||
priority: 30
|
||||
|
||||
## LOCAL HOOKS
|
||||
- repo: local
|
||||
@@ -128,6 +142,7 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
priority: 30
|
||||
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
@@ -138,6 +153,7 @@ repos:
|
||||
language: system
|
||||
pass_filenames: false
|
||||
stages: ["pre-commit", "pre-push"]
|
||||
priority: 40
|
||||
|
||||
- id: bandit
|
||||
name: bandit
|
||||
@@ -146,26 +162,8 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
files: '.*\.py'
|
||||
exclude:
|
||||
{ glob: ["{contrib,skills}/**", "**/.venv/**", "**/*_test.py"] }
|
||||
|
||||
- id: safety
|
||||
name: safety
|
||||
description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities"
|
||||
# Accepted CVEs, severity threshold, and ignore expirations live in .safety-policy.yml
|
||||
entry: safety check --policy-file .safety-policy.yml
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files:
|
||||
{
|
||||
glob:
|
||||
[
|
||||
"**/pyproject.toml",
|
||||
"**/poetry.lock",
|
||||
"**/requirements*.txt",
|
||||
".safety-policy.yml",
|
||||
],
|
||||
}
|
||||
exclude: { glob: ["{contrib,skills}/**", "**/.venv/**", "**/*_test.py"] }
|
||||
priority: 40
|
||||
|
||||
- id: vulture
|
||||
name: vulture
|
||||
@@ -174,3 +172,4 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
files: '.*\.py'
|
||||
priority: 40
|
||||
|
||||
+2
-6
@@ -11,15 +11,11 @@ build:
|
||||
python: "3.11"
|
||||
jobs:
|
||||
post_create_environment:
|
||||
# Install poetry
|
||||
# https://python-poetry.org/docs/#installing-manually
|
||||
- python -m pip install poetry==2.3.4
|
||||
- python -m pip install uv==0.11.14
|
||||
post_install:
|
||||
# Install dependencies with 'docs' dependency group
|
||||
# https://python-poetry.org/docs/managing-dependencies/#dependency-groups
|
||||
# VIRTUAL_ENV needs to be set manually for now.
|
||||
# See https://github.com/readthedocs/readthedocs.org/pull/11152/
|
||||
- VIRTUAL_ENV=${READTHEDOCS_VIRTUALENV_PATH} python -m poetry install --only=docs
|
||||
- VIRTUAL_ENV=${READTHEDOCS_VIRTUALENV_PATH} uv sync --group docs --no-install-project
|
||||
|
||||
mkdocs:
|
||||
configuration: mkdocs.yml
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# Safety policy for `safety check` (Safety CLI 3.x, v2 schema).
|
||||
# Applied in: .pre-commit-config.yaml, .github/workflows/api-security.yml,
|
||||
# .github/workflows/sdk-security.yml via `--policy-file`.
|
||||
#
|
||||
# Validate: poetry run safety validate policy_file --path .safety-policy.yml
|
||||
|
||||
security:
|
||||
# Scan unpinned requirements too. Prowler pins via poetry.lock, so this is
|
||||
# defensive against accidental unpinned entries.
|
||||
ignore-unpinned-requirements: False
|
||||
|
||||
# CVSS severity filter. 7 = report only HIGH (7.0–8.9) and CRITICAL (9.0–10.0).
|
||||
# Reference: 9=CRITICAL only, 7=CRITICAL+HIGH, 4=CRITICAL+HIGH+MEDIUM.
|
||||
ignore-cvss-severity-below: 7
|
||||
|
||||
# Unknown severity is unrated, not safe. Keep False so unrated CVEs still fail
|
||||
# the build and get a human eye. Flip to True only if noise is unmanageable.
|
||||
ignore-cvss-unknown-severity: False
|
||||
|
||||
# Fail the build when a non-ignored vulnerability is found.
|
||||
continue-on-vulnerability-error: False
|
||||
|
||||
# Explicit accepted vulnerabilities. Each entry MUST have a reason and an
|
||||
# expiry. Expired entries fail the scan, forcing re-audit.
|
||||
ignore-vulnerabilities:
|
||||
77744:
|
||||
reason: "Botocore requires urllib3 1.X. Remove once upgraded to urllib3 2.X."
|
||||
expires: '2026-10-22'
|
||||
77745:
|
||||
reason: "Botocore requires urllib3 1.X. Remove once upgraded to urllib3 2.X."
|
||||
expires: '2026-10-22'
|
||||
79023:
|
||||
reason: "knack ReDoS; blocked until azure-cli-core (via cartography) allows knack >=0.13.0."
|
||||
expires: '2026-10-22'
|
||||
79027:
|
||||
reason: "knack ReDoS; blocked until azure-cli-core (via cartography) allows knack >=0.13.0."
|
||||
expires: '2026-10-22'
|
||||
86217:
|
||||
reason: "alibabacloud-tea-openapi==0.4.3 blocks upgrade to cryptography >=46.0.0."
|
||||
expires: '2026-10-22'
|
||||
71600:
|
||||
reason: "CVE-2024-1135 false positive. Fixed in gunicorn 22.0.0; project uses 23.0.0."
|
||||
expires: '2026-10-22'
|
||||
70612:
|
||||
reason: "TBD - audit required. Reason not documented in prior --ignore list."
|
||||
expires: '2026-07-22'
|
||||
66963:
|
||||
reason: "TBD - audit required. Reason not documented in prior --ignore list."
|
||||
expires: '2026-07-22'
|
||||
74429:
|
||||
reason: "TBD - audit required. Reason not documented in prior --ignore list."
|
||||
expires: '2026-07-22'
|
||||
76352:
|
||||
reason: "TBD - audit required. Reason not documented in prior --ignore list."
|
||||
expires: '2026-07-22'
|
||||
76353:
|
||||
reason: "TBD - audit required. Reason not documented in prior --ignore list."
|
||||
expires: '2026-07-22'
|
||||
@@ -15,7 +15,7 @@ Use these skills for detailed patterns on-demand:
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) |
|
||||
| `nextjs-15` | App Router, Server Actions, streaming | [SKILL.md](skills/nextjs-15/SKILL.md) |
|
||||
| `nextjs-16` | App Router, Server Actions, proxy.ts, streaming | [SKILL.md](skills/nextjs-16/SKILL.md) |
|
||||
| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) |
|
||||
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
|
||||
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
|
||||
@@ -60,11 +60,14 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding a compliance output formatter (per-provider class + table dispatcher) | `prowler-compliance` |
|
||||
| Adding indexes or constraints to database tables | `django-migration-psql` |
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-15` |
|
||||
| App Router / Server Actions | `nextjs-16` |
|
||||
| Auditing check-to-requirement mappings as a cloud auditor | `prowler-compliance` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Configuring MCP servers in agentic workflows | `gh-aw` |
|
||||
@@ -78,6 +81,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating new checks | `prowler-sdk-check` |
|
||||
| Creating new skills | `skill-creator` |
|
||||
| Creating or reviewing Django migrations | `django-migration-psql` |
|
||||
| Creating/modifying Prowler UI components | `prowler-ui` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
@@ -85,6 +89,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Debugging gh-aw compilation errors | `gh-aw` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| Fixing bug | `tdd` |
|
||||
| Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs) | `prowler-compliance` |
|
||||
| General Prowler development questions | `prowler` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Implementing feature | `tdd` |
|
||||
@@ -102,6 +107,8 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Testing hooks or utilities | `vitest` |
|
||||
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
|
||||
@@ -129,6 +136,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Writing React components | `react-19` |
|
||||
| Writing TypeScript types/interfaces | `typescript` |
|
||||
| Writing Vitest tests | `vitest` |
|
||||
| Writing data backfill or data migration | `django-migration-psql` |
|
||||
| Writing documentation | `prowler-docs` |
|
||||
| Writing unit tests for UI | `vitest` |
|
||||
|
||||
@@ -140,9 +148,9 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure,
|
||||
|
||||
| Component | Location | Tech Stack |
|
||||
|-----------|----------|------------|
|
||||
| SDK | `prowler/` | Python 3.10+, Poetry 2.3+ |
|
||||
| SDK | `prowler/` | Python 3.10+, uv |
|
||||
| API | `api/` | Django 5.1, DRF, Celery |
|
||||
| UI | `ui/` | Next.js 15, React 19, Tailwind 4 |
|
||||
| UI | `ui/` | Next.js 16, React 19, Tailwind 4 |
|
||||
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
|
||||
| Dashboard | `dashboard/` | Dash, Plotly |
|
||||
|
||||
@@ -152,13 +160,13 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure,
|
||||
|
||||
```bash
|
||||
# Setup
|
||||
poetry install --with dev
|
||||
poetry run prek install
|
||||
uv sync
|
||||
uv run prek install
|
||||
|
||||
# Code quality
|
||||
poetry run make lint
|
||||
poetry run make format
|
||||
poetry run prek run --all-files
|
||||
uv run make lint
|
||||
uv run make format
|
||||
uv run prek run --all-files
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+29
-6
@@ -1,11 +1,34 @@
|
||||
# Do you want to learn on how to...
|
||||
|
||||
- Contribute with your code or fixes to Prowler
|
||||
- Create a new check for a provider
|
||||
- Create a new security compliance framework
|
||||
- Add a custom output format
|
||||
- Add a new integration
|
||||
- Contribute with documentation
|
||||
- [Contribute with your code or fixes to Prowler](https://docs.prowler.com/developer-guide/introduction)
|
||||
- [Create a new provider](https://docs.prowler.com/developer-guide/provider)
|
||||
- [Create a new service](https://docs.prowler.com/developer-guide/services)
|
||||
- [Create a new check for a provider](https://docs.prowler.com/developer-guide/checks)
|
||||
- [Create a new security compliance framework](https://docs.prowler.com/developer-guide/security-compliance-framework)
|
||||
- [Add a custom output format](https://docs.prowler.com/developer-guide/outputs)
|
||||
- [Add a new integration](https://docs.prowler.com/developer-guide/integrations)
|
||||
- [Contribute with documentation](https://docs.prowler.com/developer-guide/documentation)
|
||||
- [Write unit tests](https://docs.prowler.com/developer-guide/unit-testing)
|
||||
- [Write integration tests](https://docs.prowler.com/developer-guide/integration-testing)
|
||||
- [Write end-to-end tests](https://docs.prowler.com/developer-guide/end2end-testing)
|
||||
- [Debug Prowler](https://docs.prowler.com/developer-guide/debugging)
|
||||
- [Configure checks](https://docs.prowler.com/developer-guide/configurable-checks)
|
||||
- [Rename checks](https://docs.prowler.com/developer-guide/renaming-checks)
|
||||
- [Follow the check metadata guidelines](https://docs.prowler.com/developer-guide/check-metadata-guidelines)
|
||||
- [Extend the MCP server](https://docs.prowler.com/developer-guide/mcp-server)
|
||||
- [Extend Lighthouse AI](https://docs.prowler.com/developer-guide/lighthouse-architecture)
|
||||
- [Add AI skills](https://docs.prowler.com/developer-guide/ai-skills)
|
||||
|
||||
Provider-specific developer notes:
|
||||
|
||||
- [AWS](https://docs.prowler.com/developer-guide/aws-details)
|
||||
- [Azure](https://docs.prowler.com/developer-guide/azure-details)
|
||||
- [Google Cloud](https://docs.prowler.com/developer-guide/gcp-details)
|
||||
- [Alibaba Cloud](https://docs.prowler.com/developer-guide/alibabacloud-details)
|
||||
- [Kubernetes](https://docs.prowler.com/developer-guide/kubernetes-details)
|
||||
- [Microsoft 365](https://docs.prowler.com/developer-guide/m365-details)
|
||||
- [GitHub](https://docs.prowler.com/developer-guide/github-details)
|
||||
- [LLM](https://docs.prowler.com/developer-guide/llm-details)
|
||||
|
||||
Want some swag as appreciation for your contribution?
|
||||
|
||||
|
||||
+7
-7
@@ -6,7 +6,7 @@ LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.69.2
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
@@ -78,7 +78,7 @@ WORKDIR /home/prowler
|
||||
# Copy necessary files
|
||||
COPY prowler/ /home/prowler/prowler/
|
||||
COPY dashboard/ /home/prowler/dashboard/
|
||||
COPY pyproject.toml /home/prowler
|
||||
COPY pyproject.toml uv.lock /home/prowler/
|
||||
COPY README.md /home/prowler/
|
||||
COPY prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
@@ -87,17 +87,17 @@ ENV HOME='/home/prowler'
|
||||
ENV PATH="${HOME}/.local/bin:${PATH}"
|
||||
#hadolint ignore=DL3013
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir poetry==2.3.4
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
|
||||
RUN poetry install --compile && \
|
||||
rm -rf ~/.cache/pip
|
||||
RUN uv sync --compile-bytecode && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
# Install PowerShell modules
|
||||
RUN poetry run python prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
RUN .venv/bin/python prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
# Remove deprecated dash dependencies
|
||||
RUN pip uninstall dash-html-components -y && \
|
||||
pip uninstall dash-core-components -y
|
||||
|
||||
USER prowler
|
||||
ENTRYPOINT ["poetry", "run", "prowler"]
|
||||
ENTRYPOINT [".venv/bin/prowler"]
|
||||
|
||||
@@ -23,7 +23,7 @@ format: ## Format Code
|
||||
|
||||
lint: ## Lint Code
|
||||
@echo "Running flake8..."
|
||||
flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib
|
||||
flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude .venv,contrib
|
||||
@echo "Running black... "
|
||||
black --check .
|
||||
@echo "Running pylint..."
|
||||
@@ -35,7 +35,7 @@ pypi-clean: ## Delete the distribution files
|
||||
|
||||
pypi-build: ## Build package
|
||||
$(MAKE) pypi-clean && \
|
||||
poetry build
|
||||
uv build
|
||||
|
||||
pypi-upload: ## Upload package
|
||||
python3 -m twine upload --repository pypi dist/*
|
||||
@@ -56,4 +56,3 @@ run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, MCP,
|
||||
|
||||
##@ Development Environment
|
||||
build-and-run-api-dev: build-no-cache-dev run-api-dev
|
||||
|
||||
|
||||
@@ -104,22 +104,23 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 572 | 83 | 41 | 17 | Official | UI, API, CLI |
|
||||
| Azure | 165 | 20 | 18 | 13 | Official | UI, API, CLI |
|
||||
| GCP | 100 | 13 | 15 | 11 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 7 | 9 | Official | UI, API, CLI |
|
||||
| GitHub | 21 | 2 | 1 | 2 | Official | UI, API, CLI |
|
||||
| M365 | 89 | 9 | 4 | 5 | Official | UI, API, CLI |
|
||||
| OCI | 48 | 13 | 3 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 61 | 9 | 3 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 2 | 0 | 5 | Official | UI, API, CLI |
|
||||
| AWS | 595 | 84 | 43 | 17 | Official | UI, API, CLI |
|
||||
| Azure | 167 | 22 | 19 | 16 | Official | UI, API, CLI |
|
||||
| GCP | 102 | 18 | 17 | 12 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 7 | 11 | Official | UI, API, CLI |
|
||||
| GitHub | 24 | 3 | 1 | 5 | Official | UI, API, CLI |
|
||||
| M365 | 101 | 10 | 4 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 51 | 14 | 4 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 61 | 9 | 4 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 0 | 5 | Official | UI, API, CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 8 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| Image | N/A | N/A | N/A | N/A | Official | CLI, API |
|
||||
| Google Workspace | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| OpenStack | 27 | 4 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Vercel | 30 | 6 | 0 | 5 | Official | CLI |
|
||||
| Google Workspace | 25 | 4 | 2 | 4 | Official | UI, API, CLI |
|
||||
| OpenStack | 34 | 5 | 0 | 9 | Official | UI, API, CLI |
|
||||
| Vercel | 26 | 6 | 0 | 5 | Official | UI, API, CLI |
|
||||
| Okta | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
@@ -176,7 +177,7 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
**Requirements**
|
||||
|
||||
* `git` installed.
|
||||
* `poetry` v2 installed: [poetry installation](https://python-poetry.org/docs/#installation).
|
||||
* `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
* `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
@@ -185,8 +186,8 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/api
|
||||
poetry install
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
set -a
|
||||
source .env
|
||||
docker compose up postgres valkey -d
|
||||
@@ -194,11 +195,6 @@ cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
gunicorn -c config/guniconf.py config.wsgi:application
|
||||
```
|
||||
> [!IMPORTANT]
|
||||
> As of Poetry v2.0.0, the `poetry shell` command has been deprecated. Use `poetry env activate` instead for environment activation.
|
||||
>
|
||||
> If your Poetry version is below v2.0.0, continue using `poetry shell` to activate your environment.
|
||||
> For further guidance, refer to the Poetry Environment Activation Guide https://python-poetry.org/docs/managing-environments/#activating-the-environment.
|
||||
|
||||
> After completing the setup, access the API documentation at http://localhost:8080/api/v1/docs.
|
||||
|
||||
@@ -207,8 +203,8 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/api
|
||||
poetry install
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
set -a
|
||||
source .env
|
||||
cd src/backend
|
||||
@@ -220,8 +216,8 @@ python -m celery -A config.celery worker -l info -E
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/api
|
||||
poetry install
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
set -a
|
||||
source .env
|
||||
cd src/backend
|
||||
@@ -282,24 +278,18 @@ The container images are available here:
|
||||
|
||||
### From GitHub
|
||||
|
||||
Python >=3.10, <3.13 is required with pip and Poetry:
|
||||
Python >=3.10, <3.13 is required with [uv](https://docs.astral.sh/uv/):
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler
|
||||
eval $(poetry env activate)
|
||||
poetry install
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
python prowler-cli.py -v
|
||||
```
|
||||
> [!IMPORTANT]
|
||||
> To clone Prowler on Windows, configure Git to support long file paths by running the following command: `git config core.longpaths true`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> As of Poetry v2.0.0, the `poetry shell` command has been deprecated. Use `poetry env activate` instead for environment activation.
|
||||
>
|
||||
> If your Poetry version is below v2.0.0, continue using `poetry shell` to activate your environment.
|
||||
> For further guidance, refer to the Poetry Environment Activation Guide https://python-poetry.org/docs/managing-environments/#activating-the-environment.
|
||||
|
||||
# 🛡️ GitHub Action
|
||||
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
|
||||
+1
-1
@@ -167,7 +167,7 @@ runs:
|
||||
|
||||
- name: Upload SARIF to GitHub Code Scanning
|
||||
if: always() && inputs.upload-sarif == 'true' && steps.find-sarif.outputs.sarif_path != ''
|
||||
uses: github/codeql-action/upload-sarif@d4b3ca9fa7f69d38bfcd667bdc45bc373d16277e # v4
|
||||
uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
sarif_file: ${{ steps.find-sarif.outputs.sarif_path }}
|
||||
category: ${{ inputs.sarif-category }}
|
||||
|
||||
+8
-8
@@ -124,24 +124,24 @@ api/src/backend/
|
||||
|
||||
```bash
|
||||
# Development
|
||||
poetry run python src/backend/manage.py runserver
|
||||
poetry run celery -A config.celery worker -l INFO
|
||||
uv run python src/backend/manage.py runserver
|
||||
uv run celery -A config.celery worker -l INFO
|
||||
|
||||
# Database
|
||||
poetry run python src/backend/manage.py makemigrations
|
||||
poetry run python src/backend/manage.py migrate
|
||||
uv run python src/backend/manage.py makemigrations
|
||||
uv run python src/backend/manage.py migrate
|
||||
|
||||
# Testing & Linting
|
||||
poetry run pytest -x --tb=short
|
||||
poetry run make lint
|
||||
uv run pytest -x --tb=short
|
||||
uv run make lint
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## QA CHECKLIST
|
||||
|
||||
- [ ] `poetry run pytest` passes
|
||||
- [ ] `poetry run make lint` passes
|
||||
- [ ] `uv run pytest` passes
|
||||
- [ ] `uv run make lint` passes
|
||||
- [ ] Migrations created if models changed
|
||||
- [ ] New endpoints have `@extend_schema` decorators
|
||||
- [ ] RLS properly applied for tenant data
|
||||
|
||||
+55
-1
@@ -2,6 +2,60 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.28.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- GIN index on `findings(categories, resource_services, resource_regions, resource_types)` to speed up `/api/v1/finding-groups` array filters [(#11001)](https://github.com/prowler-cloud/prowler/pull/11001)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Replace `poetry` with `uv` (`0.11.14`) as the API package manager; migrate `pyproject.toml` to `[dependency-groups]` and regenerate as `uv.lock` [(#10775)](https://github.com/prowler-cloud/prowler/pull/10775)
|
||||
- Remove orphaned `gin_resources_search_idx` declaration from `Resource.Meta.indexes` (DB index dropped in `0072_drop_unused_indexes`) [(#11001)](https://github.com/prowler-cloud/prowler/pull/11001)
|
||||
|
||||
---
|
||||
|
||||
## [1.27.2] (Prowler UNRELEASED)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: BEDROCK-001 and BEDROCK-002 now target roles trusting `bedrock-agentcore.amazonaws.com` instead of `bedrock.amazonaws.com`, eliminating false positives against regular Bedrock service roles (Agents, Knowledge Bases, model invocation) [(#11141)](https://github.com/prowler-cloud/prowler/pull/11141)
|
||||
|
||||
---
|
||||
|
||||
## [1.27.1] (Prowler v5.26.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `POST /api/v1/scans` was intermittently failing with `Scan matching query does not exist` in the `scan-perform` worker; the Celery task is now published via `transaction.on_commit` so the worker cannot read the Scan before the dispatch-wide transaction commits [(#11122)](https://github.com/prowler-cloud/prowler/pull/11122)
|
||||
|
||||
---
|
||||
|
||||
## [1.27.0] (Prowler v5.26.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `scan-reset-ephemeral-resources` post-scan task zeroes `failed_findings_count` for resources missing from the latest full-scope scan, keeping ephemeral resources from polluting the Resources page sort [(#10929)](https://github.com/prowler-cloud/prowler/pull/10929)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- ASD Essential Eight (AWS) compliance framework support [(#10982)](https://github.com/prowler-cloud/prowler/pull/10982)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `trivy` binary from 0.69.2 to 0.70.0 and `cryptography` from 46.0.6 to 46.0.7 (transitive via prowler SDK) in the API image for CVE-2026-33186 and CVE-2026-39892 [(#10978)](https://github.com/prowler-cloud/prowler/pull/10978)
|
||||
|
||||
---
|
||||
|
||||
## [1.26.1] (Prowler v5.25.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: AWS scans no longer fail when enabled regions cannot be retrieved, and scans stuck in `scheduled` state are now cleaned up after the stale threshold [(#10917)](https://github.com/prowler-cloud/prowler/pull/10917)
|
||||
- Scan report and compliance downloads now redirect to a presigned S3 URL instead of streaming through the API worker, preventing gunicorn timeouts on large files [(#10927)](https://github.com/prowler-cloud/prowler/pull/10927)
|
||||
|
||||
---
|
||||
|
||||
## [1.26.0] (Prowler v5.25.0)
|
||||
|
||||
### 🚀 Added
|
||||
@@ -12,7 +66,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Allows tenant owners to expel users from their organizations [(#10787)](https://github.com/prowler-cloud/prowler/pull/10787)
|
||||
- Allows tenant owners to expel users from their organizations [(#10787)](https://github.com/prowler-cloud/prowler/pull/10787)
|
||||
- `aggregate_findings`, `aggregate_attack_surface`, `aggregate_scan_resource_group_summaries` and `aggregate_scan_category_summaries` now upsert via `bulk_create(update_conflicts=True, ...)` instead of the prior `ignore_conflicts=True` / plain INSERT / `already backfilled` short-circuit. Re-runs triggered by the post-mute reaggregation pipeline no longer trip the `unique_*_per_scan` constraints nor silently drop updates, and are race-safe under concurrent writers (e.g. scan completion overlapping with a fresh mute rule) [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843)
|
||||
- Rename the scan-category and scan-resource-group summary aggregators from `backfill_*` to `aggregate_*` [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843)
|
||||
|
||||
|
||||
+8
-7
@@ -5,7 +5,7 @@ LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.69.2
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
@@ -14,6 +14,7 @@ ENV ZIZMOR_VERSION=${ZIZMOR_VERSION}
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
wget \
|
||||
git \
|
||||
libicu72 \
|
||||
gcc \
|
||||
g++ \
|
||||
@@ -88,18 +89,18 @@ WORKDIR /home/prowler
|
||||
# Ensure output directory exists
|
||||
RUN mkdir -p /tmp/prowler_api_output
|
||||
|
||||
COPY pyproject.toml ./
|
||||
COPY pyproject.toml uv.lock ./
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir poetry==2.3.4
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
|
||||
ENV PATH="/home/prowler/.local/bin:$PATH"
|
||||
|
||||
# Add `--no-root` to avoid installing the current project as a package
|
||||
RUN poetry install --no-root && \
|
||||
rm -rf ~/.cache/pip
|
||||
# Add `--no-install-project` to avoid installing the current project as a package
|
||||
RUN uv sync --no-install-project && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
RUN poetry run python "$(poetry env info --path)/src/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py"
|
||||
RUN .venv/bin/python .venv/lib/python3.12/site-packages/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
COPY src/backend/ ./backend/
|
||||
COPY docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
|
||||
+12
-19
@@ -25,12 +25,11 @@ If you don’t set `DJANGO_TOKEN_SIGNING_KEY` or `DJANGO_TOKEN_VERIFYING_KEY`, t
|
||||
**Important note**: Every Prowler version (or repository branches and tags) could have different variables set in its `.env` file. Please use the `.env` file that corresponds with each version.
|
||||
|
||||
## Local deployment
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the Poetry interpreter, not before. Otherwise, variables will not be loaded properly.
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the virtual environment, not before. Otherwise, variables will not be loaded properly.
|
||||
|
||||
To do this, you can run:
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
set -a
|
||||
source .env
|
||||
```
|
||||
@@ -78,7 +77,7 @@ docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
|
||||
## Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `poetry` and `docker compose` are installed.
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
|
||||
@@ -90,11 +89,10 @@ git clone https://github.com/prowler-cloud/api.git
|
||||
git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
### Install all dependencies with Poetry
|
||||
### Install all dependencies with uv
|
||||
|
||||
```console
|
||||
poetry install
|
||||
poetry shell
|
||||
uv sync
|
||||
```
|
||||
|
||||
## Start the PostgreSQL Database and Valkey
|
||||
@@ -139,7 +137,7 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
## Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `poetry` and `docker compose` are installed.
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
|
||||
@@ -165,11 +163,10 @@ docker compose up postgres valkey -d
|
||||
|
||||
### Install the Python dependencies
|
||||
|
||||
> You must have Poetry installed
|
||||
> You must have uv installed
|
||||
|
||||
```console
|
||||
poetry install
|
||||
poetry shell
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Apply migrations
|
||||
@@ -246,9 +243,8 @@ docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
uv run python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
## Apply fixtures
|
||||
@@ -256,9 +252,8 @@ python manage.py migrate --database admin
|
||||
Fixtures are used to populate the database with initial development data.
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
cd src/backend
|
||||
python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
uv run python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
```
|
||||
|
||||
> The default credentials are `dev@prowler.com:Thisisapassword123@` or `dev2@prowler.com:Thisisapassword123@`
|
||||
@@ -270,9 +265,8 @@ Note that the tests will fail if you use the same `.env` file as the development
|
||||
For best results, run in a new shell with no environment variables set.
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
cd src/backend
|
||||
pytest
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
# Custom commands
|
||||
@@ -284,8 +278,7 @@ Django provides a way to create custom commands that can be run from the command
|
||||
To run a custom command, you need to be in the `prowler/api/src/backend` directory and run:
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
python manage.py <command_name>
|
||||
uv run python manage.py <command_name>
|
||||
```
|
||||
|
||||
## Generate dummy data
|
||||
@@ -308,7 +301,7 @@ This command creates, for a given tenant, a provider, scan and a set of findings
|
||||
### Example
|
||||
|
||||
```console
|
||||
~/backend $ poetry run python manage.py findings --tenant
|
||||
~/backend $ uv run python manage.py findings --tenant
|
||||
fffb1893-3fc7-4623-a5d9-fae47da1c528 --findings 25000 --re
|
||||
sources 1000 --batch 5000 --alias test-script
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ apply_migrations() {
|
||||
echo "Applying database migrations..."
|
||||
|
||||
# Fix Inconsistent migration history after adding sites app
|
||||
poetry run python manage.py check_and_fix_socialaccount_sites_migration --database admin
|
||||
uv run python manage.py check_and_fix_socialaccount_sites_migration --database admin
|
||||
|
||||
poetry run python manage.py migrate --database admin
|
||||
uv run python manage.py migrate --database admin
|
||||
}
|
||||
|
||||
apply_fixtures() {
|
||||
@@ -15,19 +15,19 @@ apply_fixtures() {
|
||||
for fixture in api/fixtures/dev/*.json; do
|
||||
if [ -f "$fixture" ]; then
|
||||
echo "Loading $fixture"
|
||||
poetry run python manage.py loaddata "$fixture" --database admin
|
||||
uv run python manage.py loaddata "$fixture" --database admin
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
start_dev_server() {
|
||||
echo "Starting the development server..."
|
||||
poetry run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
}
|
||||
|
||||
start_prod_server() {
|
||||
echo "Starting the Gunicorn server..."
|
||||
poetry run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
}
|
||||
|
||||
resolve_worker_hostname() {
|
||||
@@ -47,7 +47,7 @@ resolve_worker_hostname() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
poetry run python -m celery -A config.celery worker \
|
||||
uv run python -m celery -A config.celery worker \
|
||||
-n "$(resolve_worker_hostname)" \
|
||||
-l "${DJANGO_LOGGING_LEVEL:-info}" \
|
||||
-Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \
|
||||
@@ -56,7 +56,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
poetry run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
manage_db_partitions() {
|
||||
@@ -64,7 +64,7 @@ manage_db_partitions() {
|
||||
echo "Managing DB partitions..."
|
||||
# For now we skip the deletion of partitions until we define the data retention policy
|
||||
# --yes auto approves the operation without the need of an interactive terminal
|
||||
poetry run python manage.py pgpartition --using admin --skip-delete --yes
|
||||
uv run python manage.py pgpartition --using admin --skip-delete --yes
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
Generated
-9427
File diff suppressed because it is too large
Load Diff
+393
-27
@@ -1,6 +1,24 @@
|
||||
[build-system]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
requires = ["poetry-core"]
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"bandit==1.7.9",
|
||||
"coverage==7.5.4",
|
||||
"django-silk==5.3.2",
|
||||
"docker==7.1.0",
|
||||
"filelock==3.20.3",
|
||||
"freezegun==1.5.1",
|
||||
"mypy==1.10.1",
|
||||
"pylint==3.2.5",
|
||||
"pytest==9.0.3",
|
||||
"pytest-cov==5.0.0",
|
||||
"pytest-django==4.8.0",
|
||||
"pytest-env==1.1.3",
|
||||
"pytest-randomly==3.15.0",
|
||||
"pytest-xdist==3.6.1",
|
||||
"ruff==0.5.0",
|
||||
"tqdm==4.67.1",
|
||||
"vulture==2.14",
|
||||
"prek==0.3.9"
|
||||
]
|
||||
|
||||
[project]
|
||||
authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
|
||||
@@ -50,28 +68,376 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.26.0"
|
||||
version = "1.28.0"
|
||||
|
||||
[project.scripts]
|
||||
celery = "src.backend.config.settings.celery"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
bandit = "1.7.9"
|
||||
coverage = "7.5.4"
|
||||
django-silk = "5.3.2"
|
||||
docker = "7.1.0"
|
||||
filelock = "3.20.3"
|
||||
freezegun = "1.5.1"
|
||||
mypy = "1.10.1"
|
||||
pylint = "3.2.5"
|
||||
pytest = "9.0.3"
|
||||
pytest-cov = "5.0.0"
|
||||
pytest-django = "4.8.0"
|
||||
pytest-env = "1.1.3"
|
||||
pytest-randomly = "3.15.0"
|
||||
pytest-xdist = "3.6.1"
|
||||
ruff = "0.5.0"
|
||||
safety = "3.7.0"
|
||||
tqdm = "4.67.1"
|
||||
vulture = "2.14"
|
||||
prek = "0.3.9"
|
||||
[tool.uv]
|
||||
# Transitive pins matching master to avoid silent drift; bump deliberately.
|
||||
constraint-dependencies = [
|
||||
"about-time==4.2.1",
|
||||
"adal==1.2.7",
|
||||
"aioboto3==15.5.0",
|
||||
"aiobotocore==2.25.1",
|
||||
"aiofiles==24.1.0",
|
||||
"aiohappyeyeballs==2.6.1",
|
||||
"aiohttp==3.13.5",
|
||||
"aioitertools==0.13.0",
|
||||
"aiosignal==1.4.0",
|
||||
"alibabacloud-actiontrail20200706==2.4.1",
|
||||
"alibabacloud-credentials==1.0.3",
|
||||
"alibabacloud-credentials-api==1.0.0",
|
||||
"alibabacloud-cs20151215==6.1.0",
|
||||
"alibabacloud-darabonba-array==0.1.0",
|
||||
"alibabacloud-darabonba-encode-util==0.0.2",
|
||||
"alibabacloud-darabonba-map==0.0.1",
|
||||
"alibabacloud-darabonba-signature-util==0.0.4",
|
||||
"alibabacloud-darabonba-string==0.0.4",
|
||||
"alibabacloud-darabonba-time==0.0.1",
|
||||
"alibabacloud-ecs20140526==7.2.5",
|
||||
"alibabacloud-endpoint-util==0.0.4",
|
||||
"alibabacloud-gateway-oss==0.0.17",
|
||||
"alibabacloud-gateway-oss-util==0.0.3",
|
||||
"alibabacloud-gateway-sls==0.4.0",
|
||||
"alibabacloud-gateway-sls-util==0.4.0",
|
||||
"alibabacloud-gateway-spi==0.0.3",
|
||||
"alibabacloud-openapi-util==0.2.4",
|
||||
"alibabacloud-oss-util==0.0.6",
|
||||
"alibabacloud-oss20190517==1.0.6",
|
||||
"alibabacloud-ram20150501==1.2.0",
|
||||
"alibabacloud-rds20140815==12.0.0",
|
||||
"alibabacloud-sas20181203==6.1.0",
|
||||
"alibabacloud-sls20201230==5.9.0",
|
||||
"alibabacloud-sts20150401==1.1.6",
|
||||
"alibabacloud-tea==0.4.3",
|
||||
"alibabacloud-tea-openapi==0.4.4",
|
||||
"alibabacloud-tea-util==0.3.14",
|
||||
"alibabacloud-tea-xml==0.0.3",
|
||||
"alibabacloud-vpc20160428==6.13.0",
|
||||
"alive-progress==3.3.0",
|
||||
"aliyun-log-fastpb==0.2.0",
|
||||
"amqp==5.3.1",
|
||||
"annotated-types==0.7.0",
|
||||
"anyio==4.12.1",
|
||||
"applicationinsights==0.11.10",
|
||||
"apscheduler==3.11.2",
|
||||
"argcomplete==3.5.3",
|
||||
"asgiref==3.11.0",
|
||||
"astroid==3.2.4",
|
||||
"async-timeout==5.0.1",
|
||||
"attrs==25.4.0",
|
||||
"authlib==1.6.9",
|
||||
"autopep8==2.3.2",
|
||||
"awsipranges==0.3.3",
|
||||
"azure-cli-core==2.83.0",
|
||||
"azure-cli-telemetry==1.1.0",
|
||||
"azure-common==1.1.28",
|
||||
"azure-core==1.38.1",
|
||||
"azure-identity==1.21.0",
|
||||
"azure-keyvault-certificates==4.10.0",
|
||||
"azure-keyvault-keys==4.10.0",
|
||||
"azure-keyvault-secrets==4.10.0",
|
||||
"azure-mgmt-apimanagement==5.0.0",
|
||||
"azure-mgmt-applicationinsights==4.1.0",
|
||||
"azure-mgmt-authorization==4.0.0",
|
||||
"azure-mgmt-compute==34.0.0",
|
||||
"azure-mgmt-containerinstance==10.1.0",
|
||||
"azure-mgmt-containerregistry==12.0.0",
|
||||
"azure-mgmt-containerservice==34.1.0",
|
||||
"azure-mgmt-core==1.6.0",
|
||||
"azure-mgmt-cosmosdb==9.7.0",
|
||||
"azure-mgmt-databricks==2.0.0",
|
||||
"azure-mgmt-datafactory==9.2.0",
|
||||
"azure-mgmt-eventgrid==10.4.0",
|
||||
"azure-mgmt-eventhub==11.2.0",
|
||||
"azure-mgmt-keyvault==10.3.1",
|
||||
"azure-mgmt-loganalytics==12.0.0",
|
||||
"azure-mgmt-logic==10.0.0",
|
||||
"azure-mgmt-monitor==6.0.2",
|
||||
"azure-mgmt-network==28.1.0",
|
||||
"azure-mgmt-postgresqlflexibleservers==1.1.0",
|
||||
"azure-mgmt-rdbms==10.1.0",
|
||||
"azure-mgmt-recoveryservices==3.1.0",
|
||||
"azure-mgmt-recoveryservicesbackup==9.2.0",
|
||||
"azure-mgmt-resource==24.0.0",
|
||||
"azure-mgmt-search==9.1.0",
|
||||
"azure-mgmt-security==7.0.0",
|
||||
"azure-mgmt-sql==3.0.1",
|
||||
"azure-mgmt-storage==22.1.1",
|
||||
"azure-mgmt-subscription==3.1.1",
|
||||
"azure-mgmt-synapse==2.0.0",
|
||||
"azure-mgmt-web==8.0.0",
|
||||
"azure-monitor-query==2.0.0",
|
||||
"azure-storage-blob==12.24.1",
|
||||
"azure-synapse-artifacts==0.21.0",
|
||||
"backoff==2.2.1",
|
||||
"bandit==1.7.9",
|
||||
"billiard==4.2.4",
|
||||
"blinker==1.9.0",
|
||||
"boto3==1.40.61",
|
||||
"botocore==1.40.61",
|
||||
"cartography==0.135.0",
|
||||
"celery==5.6.2",
|
||||
"certifi==2026.1.4",
|
||||
"cffi==2.0.0",
|
||||
"charset-normalizer==3.4.4",
|
||||
"circuitbreaker==2.1.3",
|
||||
"click==8.3.1",
|
||||
"click-didyoumean==0.3.1",
|
||||
"click-plugins==1.1.1.2",
|
||||
"click-repl==0.3.0",
|
||||
"cloudflare==4.3.1",
|
||||
"colorama==0.4.6",
|
||||
"contextlib2==21.6.0",
|
||||
"contourpy==1.3.3",
|
||||
"coverage==7.5.4",
|
||||
"cron-descriptor==1.4.5",
|
||||
"crowdstrike-falconpy==1.6.0",
|
||||
"cryptography==46.0.7",
|
||||
"cycler==0.12.1",
|
||||
"darabonba-core==1.0.5",
|
||||
"dash==3.1.1",
|
||||
"dash-bootstrap-components==2.0.3",
|
||||
"debugpy==1.8.20",
|
||||
"decorator==5.2.1",
|
||||
"defusedxml==0.7.1",
|
||||
"detect-secrets==1.5.0",
|
||||
"dill==0.4.1",
|
||||
"distro==1.9.0",
|
||||
"dj-rest-auth==7.0.1",
|
||||
"django==5.1.15",
|
||||
"django-allauth==65.15.0",
|
||||
"django-celery-beat==2.9.0",
|
||||
"django-celery-results==2.6.0",
|
||||
"django-cors-headers==4.4.0",
|
||||
"django-environ==0.11.2",
|
||||
"django-filter==24.3",
|
||||
"django-guid==3.5.0",
|
||||
"django-postgres-extra==2.0.9",
|
||||
"django-silk==5.3.2",
|
||||
"django-timezone-field==7.2.1",
|
||||
"djangorestframework==3.15.2",
|
||||
"djangorestframework-jsonapi==7.0.2",
|
||||
"djangorestframework-simplejwt==5.5.1",
|
||||
"dnspython==2.8.0",
|
||||
"docker==7.1.0",
|
||||
"dogpile-cache==1.5.0",
|
||||
"dparse==0.6.4",
|
||||
"drf-extensions==0.8.0",
|
||||
"drf-nested-routers==0.95.0",
|
||||
"drf-simple-apikey==2.2.1",
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"dulwich==0.23.0",
|
||||
"duo-client==5.5.0",
|
||||
"durationpy==0.10",
|
||||
"email-validator==2.2.0",
|
||||
"execnet==2.1.2",
|
||||
"filelock==3.20.3",
|
||||
"flask==3.1.3",
|
||||
"fonttools==4.62.1",
|
||||
"freezegun==1.5.1",
|
||||
"frozenlist==1.8.0",
|
||||
"gevent==25.9.1",
|
||||
"google-api-core==2.29.0",
|
||||
"google-api-python-client==2.163.0",
|
||||
"google-auth==2.48.0",
|
||||
"google-auth-httplib2==0.2.0",
|
||||
"google-cloud-access-context-manager==0.3.0",
|
||||
"google-cloud-asset==4.2.0",
|
||||
"google-cloud-org-policy==1.16.0",
|
||||
"google-cloud-os-config==1.23.0",
|
||||
"google-cloud-resource-manager==1.16.0",
|
||||
"googleapis-common-protos==1.72.0",
|
||||
"gprof2dot==2025.4.14",
|
||||
"graphemeu==0.7.2",
|
||||
"greenlet==3.3.1",
|
||||
"grpc-google-iam-v1==0.14.3",
|
||||
"grpcio==1.76.0",
|
||||
"grpcio-status==1.76.0",
|
||||
"gunicorn==23.0.0",
|
||||
"h11==0.16.0",
|
||||
"h2==4.3.0",
|
||||
"hpack==4.1.0",
|
||||
"httpcore==1.0.9",
|
||||
"httplib2==0.31.2",
|
||||
"httpx==0.28.1",
|
||||
"humanfriendly==10.0",
|
||||
"hyperframe==6.1.0",
|
||||
"iamdata==0.1.202602021",
|
||||
"idna==3.11",
|
||||
"importlib-metadata==8.7.1",
|
||||
"inflection==0.5.1",
|
||||
"iniconfig==2.3.0",
|
||||
"iso8601==2.1.0",
|
||||
"isodate==0.7.2",
|
||||
"isort==5.13.2",
|
||||
"itsdangerous==2.2.0",
|
||||
"jinja2==3.1.6",
|
||||
"jiter==0.13.0",
|
||||
"jmespath==1.1.0",
|
||||
"joblib==1.5.3",
|
||||
"jsonpatch==1.33",
|
||||
"jsonpickle==4.1.1",
|
||||
"jsonpointer==3.0.0",
|
||||
"jsonschema==4.23.0",
|
||||
"jsonschema-specifications==2025.9.1",
|
||||
"keystoneauth1==5.13.0",
|
||||
"kiwisolver==1.4.9",
|
||||
"knack==0.11.0",
|
||||
"kombu==5.6.2",
|
||||
"kubernetes==32.0.1",
|
||||
"lxml==5.3.2",
|
||||
"lz4==4.4.5",
|
||||
"markdown==3.10.2",
|
||||
"markdown-it-py==4.0.0",
|
||||
"markupsafe==3.0.3",
|
||||
"marshmallow==4.3.0",
|
||||
"matplotlib==3.10.8",
|
||||
"mccabe==0.7.0",
|
||||
"mdurl==0.1.2",
|
||||
"microsoft-kiota-abstractions==1.9.2",
|
||||
"microsoft-kiota-authentication-azure==1.9.2",
|
||||
"microsoft-kiota-http==1.9.2",
|
||||
"microsoft-kiota-serialization-form==1.9.2",
|
||||
"microsoft-kiota-serialization-json==1.9.2",
|
||||
"microsoft-kiota-serialization-multipart==1.9.2",
|
||||
"microsoft-kiota-serialization-text==1.9.2",
|
||||
"microsoft-security-utilities-secret-masker==1.0.0b4",
|
||||
"msal==1.35.0b1",
|
||||
"msal-extensions==1.2.0",
|
||||
"msgraph-core==1.3.8",
|
||||
"msgraph-sdk==1.55.0",
|
||||
"msrest==0.7.1",
|
||||
"msrestazure==0.6.4.post1",
|
||||
"multidict==6.7.1",
|
||||
"mypy==1.10.1",
|
||||
"mypy-extensions==1.1.0",
|
||||
"narwhals==2.16.0",
|
||||
"neo4j==6.1.0",
|
||||
"nest-asyncio==1.6.0",
|
||||
"nltk==3.9.4",
|
||||
"numpy==2.0.2",
|
||||
"oauthlib==3.3.1",
|
||||
"oci==2.169.0",
|
||||
"openai==1.109.1",
|
||||
"openstacksdk==4.2.0",
|
||||
"opentelemetry-api==1.39.1",
|
||||
"opentelemetry-sdk==1.39.1",
|
||||
"opentelemetry-semantic-conventions==0.60b1",
|
||||
"os-service-types==1.8.2",
|
||||
"packageurl-python==0.17.6",
|
||||
"packaging==26.0",
|
||||
"pagerduty==6.1.0",
|
||||
"pandas==2.2.3",
|
||||
"pbr==7.0.3",
|
||||
"pillow==12.2.0",
|
||||
"pkginfo==1.12.1.2",
|
||||
"platformdirs==4.5.1",
|
||||
"plotly==6.5.2",
|
||||
"pluggy==1.6.0",
|
||||
"policyuniverse==1.5.1.20231109",
|
||||
"portalocker==2.10.1",
|
||||
"prek==0.3.9",
|
||||
"prompt-toolkit==3.0.52",
|
||||
"propcache==0.4.1",
|
||||
"proto-plus==1.27.0",
|
||||
"protobuf==6.33.5",
|
||||
"psutil==7.2.2",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"py-deviceid==0.1.1",
|
||||
"py-iam-expand==0.1.0",
|
||||
"py-ocsf-models==0.8.1",
|
||||
"pyasn1==0.6.3",
|
||||
"pyasn1-modules==0.4.2",
|
||||
"pycodestyle==2.14.0",
|
||||
"pycparser==3.0",
|
||||
"pydantic==2.12.5",
|
||||
"pydantic-core==2.41.5",
|
||||
"pygithub==2.8.0",
|
||||
"pygments==2.20.0",
|
||||
"pyjwt==2.12.1",
|
||||
"pylint==3.2.5",
|
||||
"pymsalruntime==0.18.1",
|
||||
"pynacl==1.6.2",
|
||||
"pyopenssl==26.0.0",
|
||||
"pyparsing==3.3.2",
|
||||
"pyreadline3==3.5.4",
|
||||
"pysocks==1.7.1",
|
||||
"pytest==9.0.3",
|
||||
"pytest-celery==1.3.0",
|
||||
"pytest-cov==5.0.0",
|
||||
"pytest-django==4.8.0",
|
||||
"pytest-docker-tools==3.1.9",
|
||||
"pytest-env==1.1.3",
|
||||
"pytest-randomly==3.15.0",
|
||||
"pytest-xdist==3.6.1",
|
||||
"python-crontab==3.3.0",
|
||||
"python-dateutil==2.9.0.post0",
|
||||
"python-digitalocean==1.17.0",
|
||||
"python3-saml==1.16.0",
|
||||
"pytz==2025.1",
|
||||
"pywin32==311",
|
||||
"pyyaml==6.0.3",
|
||||
"redis==7.1.0",
|
||||
"referencing==0.37.0",
|
||||
"regex==2026.1.15",
|
||||
"reportlab==4.4.10",
|
||||
"requests==2.33.1",
|
||||
"requests-file==3.0.1",
|
||||
"requests-oauthlib==2.0.0",
|
||||
"requestsexceptions==1.4.0",
|
||||
"retrying==1.4.2",
|
||||
"rich==14.3.2",
|
||||
"rpds-py==0.30.0",
|
||||
"rsa==4.9.1",
|
||||
"ruamel-yaml==0.19.1",
|
||||
"ruff==0.5.0",
|
||||
"s3transfer==0.14.0",
|
||||
"scaleway==2.10.3",
|
||||
"scaleway-core==2.10.3",
|
||||
"schema==0.7.5",
|
||||
"sentry-sdk==2.56.0",
|
||||
"setuptools==80.10.2",
|
||||
"shellingham==1.5.4",
|
||||
"shodan==1.31.0",
|
||||
"six==1.17.0",
|
||||
"slack-sdk==3.39.0",
|
||||
"sniffio==1.3.1",
|
||||
"sqlparse==0.5.5",
|
||||
"statsd==4.0.1",
|
||||
"std-uritemplate==2.0.8",
|
||||
"stevedore==5.6.0",
|
||||
"tabulate==0.9.0",
|
||||
"tenacity==9.1.2",
|
||||
"tldextract==5.3.1",
|
||||
"tomlkit==0.14.0",
|
||||
"tqdm==4.67.1",
|
||||
"typer==0.21.1",
|
||||
"types-aiobotocore-ecr==3.1.1",
|
||||
"typing-extensions==4.15.0",
|
||||
"typing-inspection==0.4.2",
|
||||
"tzdata==2025.3",
|
||||
"tzlocal==5.3.1",
|
||||
"uritemplate==4.2.0",
|
||||
"urllib3==2.6.3",
|
||||
"uuid6==2024.7.10",
|
||||
"vine==5.1.0",
|
||||
"vulture==2.14",
|
||||
"wcwidth==0.5.3",
|
||||
"websocket-client==1.9.0",
|
||||
"werkzeug==3.1.7",
|
||||
"workos==6.0.4",
|
||||
"wrapt==1.17.3",
|
||||
"xlsxwriter==3.2.9",
|
||||
"xmlsec==1.3.14",
|
||||
"xmltodict==1.0.2",
|
||||
"yarl==1.22.0",
|
||||
"zipp==3.23.0",
|
||||
"zope-event==6.1",
|
||||
"zope-interface==8.2",
|
||||
"zstd==1.5.7.3"
|
||||
]
|
||||
# prowler@master needs okta==3.4.2; cartography 0.135.0 declares okta<1.0.0 for an
|
||||
# integration prowler does not import.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2"
|
||||
]
|
||||
|
||||
@@ -52,7 +52,7 @@ class ApiConfig(AppConfig):
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip Neo4j initialization during tests, some Django commands, and Celery
|
||||
# Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook)
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
@@ -64,7 +64,7 @@ class ApiConfig(AppConfig):
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping Neo4j initialization because tests, some Django commands or Celery"
|
||||
"Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)"
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
@@ -484,8 +484,8 @@ AWS_BEDROCK_PRIVESC_PASSROLE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust Bedrock service (can be passed to Bedrock)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock.amazonaws.com'}})
|
||||
// Find roles that trust the Bedrock AgentCore service (can be passed to a code interpreter)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
@@ -536,8 +536,8 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust Bedrock service (already attached to existing code interpreters)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock.amazonaws.com'}})
|
||||
// Find roles that trust the Bedrock AgentCore service (already attached to existing code interpreters)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
|
||||
|
||||
WITH collect(path_principal) + collect(path_target) AS paths
|
||||
UNWIND paths AS p
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
from functools import partial
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
from api.db_utils import create_index_on_partitions, drop_index_on_partitions
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
("api", "0090_attack_paths_cleanup_priority"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(
|
||||
partial(
|
||||
create_index_on_partitions,
|
||||
parent_table="findings",
|
||||
index_name="gin_find_arrays_idx",
|
||||
columns="categories, resource_services, resource_regions, resource_types",
|
||||
method="GIN",
|
||||
all_partitions=True,
|
||||
),
|
||||
reverse_code=partial(
|
||||
drop_index_on_partitions,
|
||||
parent_table="findings",
|
||||
index_name="gin_find_arrays_idx",
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,73 @@
|
||||
import django.contrib.postgres.indexes
|
||||
from django.db import migrations
|
||||
|
||||
INDEX_NAME = "gin_find_arrays_idx"
|
||||
PARENT_TABLE = "findings"
|
||||
|
||||
|
||||
def create_parent_and_attach(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
# Idempotent: the parent index may already exist if it was created
|
||||
# manually on an environment before this migration ran.
|
||||
cursor.execute(
|
||||
f"CREATE INDEX IF NOT EXISTS {INDEX_NAME} ON ONLY {PARENT_TABLE} "
|
||||
f"USING gin (categories, resource_services, resource_regions, resource_types)"
|
||||
)
|
||||
cursor.execute(
|
||||
"SELECT inhrelid::regclass::text "
|
||||
"FROM pg_inherits "
|
||||
"WHERE inhparent = %s::regclass",
|
||||
[PARENT_TABLE],
|
||||
)
|
||||
for (partition,) in cursor.fetchall():
|
||||
child_idx = f"{partition.replace('.', '_')}_{INDEX_NAME}"
|
||||
# ALTER INDEX ... ATTACH PARTITION has no IF NOT ATTACHED clause,
|
||||
# so check pg_inherits first to keep the migration re-runnable.
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class p ON p.oid = i.inhparent
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE p.relname = %s AND c.relname = %s
|
||||
""",
|
||||
[INDEX_NAME, child_idx],
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
cursor.execute(f"ALTER INDEX {INDEX_NAME} ATTACH PARTITION {child_idx}")
|
||||
|
||||
|
||||
def drop_parent_index(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}")
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0091_findings_arrays_gin_index_partitions"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.SeparateDatabaseAndState(
|
||||
state_operations=[
|
||||
migrations.AddIndex(
|
||||
model_name="finding",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=[
|
||||
"categories",
|
||||
"resource_services",
|
||||
"resource_regions",
|
||||
"resource_types",
|
||||
],
|
||||
name=INDEX_NAME,
|
||||
),
|
||||
),
|
||||
],
|
||||
database_operations=[
|
||||
migrations.RunPython(
|
||||
create_parent_and_attach,
|
||||
reverse_code=drop_parent_index,
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -595,10 +595,40 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
|
||||
_SCOPING_SCANNER_ARG_KEYS_CACHE: tuple[str, ...] | None = None
|
||||
|
||||
@classmethod
|
||||
def get_scoping_scanner_arg_keys(cls) -> tuple[str, ...]:
|
||||
"""Return the scanner_args keys that mark a scan as scoped.
|
||||
|
||||
Derived from ``prowler.lib.scan.scan.Scan.__init__`` so the API stays
|
||||
in sync with whatever the SDK actually accepts as filters. Cached at
|
||||
class level — the signature is stable for the process lifetime.
|
||||
"""
|
||||
if cls._SCOPING_SCANNER_ARG_KEYS_CACHE is None:
|
||||
import inspect
|
||||
|
||||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
|
||||
params = inspect.signature(ProwlerScan.__init__).parameters
|
||||
cls._SCOPING_SCANNER_ARG_KEYS_CACHE = tuple(
|
||||
name for name in params if name not in ("self", "provider")
|
||||
)
|
||||
return cls._SCOPING_SCANNER_ARG_KEYS_CACHE
|
||||
|
||||
class TriggerChoices(models.TextChoices):
|
||||
SCHEDULED = "scheduled", _("Scheduled")
|
||||
MANUAL = "manual", _("Manual")
|
||||
|
||||
# Trigger values for scans that ran the SDK end-to-end. Imported scans (or
|
||||
# any future trigger) are intentionally NOT in this set — they may carry
|
||||
# only a partial slice of resources, so post-scan logic that depends on a
|
||||
# full-scope sweep (e.g. resetting ephemeral resource findings) must skip
|
||||
# them by default.
|
||||
LIVE_SCAN_TRIGGERS = frozenset(
|
||||
(TriggerChoices.SCHEDULED.value, TriggerChoices.MANUAL.value)
|
||||
)
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
|
||||
name = models.CharField(
|
||||
blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)]
|
||||
@@ -681,6 +711,24 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scans"
|
||||
|
||||
def is_full_scope(self) -> bool:
|
||||
"""Return True if this scan ran with no scoping filters at all.
|
||||
|
||||
Used to gate post-scan operations (such as resetting the
|
||||
failed_findings_count of resources missing from the scan) that are only
|
||||
safe when the scan covered every check, service, and category. Imported
|
||||
scans are NOT full-scope by definition — they may carry only a partial
|
||||
slice of resources, so they're rejected via ``trigger`` even before the
|
||||
scanner_args check.
|
||||
"""
|
||||
if self.trigger not in self.LIVE_SCAN_TRIGGERS:
|
||||
return False
|
||||
scanner_args = self.scanner_args or {}
|
||||
for key in self.get_scoping_scanner_arg_keys():
|
||||
if scanner_args.get(key):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
@@ -898,7 +946,6 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
OpClass(Upper("name"), name="gin_trgm_ops"),
|
||||
name="res_name_trgm_idx",
|
||||
),
|
||||
GinIndex(fields=["text_search"], name="gin_resources_search_idx"),
|
||||
models.Index(fields=["tenant_id", "id"], name="resources_tenant_id_idx"),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
@@ -1104,6 +1151,15 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
|
||||
fields=["tenant_id", "scan_id", "check_id"],
|
||||
name="find_tenant_scan_check_idx",
|
||||
),
|
||||
GinIndex(
|
||||
fields=[
|
||||
"categories",
|
||||
"resource_services",
|
||||
"resource_regions",
|
||||
"resource_types",
|
||||
],
|
||||
name="gin_find_arrays_idx",
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
|
||||
+1620
-58
File diff suppressed because it is too large
Load Diff
@@ -3841,9 +3841,14 @@ class TestScanViewSet:
|
||||
"prowler-output-123_threatscore_report.pdf",
|
||||
)
|
||||
|
||||
presigned_url = (
|
||||
"https://test-bucket.s3.amazonaws.com/"
|
||||
"tenant-id/scan-id/threatscore/prowler-output-123_threatscore_report.pdf"
|
||||
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300"
|
||||
)
|
||||
mock_s3_client = Mock()
|
||||
mock_s3_client.list_objects_v2.return_value = {"Contents": [{"Key": pdf_key}]}
|
||||
mock_s3_client.get_object.return_value = {"Body": io.BytesIO(b"pdf-bytes")}
|
||||
mock_s3_client.generate_presigned_url.return_value = presigned_url
|
||||
|
||||
mock_env_str.return_value = bucket
|
||||
mock_get_s3_client.return_value = mock_s3_client
|
||||
@@ -3852,19 +3857,26 @@ class TestScanViewSet:
|
||||
url = reverse("scan-threatscore", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"] == "application/pdf"
|
||||
assert response["Content-Disposition"].endswith(
|
||||
'"prowler-output-123_threatscore_report.pdf"'
|
||||
)
|
||||
assert response.content == b"pdf-bytes"
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
assert response["Location"] == presigned_url
|
||||
mock_s3_client.list_objects_v2.assert_called_once()
|
||||
mock_s3_client.get_object.assert_called_once_with(Bucket=bucket, Key=pdf_key)
|
||||
mock_s3_client.generate_presigned_url.assert_called_once_with(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": bucket,
|
||||
"Key": pdf_key,
|
||||
"ResponseContentDisposition": (
|
||||
'attachment; filename="prowler-output-123_threatscore_report.pdf"'
|
||||
),
|
||||
"ResponseContentType": "application/pdf",
|
||||
},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
|
||||
def test_report_s3_success(self, authenticated_client, scans_fixture, monkeypatch):
|
||||
"""
|
||||
When output_location is an S3 URL and the S3 client returns the file successfully,
|
||||
the view should return the ZIP file with HTTP 200 and proper headers.
|
||||
When output_location is an S3 URL and the object exists,
|
||||
the view should return a 302 redirect to a presigned S3 URL.
|
||||
"""
|
||||
scan = scans_fixture[0]
|
||||
bucket = "test-bucket"
|
||||
@@ -3878,22 +3890,33 @@ class TestScanViewSet:
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
presigned_url = (
|
||||
"https://test-bucket.s3.amazonaws.com/report.zip"
|
||||
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300"
|
||||
)
|
||||
|
||||
class FakeS3Client:
|
||||
def get_object(self, Bucket, Key):
|
||||
def head_object(self, Bucket, Key):
|
||||
assert Bucket == bucket
|
||||
assert Key == key
|
||||
return {"Body": io.BytesIO(b"s3 zip content")}
|
||||
return {}
|
||||
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
assert ClientMethod == "get_object"
|
||||
assert Params["Bucket"] == bucket
|
||||
assert Params["Key"] == key
|
||||
assert Params["ResponseContentDisposition"] == (
|
||||
'attachment; filename="report.zip"'
|
||||
)
|
||||
assert ExpiresIn == 300
|
||||
return presigned_url
|
||||
|
||||
monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client())
|
||||
|
||||
url = reverse("scan-report", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
assert response.status_code == 200
|
||||
expected_filename = os.path.basename("report.zip")
|
||||
content_disposition = response.get("Content-Disposition")
|
||||
assert content_disposition.startswith('attachment; filename="')
|
||||
assert f'filename="{expected_filename}"' in content_disposition
|
||||
assert response.content == b"s3 zip content"
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
assert response["Location"] == presigned_url
|
||||
|
||||
def test_report_s3_success_no_local_files(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
@@ -4032,23 +4055,31 @@ class TestScanViewSet:
|
||||
)
|
||||
|
||||
match_key = "path/compliance/mitre_attack_aws.csv"
|
||||
presigned_url = (
|
||||
"https://test-bucket.s3.amazonaws.com/path/compliance/mitre_attack_aws.csv"
|
||||
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300"
|
||||
)
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
return {"Contents": [{"Key": match_key}]}
|
||||
|
||||
def get_object(self, Bucket, Key):
|
||||
return {"Body": io.BytesIO(b"ignored")}
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
assert ClientMethod == "get_object"
|
||||
assert Params["Key"] == match_key
|
||||
assert Params["ResponseContentDisposition"] == (
|
||||
'attachment; filename="mitre_attack_aws.csv"'
|
||||
)
|
||||
assert ExpiresIn == 300
|
||||
return presigned_url
|
||||
|
||||
monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client())
|
||||
|
||||
framework = match_key.split("/")[-1].split(".")[0]
|
||||
url = reverse("scan-compliance", kwargs={"pk": scan.id, "name": framework})
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
cd = resp["Content-Disposition"]
|
||||
assert cd.startswith('attachment; filename="')
|
||||
assert cd.endswith('filename="mitre_attack_aws.csv"')
|
||||
assert resp.status_code == status.HTTP_302_FOUND
|
||||
assert resp["Location"] == presigned_url
|
||||
|
||||
def test_compliance_s3_not_found(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
@@ -4251,8 +4282,8 @@ class TestScanViewSet:
|
||||
scan.save()
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.get_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "NoSuchKey"}}, "GetObject"
|
||||
fake_client.head_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "NoSuchKey"}}, "HeadObject"
|
||||
)
|
||||
mock_get_s3_client.return_value = fake_client
|
||||
|
||||
@@ -4275,8 +4306,8 @@ class TestScanViewSet:
|
||||
scan.save()
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.get_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "AccessDenied"}}, "GetObject"
|
||||
fake_client.head_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "AccessDenied"}}, "HeadObject"
|
||||
)
|
||||
mock_get_s3_client.return_value = fake_client
|
||||
|
||||
|
||||
+149
-56
@@ -4,6 +4,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -16,7 +17,7 @@ from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
|
||||
from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter
|
||||
from allauth.socialaccount.providers.saml.views import FinishACSView, LoginView
|
||||
from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
|
||||
from celery import chain
|
||||
from celery import chain, states
|
||||
from celery.result import AsyncResult
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
@@ -53,13 +54,14 @@ from django.db.models import (
|
||||
)
|
||||
from django.db.models.fields.json import KeyTextTransform
|
||||
from django.db.models.functions import Cast, Coalesce, RowNumber
|
||||
from django.http import HttpResponse, QueryDict
|
||||
from django.http import HttpResponse, HttpResponseBase, HttpResponseRedirect, QueryDict
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import reverse
|
||||
from django.utils.dateparse import parse_date
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.views.decorators.cache import cache_control
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from drf_spectacular.settings import spectacular_settings
|
||||
from drf_spectacular.types import OpenApiTypes
|
||||
from drf_spectacular.utils import (
|
||||
@@ -422,7 +424,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.26.0"
|
||||
spectacular_settings.VERSION = "1.28.0"
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -2080,24 +2082,38 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
},
|
||||
)
|
||||
|
||||
def _load_file(self, path_pattern, s3=False, bucket=None, list_objects=False):
|
||||
def _load_file(
|
||||
self,
|
||||
path_pattern,
|
||||
s3=False,
|
||||
bucket=None,
|
||||
list_objects=False,
|
||||
content_type=None,
|
||||
):
|
||||
"""
|
||||
Loads a binary file (e.g., ZIP or CSV) and returns its content and filename.
|
||||
Resolve a report file location and return the bytes (filesystem) or a redirect (S3).
|
||||
|
||||
Depending on the input parameters, this method supports loading:
|
||||
- From S3 using a direct key.
|
||||
- From S3 by listing objects under a prefix and matching suffix.
|
||||
- From the local filesystem using glob pattern matching.
|
||||
- From S3 using a direct key, returns a 302 to a short-lived presigned URL.
|
||||
- From S3 by listing objects under a prefix and matching suffix, returns a 302 to a short-lived presigned URL.
|
||||
- From the local filesystem using glob pattern matching, returns the file bytes.
|
||||
|
||||
The S3 branch never streams bytes through the worker; this prevents gunicorn
|
||||
worker timeouts on large reports.
|
||||
|
||||
Args:
|
||||
path_pattern (str): The key or glob pattern representing the file location.
|
||||
s3 (bool, optional): Whether the file is stored in S3. Defaults to False.
|
||||
bucket (str, optional): The name of the S3 bucket, required if `s3=True`. Defaults to None.
|
||||
list_objects (bool, optional): If True and `s3=True`, list objects by prefix to find the file. Defaults to False.
|
||||
content_type (str, optional): On the S3 branch, forwarded as `ResponseContentType`
|
||||
so the presigned download advertises the same Content-Type the API used to send.
|
||||
Ignored on the filesystem branch.
|
||||
|
||||
Returns:
|
||||
tuple[bytes, str]: A tuple containing the file content as bytes and the filename if successful.
|
||||
Response: A DRF `Response` object with an appropriate status and error detail if an error occurs.
|
||||
tuple[bytes, str]: For the filesystem branch, the file content and filename.
|
||||
HttpResponseRedirect: For the S3 branch on success, a 302 redirect to a presigned `GetObject` URL.
|
||||
Response: For any error path, a DRF `Response` with an appropriate status and detail.
|
||||
"""
|
||||
if s3:
|
||||
try:
|
||||
@@ -2144,25 +2160,45 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
# path_pattern here is prefix, but in compliance we build correct suffix check before
|
||||
key = keys[0]
|
||||
else:
|
||||
# path_pattern is exact key
|
||||
# path_pattern is exact key; HEAD before presigning to preserve the 404 contract.
|
||||
key = path_pattern
|
||||
try:
|
||||
s3_obj = client.get_object(Bucket=bucket, Key=key)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code")
|
||||
if code == "NoSuchKey":
|
||||
try:
|
||||
client.head_object(Bucket=bucket, Key=key)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code")
|
||||
if code in ("NoSuchKey", "404"):
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
{"detail": "There is a problem with credentials."},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
return Response(
|
||||
{"detail": "There is a problem with credentials."},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
content = s3_obj["Body"].read()
|
||||
|
||||
filename = os.path.basename(key)
|
||||
# escape quotes and strip CR/LF so a malformed key cannot break out of the header
|
||||
safe_filename = (
|
||||
filename.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\r", "")
|
||||
.replace("\n", "")
|
||||
)
|
||||
params = {
|
||||
"Bucket": bucket,
|
||||
"Key": key,
|
||||
"ResponseContentDisposition": f'attachment; filename="{safe_filename}"',
|
||||
}
|
||||
if content_type:
|
||||
params["ResponseContentType"] = content_type
|
||||
url = client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params=params,
|
||||
ExpiresIn=300,
|
||||
)
|
||||
return HttpResponseRedirect(url)
|
||||
else:
|
||||
files = glob.glob(path_pattern)
|
||||
if not files:
|
||||
@@ -2205,12 +2241,16 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
loader = self._load_file(
|
||||
key_prefix, s3=True, bucket=bucket, list_objects=False
|
||||
key_prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=False,
|
||||
content_type="application/x-zip-compressed",
|
||||
)
|
||||
else:
|
||||
loader = self._load_file(scan.output_location, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2248,13 +2288,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix), "compliance", f"{name}.csv"
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="text/csv",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "compliance", f"*_{name}.csv")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2287,13 +2333,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"cis",
|
||||
"*_cis_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "cis", "*_cis_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2327,13 +2379,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"threatscore",
|
||||
"*_threatscore_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "threatscore", "*_threatscore_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2367,13 +2425,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"ens",
|
||||
"*_ens_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "ens", "*_ens_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2406,13 +2470,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"nis2",
|
||||
"*_nis2_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "nis2", "*_nis2_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2445,13 +2515,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"csa",
|
||||
"*_csa_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "csa", "*_csa_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2460,28 +2536,45 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
def create(self, request, *args, **kwargs):
|
||||
input_serializer = self.get_serializer(data=request.data)
|
||||
input_serializer.is_valid(raise_exception=True)
|
||||
|
||||
# Broker publish is deferred to on_commit so the worker cannot read
|
||||
# Scan before BaseRLSViewSet's dispatch-wide atomic commits.
|
||||
pre_task_id = str(uuid.uuid4())
|
||||
|
||||
with transaction.atomic():
|
||||
scan = input_serializer.save()
|
||||
with transaction.atomic():
|
||||
task = perform_scan_task.apply_async(
|
||||
kwargs={
|
||||
"tenant_id": self.request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
},
|
||||
scan.task_id = pre_task_id
|
||||
scan.save(update_fields=["task_id"])
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
task_result, _ = TaskResult.objects.get_or_create(
|
||||
task_id=pre_task_id,
|
||||
defaults={"status": states.PENDING, "task_name": "scan-perform"},
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=pre_task_id,
|
||||
tenant_id=self.request.tenant_id,
|
||||
defaults={"task_runner_task": task_result},
|
||||
)
|
||||
|
||||
prowler_task = Task.objects.get(id=task.id)
|
||||
scan.task_id = task.id
|
||||
scan.save(update_fields=["task_id"])
|
||||
scan_kwargs = {
|
||||
"tenant_id": self.request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
}
|
||||
|
||||
transaction.on_commit(
|
||||
lambda: perform_scan_task.apply_async(
|
||||
kwargs=scan_kwargs, task_id=pre_task_id
|
||||
)
|
||||
)
|
||||
|
||||
self.response_serializer_class = TaskSerializer
|
||||
output_serializer = self.get_serializer(prowler_task)
|
||||
|
||||
@@ -49,7 +49,7 @@ def start_aws_ingestion(
|
||||
}
|
||||
|
||||
boto3_session = get_boto3_session(prowler_api_provider, prowler_sdk_provider)
|
||||
regions: list[str] = list(prowler_sdk_provider._enabled_regions)
|
||||
regions: list[str] = resolve_aws_regions(prowler_api_provider, prowler_sdk_provider)
|
||||
requested_syncs = list(cartography_aws.RESOURCE_FUNCTIONS.keys())
|
||||
|
||||
sync_args = cartography_aws._build_aws_sync_kwargs(
|
||||
@@ -226,6 +226,48 @@ def get_boto3_session(
|
||||
return boto3_session
|
||||
|
||||
|
||||
def resolve_aws_regions(
|
||||
prowler_api_provider: ProwlerAPIProvider,
|
||||
prowler_sdk_provider: ProwlerSDKProvider,
|
||||
) -> list[str]:
|
||||
"""Resolve the regions to scan, falling back when `_enabled_regions` is `None`.
|
||||
|
||||
The SDK silently sets `_enabled_regions` to `None` when `ec2:DescribeRegions`
|
||||
fails (missing IAM permission, transient error). Without a fallback the
|
||||
Cartography ingestion crashes with a non-actionable `TypeError`. Try the
|
||||
user's `audited_regions` next, then the partition's static region list.
|
||||
Excluded regions are honored on every branch.
|
||||
"""
|
||||
if prowler_sdk_provider._enabled_regions is not None:
|
||||
regions = set(prowler_sdk_provider._enabled_regions)
|
||||
|
||||
elif prowler_sdk_provider.identity.audited_regions:
|
||||
regions = set(prowler_sdk_provider.identity.audited_regions)
|
||||
|
||||
else:
|
||||
partition = prowler_sdk_provider.identity.partition
|
||||
try:
|
||||
regions = prowler_sdk_provider.get_available_aws_service_regions(
|
||||
"ec2", partition
|
||||
)
|
||||
|
||||
except KeyError:
|
||||
raise RuntimeError(
|
||||
f"No region data available for partition {partition!r}; "
|
||||
f"cannot determine regions to scan for "
|
||||
f"{prowler_api_provider.uid}"
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
f"Could not enumerate enabled regions for AWS account "
|
||||
f"{prowler_api_provider.uid}; falling back to all regions in "
|
||||
f"partition {partition!r}"
|
||||
)
|
||||
|
||||
excluded = set(getattr(prowler_sdk_provider, "_excluded_regions", None) or ())
|
||||
return sorted(regions - excluded)
|
||||
|
||||
|
||||
def get_aioboto3_session(boto3_session: boto3.Session) -> aioboto3.Session:
|
||||
return aioboto3.Session(botocore_session=boto3_session._session)
|
||||
|
||||
|
||||
@@ -18,28 +18,45 @@ logger = get_task_logger(__name__)
|
||||
|
||||
def cleanup_stale_attack_paths_scans() -> dict:
|
||||
"""
|
||||
Find `EXECUTING` `AttackPathsScan` scans whose workers are dead or that have
|
||||
exceeded the stale threshold, and mark them as `FAILED`.
|
||||
Mark stale `AttackPathsScan` rows as `FAILED`.
|
||||
|
||||
Two-pass detection:
|
||||
Covers two stuck-state scenarios:
|
||||
1. `EXECUTING` scans whose workers are dead, or that have exceeded the
|
||||
stale threshold while alive.
|
||||
2. `SCHEDULED` scans that never made it to a worker — parent scan
|
||||
crashed before dispatch, broker lost the message, etc. Detected by
|
||||
age plus the parent `Scan` no longer being in flight.
|
||||
"""
|
||||
threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
cutoff = now - threshold
|
||||
|
||||
cleaned_up: list[str] = []
|
||||
cleaned_up.extend(_cleanup_stale_executing_scans(cutoff))
|
||||
cleaned_up.extend(_cleanup_stale_scheduled_scans(cutoff))
|
||||
|
||||
logger.info(
|
||||
f"Stale `AttackPathsScan` cleanup: {len(cleaned_up)} scan(s) cleaned up"
|
||||
)
|
||||
return {"cleaned_up_count": len(cleaned_up), "scan_ids": cleaned_up}
|
||||
|
||||
|
||||
def _cleanup_stale_executing_scans(cutoff: datetime) -> list[str]:
|
||||
"""
|
||||
Two-pass detection for `EXECUTING` scans:
|
||||
1. If `TaskResult.worker` exists, ping the worker.
|
||||
- Dead worker: cleanup immediately (any age).
|
||||
- Alive + past threshold: revoke the task, then cleanup.
|
||||
- Alive + within threshold: skip.
|
||||
2. If no worker field: fall back to time-based heuristic only.
|
||||
"""
|
||||
threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
cutoff = now - threshold
|
||||
|
||||
executing_scans = (
|
||||
executing_scans = list(
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
.filter(state=StateChoices.EXECUTING)
|
||||
.select_related("task__task_runner_task")
|
||||
)
|
||||
|
||||
# Cache worker liveness so each worker is pinged at most once
|
||||
executing_scans = list(executing_scans)
|
||||
workers = {
|
||||
tr.worker
|
||||
for scan in executing_scans
|
||||
@@ -48,7 +65,7 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
}
|
||||
worker_alive = {w: _is_worker_alive(w) for w in workers}
|
||||
|
||||
cleaned_up = []
|
||||
cleaned_up: list[str] = []
|
||||
|
||||
for scan in executing_scans:
|
||||
task_result = (
|
||||
@@ -65,9 +82,7 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
|
||||
# Alive but stale — revoke before cleanup
|
||||
_revoke_task(task_result)
|
||||
reason = (
|
||||
"Scan exceeded stale threshold — " "cleaned up by periodic task"
|
||||
)
|
||||
reason = "Scan exceeded stale threshold — cleaned up by periodic task"
|
||||
else:
|
||||
reason = "Worker dead — cleaned up by periodic task"
|
||||
else:
|
||||
@@ -82,10 +97,57 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
if _cleanup_scan(scan, task_result, reason):
|
||||
cleaned_up.append(str(scan.id))
|
||||
|
||||
logger.info(
|
||||
f"Stale `AttackPathsScan` cleanup: {len(cleaned_up)} scan(s) cleaned up"
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
"""
|
||||
Cleanup `SCHEDULED` scans that never reached a worker.
|
||||
|
||||
Detection:
|
||||
- `state == SCHEDULED`
|
||||
- `started_at < cutoff`
|
||||
- parent `Scan` is no longer in flight (terminal state or missing). This
|
||||
avoids cleaning up rows whose parent Prowler scan is legitimately still
|
||||
running.
|
||||
|
||||
For each match: revoke the queued task (best-effort; harmless if already
|
||||
consumed), atomically flip to `FAILED`, and mark the `TaskResult`. The
|
||||
temp Neo4j database is never created while `SCHEDULED`, so no drop is
|
||||
needed.
|
||||
"""
|
||||
scheduled_scans = list(
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
.filter(
|
||||
state=StateChoices.SCHEDULED,
|
||||
started_at__lt=cutoff,
|
||||
)
|
||||
.select_related("task__task_runner_task", "scan")
|
||||
)
|
||||
return {"cleaned_up_count": len(cleaned_up), "scan_ids": cleaned_up}
|
||||
|
||||
cleaned_up: list[str] = []
|
||||
parent_terminal = (
|
||||
StateChoices.COMPLETED,
|
||||
StateChoices.FAILED,
|
||||
StateChoices.CANCELLED,
|
||||
)
|
||||
|
||||
for scan in scheduled_scans:
|
||||
parent_scan = scan.scan
|
||||
if parent_scan is not None and parent_scan.state not in parent_terminal:
|
||||
continue
|
||||
|
||||
task_result = (
|
||||
getattr(scan.task, "task_runner_task", None) if scan.task else None
|
||||
)
|
||||
if task_result:
|
||||
_revoke_task(task_result, terminate=False)
|
||||
|
||||
reason = "Scan never started — cleaned up by periodic task"
|
||||
if _cleanup_scheduled_scan(scan, task_result, reason):
|
||||
cleaned_up.append(str(scan.id))
|
||||
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _is_worker_alive(worker: str) -> bool:
|
||||
@@ -98,12 +160,17 @@ def _is_worker_alive(worker: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _revoke_task(task_result) -> None:
|
||||
"""Send `SIGTERM` to a hung Celery task. Non-fatal on failure."""
|
||||
def _revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task. Non-fatal on failure.
|
||||
|
||||
`terminate=True` SIGTERMs the worker if the task is mid-execution; use
|
||||
for EXECUTING cleanup. `terminate=False` only marks the task id revoked
|
||||
across workers, so any worker pulling the queued message discards it;
|
||||
use for SCHEDULED cleanup where the task hasn't run yet.
|
||||
"""
|
||||
try:
|
||||
current_app.control.revoke(
|
||||
task_result.task_id, terminate=True, signal="SIGTERM"
|
||||
)
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
@@ -125,28 +192,64 @@ def _cleanup_scan(scan, task_result, reason: str) -> bool:
|
||||
except Exception:
|
||||
logger.exception(f"Failed to drop temp database {tmp_db_name}")
|
||||
|
||||
# 2. Lock row, verify still EXECUTING, mark FAILED — all atomic
|
||||
with rls_transaction(str(scan.tenant_id)):
|
||||
try:
|
||||
fresh_scan = AttackPathsScan.objects.select_for_update().get(id=scan.id)
|
||||
except AttackPathsScan.DoesNotExist:
|
||||
logger.warning(f"Scan {scan_id_str} no longer exists, skipping")
|
||||
return False
|
||||
fresh_scan = _finalize_failed_scan(scan, StateChoices.EXECUTING, reason)
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
if fresh_scan.state != StateChoices.EXECUTING:
|
||||
logger.info(f"Scan {scan_id_str} is now {fresh_scan.state}, skipping")
|
||||
return False
|
||||
|
||||
_mark_scan_finished(fresh_scan, StateChoices.FAILED, {"global_error": reason})
|
||||
|
||||
# 3. Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock)
|
||||
# Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock)
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=timezone.utc)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
# 4. Recover graph_data_ready if provider data still exists
|
||||
recover_graph_data_ready(fresh_scan)
|
||||
|
||||
logger.info(f"Cleaned up stale scan {scan_id_str}: {reason}")
|
||||
return True
|
||||
|
||||
|
||||
def _cleanup_scheduled_scan(scan, task_result, reason: str) -> bool:
|
||||
"""
|
||||
Clean up a `SCHEDULED` scan that never reached a worker.
|
||||
|
||||
Skips the temp Neo4j drop — the database is only created once the worker
|
||||
enters `EXECUTING`, so dropping it here just produces noisy log output.
|
||||
|
||||
Returns `True` if the scan was actually cleaned up, `False` if skipped.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
|
||||
fresh_scan = _finalize_failed_scan(scan, StateChoices.SCHEDULED, reason)
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=timezone.utc)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
logger.info(f"Cleaned up scheduled scan {scan_id_str}: {reason}")
|
||||
return True
|
||||
|
||||
|
||||
def _finalize_failed_scan(scan, expected_state: str, reason: str):
|
||||
"""
|
||||
Atomically lock the row, verify it's still in `expected_state`, and
|
||||
mark it `FAILED`. Returns the locked row on success, `None` if the
|
||||
row is gone or has already moved on.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
with rls_transaction(str(scan.tenant_id)):
|
||||
try:
|
||||
fresh_scan = AttackPathsScan.objects.select_for_update().get(id=scan.id)
|
||||
except AttackPathsScan.DoesNotExist:
|
||||
logger.warning(f"Scan {scan_id_str} no longer exists, skipping")
|
||||
return None
|
||||
|
||||
if fresh_scan.state != expected_state:
|
||||
logger.info(f"Scan {scan_id_str} is now {fresh_scan.state}, skipping")
|
||||
return None
|
||||
|
||||
_mark_scan_finished(fresh_scan, StateChoices.FAILED, {"global_error": reason})
|
||||
|
||||
return fresh_scan
|
||||
|
||||
@@ -67,25 +67,52 @@ def retrieve_attack_paths_scan(
|
||||
return None
|
||||
|
||||
|
||||
def set_attack_paths_scan_task_id(
|
||||
tenant_id: str,
|
||||
scan_pk: str,
|
||||
task_id: str,
|
||||
) -> None:
|
||||
"""Persist the Celery `task_id` on the `AttackPathsScan` row.
|
||||
|
||||
Called at dispatch time (when `apply_async` returns) so the row carries
|
||||
the task id even while still `SCHEDULED`. This lets the periodic
|
||||
cleanup revoke queued messages for scans that never reached a worker.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
ProwlerAPIAttackPathsScan.objects.filter(id=scan_pk).update(task_id=task_id)
|
||||
|
||||
|
||||
def starting_attack_paths_scan(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
task_id: str,
|
||||
cartography_config: CartographyConfig,
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.task_id = task_id
|
||||
attack_paths_scan.state = StateChoices.EXECUTING
|
||||
attack_paths_scan.started_at = datetime.now(tz=timezone.utc)
|
||||
attack_paths_scan.update_tag = cartography_config.update_tag
|
||||
) -> bool:
|
||||
"""Flip the row from `SCHEDULED` to `EXECUTING` atomically.
|
||||
|
||||
attack_paths_scan.save(
|
||||
update_fields=[
|
||||
"task_id",
|
||||
"state",
|
||||
"started_at",
|
||||
"update_tag",
|
||||
]
|
||||
)
|
||||
Returns `False` if the row is gone or has already moved past
|
||||
`SCHEDULED` (e.g., periodic cleanup raced ahead and marked it
|
||||
`FAILED` while the worker message was still in flight).
|
||||
"""
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
try:
|
||||
locked = ProwlerAPIAttackPathsScan.objects.select_for_update().get(
|
||||
id=attack_paths_scan.id
|
||||
)
|
||||
except ProwlerAPIAttackPathsScan.DoesNotExist:
|
||||
return False
|
||||
|
||||
if locked.state != StateChoices.SCHEDULED:
|
||||
return False
|
||||
|
||||
locked.state = StateChoices.EXECUTING
|
||||
locked.started_at = datetime.now(tz=timezone.utc)
|
||||
locked.update_tag = cartography_config.update_tag
|
||||
locked.save(update_fields=["state", "started_at", "update_tag"])
|
||||
|
||||
# Keep the in-memory object the caller is holding in sync.
|
||||
attack_paths_scan.state = locked.state
|
||||
attack_paths_scan.started_at = locked.started_at
|
||||
attack_paths_scan.update_tag = locked.update_tag
|
||||
return True
|
||||
|
||||
|
||||
def _mark_scan_finished(
|
||||
|
||||
@@ -97,6 +97,19 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
)
|
||||
attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id)
|
||||
|
||||
# Idempotency guard: cleanup may have flipped this row to a terminal state
|
||||
# while the message was still in flight. Bail out before touching state.
|
||||
if attack_paths_scan and attack_paths_scan.state in (
|
||||
StateChoices.FAILED,
|
||||
StateChoices.COMPLETED,
|
||||
StateChoices.CANCELLED,
|
||||
):
|
||||
logger.warning(
|
||||
f"Attack Paths scan {attack_paths_scan.id} already in terminal "
|
||||
f"state {attack_paths_scan.state}; skipping execution"
|
||||
)
|
||||
return {}
|
||||
|
||||
# Checks before starting the scan
|
||||
if not cartography_ingestion_function:
|
||||
ingestion_exceptions = {
|
||||
@@ -114,12 +127,17 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
|
||||
else:
|
||||
if not attack_paths_scan:
|
||||
# Safety net for in-flight messages or direct task invocations; dispatcher normally pre-creates the row.
|
||||
logger.warning(
|
||||
f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then"
|
||||
)
|
||||
attack_paths_scan = db_utils.create_attack_paths_scan(
|
||||
tenant_id, scan_id, prowler_api_provider.id
|
||||
)
|
||||
if attack_paths_scan and task_id:
|
||||
db_utils.set_attack_paths_scan_task_id(
|
||||
tenant_id, attack_paths_scan.id, task_id
|
||||
)
|
||||
|
||||
tmp_database_name = graph_database.get_database_name(
|
||||
attack_paths_scan.id, temporary=True
|
||||
@@ -141,9 +159,13 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
# Starting the Attack Paths scan
|
||||
db_utils.starting_attack_paths_scan(
|
||||
attack_paths_scan, task_id, tenant_cartography_config
|
||||
)
|
||||
if not db_utils.starting_attack_paths_scan(
|
||||
attack_paths_scan, tenant_cartography_config
|
||||
):
|
||||
logger.warning(
|
||||
f"Attack Paths scan {attack_paths_scan.id} no longer in SCHEDULED state; cleanup likely raced ahead"
|
||||
)
|
||||
return {}
|
||||
|
||||
scan_t0 = time.perf_counter()
|
||||
logger.info(
|
||||
|
||||
@@ -47,6 +47,9 @@ from prowler.lib.outputs.compliance.csa.csa_oraclecloud import OracleCloudCSA
|
||||
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
|
||||
from prowler.lib.outputs.compliance.asd_essential_eight.asd_essential_eight_aws import (
|
||||
ASDEssentialEightAWS,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.iso27001.iso27001_aws import AWSISO27001
|
||||
from prowler.lib.outputs.compliance.iso27001.iso27001_azure import AzureISO27001
|
||||
from prowler.lib.outputs.compliance.iso27001.iso27001_gcp import GCPISO27001
|
||||
@@ -100,6 +103,7 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("ccc_"), CCC_AWS),
|
||||
(lambda name: name.startswith("c5_"), AWSC5),
|
||||
(lambda name: name.startswith("csa_"), AWSCSA),
|
||||
(lambda name: name == "asd_essential_eight_aws", ASDEssentialEightAWS),
|
||||
],
|
||||
"azure": [
|
||||
(lambda name: name.startswith("cis_"), AzureCIS),
|
||||
|
||||
@@ -10,16 +10,29 @@ from typing import Any
|
||||
|
||||
import sentry_sdk
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||||
from config.env import env
|
||||
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
|
||||
from django.db import IntegrityError, OperationalError
|
||||
from django.db.models import Case, Count, IntegerField, Max, Min, Prefetch, Q, Sum, When
|
||||
from django.db.models import (
|
||||
Case,
|
||||
Count,
|
||||
Exists,
|
||||
IntegerField,
|
||||
Max,
|
||||
Min,
|
||||
OuterRef,
|
||||
Prefetch,
|
||||
Q,
|
||||
Sum,
|
||||
When,
|
||||
)
|
||||
from django.utils import timezone as django_timezone
|
||||
from tasks.jobs.queries import (
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||||
)
|
||||
from tasks.utils import CustomEncoder
|
||||
from tasks.utils import CustomEncoder, batched
|
||||
|
||||
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
from api.constants import SEVERITY_ORDER
|
||||
@@ -189,8 +202,9 @@ def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
|
||||
"iam_inline_policy_allows_privilege_escalation",
|
||||
},
|
||||
"ec2-imdsv1": {
|
||||
"ec2_instance_imdsv2_enabled"
|
||||
}, # AWS only - IMDSv1 enabled findings
|
||||
"ec2_instance_imdsv2_enabled",
|
||||
"ec2_instance_account_imdsv2_enabled",
|
||||
}, # AWS only - instance-level IMDSv1 exposure and account IMDS defaults
|
||||
}
|
||||
for category_name, check_ids in attack_surface_check_mappings.items():
|
||||
if check_ids is None:
|
||||
@@ -2069,3 +2083,169 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
}
|
||||
|
||||
|
||||
def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dict:
|
||||
"""Zero failed_findings_count for resources missing from a completed full-scope scan.
|
||||
|
||||
Resources that exist in the database for the scan's provider but were not
|
||||
touched by this scan are treated as ephemeral. We keep their historical
|
||||
findings, but reset the denormalized counter that drives the Resources page
|
||||
sort so they stop ranking at the top.
|
||||
|
||||
Skipped (no-op) when:
|
||||
- The scan is not in COMPLETED state.
|
||||
- The scan ran with any scoping filter in scanner_args (partial scope).
|
||||
|
||||
Query design (must scale to 500k+ resources per provider):
|
||||
Phase 1 — collect ephemeral IDs with one anti-join read.
|
||||
Outer filter ``(tenant_id, provider_id, failed_findings_count > 0)``
|
||||
uses ``resources_tenant_provider_idx``. The correlated
|
||||
``NOT EXISTS`` subquery hits the implicit unique index
|
||||
``(tenant_id, scan_id, resource_id)`` on ``ResourceScanSummary``.
|
||||
``NOT EXISTS`` (vs ``NOT IN``) is null-safe and lets the planner
|
||||
choose between hash anti-join and indexed nested-loop anti-join.
|
||||
``.iterator(chunk_size=...)`` skips the queryset cache so memory
|
||||
stays bounded while streaming UUIDs.
|
||||
Phase 2 — UPDATE in fixed-size batches.
|
||||
One large UPDATE would hold row-exclusive locks for seconds and
|
||||
create a WAL spike. Batched UPDATEs by ``id__in`` (~1k rows each)
|
||||
hit the primary key, keep each lock window ~50ms, bound WAL chunks,
|
||||
and let other writers proceed between batches.
|
||||
``failed_findings_count__gt=0`` in the UPDATE is idempotent under
|
||||
concurrent scans and skips no-op rewrites.
|
||||
Reads use the primary DB, not the replica: ``ResourceScanSummary`` rows
|
||||
were written by the same scan task that triggered this one, so replica
|
||||
lag could falsely classify scanned resources as ephemeral.
|
||||
|
||||
Scope detection (``Scan.is_full_scope()``) derives the set of scoping
|
||||
scanner_args from ``prowler.lib.scan.scan.Scan.__init__`` via
|
||||
introspection, so the API can never drift from the SDK's filter
|
||||
contract. Imported scans are also rejected by trigger — they may only
|
||||
cover a partial slice of resources.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
scan = Scan.objects.filter(tenant_id=tenant_id, id=scan_id).first()
|
||||
|
||||
if scan is None:
|
||||
logger.warning(f"Scan {scan_id} not found")
|
||||
return {"status": "skipped", "reason": "scan not found"}
|
||||
|
||||
if scan.state != StateChoices.COMPLETED:
|
||||
logger.info(f"Scan {scan_id} not completed; skipping ephemeral reset")
|
||||
return {"status": "skipped", "reason": "scan not completed"}
|
||||
|
||||
if not scan.is_full_scope():
|
||||
logger.info(
|
||||
f"Scan {scan_id} ran with scoping filters; skipping ephemeral reset"
|
||||
)
|
||||
return {"status": "skipped", "reason": "partial scan scope"}
|
||||
|
||||
# Race protection: if a newer completed full-scope scan exists for this
|
||||
# provider, our ResourceScanSummary set is stale relative to the resources'
|
||||
# current failed_findings_count values (which the newer scan already
|
||||
# refreshed). Wiping based on the older scan would zero counts the newer
|
||||
# scan just set. Skip and let the newer scan's reset task do the work; if
|
||||
# this task was delayed in the queue, that's the correct outcome.
|
||||
# `completed_at__isnull=False` is required: Postgres orders NULL first in
|
||||
# DESC, so a sibling COMPLETED scan with a missing completed_at would sort
|
||||
# as "newest" and incorrectly cause us to skip.
|
||||
with rls_transaction(tenant_id):
|
||||
latest_full_scope_scan_id = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=scan.provider_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("-completed_at", "-inserted_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
)
|
||||
if latest_full_scope_scan_id != scan.id:
|
||||
logger.info(
|
||||
f"Scan {scan_id} is not the latest completed scan for provider "
|
||||
f"{scan.provider_id}; skipping ephemeral reset"
|
||||
)
|
||||
return {"status": "skipped", "reason": "newer scan exists"}
|
||||
|
||||
# Defensive gate: ResourceScanSummary rows are written by perform_prowler_scan
|
||||
# via best-effort bulk_create. If those writes failed silently (or the scan
|
||||
# genuinely produced resources but no summaries were persisted), the
|
||||
# ~Exists(in_scan) anti-join below would classify EVERY resource for this
|
||||
# provider as ephemeral and zero their counts. Bail loudly instead.
|
||||
with rls_transaction(tenant_id):
|
||||
summaries_present = ResourceScanSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists()
|
||||
if scan.unique_resource_count > 0 and not summaries_present:
|
||||
logger.error(
|
||||
f"Scan {scan_id} reports {scan.unique_resource_count} unique "
|
||||
f"resources but no ResourceScanSummary rows are persisted; "
|
||||
f"skipping ephemeral reset to avoid wiping valid counts"
|
||||
)
|
||||
return {"status": "skipped", "reason": "summaries missing"}
|
||||
|
||||
# Stays on the primary DB intentionally. ResourceScanSummary rows are
|
||||
# written by perform_prowler_scan in the same chain that triggered this
|
||||
# task, so replica lag could return an empty/partial summary set; a stale
|
||||
# read here would classify every Resource as ephemeral and wipe valid
|
||||
# failed_findings_count values on the primary. Same rationale as
|
||||
# update_provider_compliance_scores below in this module.
|
||||
# Materializing the ID list (rather than streaming the iterator into
|
||||
# batched UPDATEs) is intentional: it lets the UPDATEs run in their own
|
||||
# short rls_transactions instead of one long transaction holding row locks
|
||||
# on every batch. At 500k UUIDs the peak memory is ~40 MB — acceptable for
|
||||
# a Celery worker — and is the better trade-off versus a multi-second
|
||||
# write-lock window blocking concurrent scans.
|
||||
with rls_transaction(tenant_id):
|
||||
in_scan = ResourceScanSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_id=OuterRef("pk"),
|
||||
)
|
||||
ephemeral_ids = list(
|
||||
Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=scan.provider_id,
|
||||
failed_findings_count__gt=0,
|
||||
)
|
||||
.filter(~Exists(in_scan))
|
||||
.values_list("id", flat=True)
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
)
|
||||
|
||||
if not ephemeral_ids:
|
||||
logger.info(f"No ephemeral resources for scan {scan_id}")
|
||||
return {
|
||||
"status": "completed",
|
||||
"scan_id": str(scan_id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
"reset": 0,
|
||||
}
|
||||
|
||||
total_updated = 0
|
||||
for batch, _ in batched(ephemeral_ids, DJANGO_FINDINGS_BATCH_SIZE):
|
||||
# batched() always yields a final tuple, which is empty when the input
|
||||
# length is an exact multiple of the batch size. Skip it so we don't
|
||||
# issue a no-op UPDATE ... WHERE id IN ().
|
||||
if not batch:
|
||||
continue
|
||||
with rls_transaction(tenant_id):
|
||||
total_updated += Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id__in=batch,
|
||||
failed_findings_count__gt=0,
|
||||
).update(failed_findings_count=0)
|
||||
|
||||
logger.info(
|
||||
f"Ephemeral resource reset for scan {scan_id}: "
|
||||
f"{total_updated} resources zeroed for provider {scan.provider_id}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"scan_id": str(scan_id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
"reset": total_updated,
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ from tasks.jobs.scan import (
|
||||
aggregate_findings,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
reset_ephemeral_resource_findings_count,
|
||||
update_provider_compliance_scores,
|
||||
)
|
||||
from tasks.utils import (
|
||||
@@ -77,6 +78,7 @@ from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
@@ -158,6 +160,13 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
generate_outputs_task.si(
|
||||
scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id
|
||||
),
|
||||
# post-scan task — runs in the parallel group so a
|
||||
# failure cannot cascade into reports or integrations. Its only
|
||||
# prerequisite is that perform_prowler_scan has committed
|
||||
# ResourceScanSummary, which is true by the time this chain fires.
|
||||
reset_ephemeral_resource_findings_count_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
),
|
||||
),
|
||||
group(
|
||||
# Use optimized task that generates both reports with shared queries
|
||||
@@ -173,10 +182,25 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
).apply_async()
|
||||
|
||||
if can_provider_run_attack_paths_scan(tenant_id, provider_id):
|
||||
perform_attack_paths_scan_task.apply_async(
|
||||
# Row is normally created upstream, so this is a safeguard so we can attach the task id below
|
||||
attack_paths_scan = attack_paths_db_utils.retrieve_attack_paths_scan(
|
||||
tenant_id, scan_id
|
||||
)
|
||||
if attack_paths_scan is None:
|
||||
attack_paths_scan = attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id, scan_id, provider_id
|
||||
)
|
||||
|
||||
# Persist the Celery task id so the periodic cleanup can revoke scans stuck in SCHEDULED
|
||||
result = perform_attack_paths_scan_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
|
||||
if attack_paths_scan and result:
|
||||
attack_paths_db_utils.set_attack_paths_scan_task_id(
|
||||
tenant_id, attack_paths_scan.id, result.task_id
|
||||
)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="provider-connection-check")
|
||||
@set_tenant
|
||||
@@ -378,7 +402,8 @@ class AttackPathsScanRLSTask(RLSTask):
|
||||
SDK initialization, or Neo4j configuration errors during setup).
|
||||
"""
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, _einfo):
|
||||
def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002
|
||||
del args # Required by Celery's Task.on_failure signature; not used.
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
scan_id = kwargs.get("scan_id")
|
||||
|
||||
@@ -775,6 +800,32 @@ def aggregate_daily_severity_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_daily_severity(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-reset-ephemeral-resources", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def reset_ephemeral_resource_findings_count_task(tenant_id: str, scan_id: str):
|
||||
"""Reset failed_findings_count for resources missing from a completed full-scope scan.
|
||||
|
||||
Failures are swallowed and returned as a status: this task lives inside the
|
||||
post-scan group, and Celery propagates group-member exceptions into the next
|
||||
chain step — meaning a crash here would block compliance reports and
|
||||
integrations. The reset is purely cosmetic (UI sort optimization), so a
|
||||
bad run is logged and absorbed rather than allowed to cascade.
|
||||
"""
|
||||
try:
|
||||
return reset_ephemeral_resource_findings_count(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — intentionally broad
|
||||
logger.exception(
|
||||
f"reset_ephemeral_resource_findings_count failed for scan {scan_id}: {exc}"
|
||||
)
|
||||
return {
|
||||
"status": "failed",
|
||||
"scan_id": str(scan_id),
|
||||
"reason": str(exc),
|
||||
}
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-finding-group-summaries", queue="overview")
|
||||
@set_tenant(keep_tenant=True)
|
||||
@handle_provider_deletion
|
||||
|
||||
@@ -135,7 +135,7 @@ class TestAttackPathsRun:
|
||||
assert result == ingestion_result
|
||||
mock_retrieve_scan.assert_called_once_with(str(tenant.id), str(scan.id))
|
||||
mock_starting.assert_called_once()
|
||||
config = mock_starting.call_args[0][2]
|
||||
config = mock_starting.call_args[0][1]
|
||||
assert config.neo4j_database == "tenant-db"
|
||||
mock_get_db_name.assert_has_calls(
|
||||
[call(attack_paths_scan.id, temporary=True), call(provider.tenant_id)]
|
||||
@@ -2732,3 +2732,143 @@ class TestCleanupStaleAttackPathsScans:
|
||||
assert result["cleaned_up_count"] == 2
|
||||
# Worker should be pinged exactly once — cache prevents second ping
|
||||
mock_alive.assert_called_once_with("shared-worker@host")
|
||||
|
||||
# `SCHEDULED` state cleanup
|
||||
def _create_scheduled_scan(
|
||||
self,
|
||||
tenant,
|
||||
provider,
|
||||
*,
|
||||
age_minutes,
|
||||
parent_state,
|
||||
with_task=True,
|
||||
):
|
||||
"""Create a SCHEDULED AttackPathsScan with a parent Scan in `parent_state`.
|
||||
|
||||
`age_minutes` controls how far in the past `started_at` is set, so
|
||||
callers can place rows safely past the cleanup cutoff.
|
||||
"""
|
||||
parent_scan = Scan.objects.create(
|
||||
name="Parent Prowler scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=parent_state,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
ap_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=parent_scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
started_at=datetime.now(tz=timezone.utc) - timedelta(minutes=age_minutes),
|
||||
)
|
||||
|
||||
task_result = None
|
||||
if with_task:
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(ap_scan.id),
|
||||
task_name="attack-paths-scan-perform",
|
||||
status="PENDING",
|
||||
)
|
||||
task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
task_runner_task=task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
ap_scan.task = task
|
||||
ap_scan.save(update_fields=["task_id"])
|
||||
|
||||
return ap_scan, task_result
|
||||
|
||||
@patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready")
|
||||
@patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database")
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.cleanup.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.cleanup._revoke_task")
|
||||
def test_cleans_up_scheduled_scan_when_parent_is_terminal(
|
||||
self,
|
||||
mock_revoke,
|
||||
mock_drop_db,
|
||||
mock_recover,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
ap_scan, task_result = self._create_scheduled_scan(
|
||||
tenant,
|
||||
provider,
|
||||
age_minutes=24 * 60 * 3, # 3 days, safely past any threshold
|
||||
parent_state=StateChoices.FAILED,
|
||||
)
|
||||
|
||||
result = cleanup_stale_attack_paths_scans()
|
||||
|
||||
assert result["cleaned_up_count"] == 1
|
||||
assert str(ap_scan.id) in result["scan_ids"]
|
||||
|
||||
ap_scan.refresh_from_db()
|
||||
assert ap_scan.state == StateChoices.FAILED
|
||||
assert ap_scan.progress == 100
|
||||
assert ap_scan.completed_at is not None
|
||||
assert ap_scan.ingestion_exceptions == {
|
||||
"global_error": "Scan never started — cleaned up by periodic task"
|
||||
}
|
||||
|
||||
# SCHEDULED revoke must NOT terminate a running worker
|
||||
mock_revoke.assert_called_once()
|
||||
assert mock_revoke.call_args.kwargs == {"terminate": False}
|
||||
|
||||
# Temp DB never created for SCHEDULED, so no drop attempted
|
||||
mock_drop_db.assert_not_called()
|
||||
# Tenant Neo4j data is untouched in this path
|
||||
mock_recover.assert_not_called()
|
||||
|
||||
task_result.refresh_from_db()
|
||||
assert task_result.status == "FAILURE"
|
||||
assert task_result.date_done is not None
|
||||
|
||||
@patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready")
|
||||
@patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database")
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.cleanup.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.cleanup._revoke_task")
|
||||
def test_skips_scheduled_scan_when_parent_still_in_flight(
|
||||
self,
|
||||
mock_revoke,
|
||||
mock_drop_db,
|
||||
mock_recover,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
ap_scan, _ = self._create_scheduled_scan(
|
||||
tenant,
|
||||
provider,
|
||||
age_minutes=24 * 60 * 3,
|
||||
parent_state=StateChoices.EXECUTING,
|
||||
)
|
||||
|
||||
result = cleanup_stale_attack_paths_scans()
|
||||
|
||||
assert result["cleaned_up_count"] == 0
|
||||
|
||||
ap_scan.refresh_from_db()
|
||||
assert ap_scan.state == StateChoices.SCHEDULED
|
||||
mock_revoke.assert_not_called()
|
||||
|
||||
@@ -24,6 +24,7 @@ from tasks.jobs.scan import (
|
||||
aggregate_findings,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
reset_ephemeral_resource_findings_count,
|
||||
update_provider_compliance_scores,
|
||||
)
|
||||
from tasks.utils import CustomEncoder
|
||||
@@ -35,6 +36,7 @@ from api.models import (
|
||||
MuteRule,
|
||||
Provider,
|
||||
Resource,
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
@@ -3851,6 +3853,7 @@ class TestAggregateAttackSurface:
|
||||
in result["privilege-escalation"]
|
||||
)
|
||||
assert "ec2_instance_imdsv2_enabled" in result["ec2-imdsv1"]
|
||||
assert "ec2_instance_account_imdsv2_enabled" in result["ec2-imdsv1"]
|
||||
|
||||
@patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@@ -4335,3 +4338,315 @@ class TestUpdateProviderComplianceScores:
|
||||
assert any("provider_compliance_scores" in c for c in calls)
|
||||
assert any("tenant_compliance_summaries" in c for c in calls)
|
||||
assert any("pg_advisory_xact_lock" in c for c in calls)
|
||||
|
||||
|
||||
class TestScanIsFullScope:
|
||||
def _live_trigger(self):
|
||||
return Scan.TriggerChoices.MANUAL
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scanner_args",
|
||||
[
|
||||
{},
|
||||
{"unrelated": "value"},
|
||||
{"checks": None},
|
||||
{"services": []},
|
||||
{"severities": ""},
|
||||
],
|
||||
)
|
||||
def test_full_scope_when_no_filters_present(self, scanner_args):
|
||||
scan = Scan(scanner_args=scanner_args, trigger=self._live_trigger())
|
||||
assert scan.is_full_scope() is True
|
||||
|
||||
def test_full_scope_covers_every_sdk_kwarg(self):
|
||||
# Lock the predicate to whatever ProwlerScan's __init__ exposes today.
|
||||
# If the SDK adds a new filter, this test still passes via the
|
||||
# introspection-driven derivation; if it adds a non-filter kwarg
|
||||
# (e.g. provider-like), keep the exclusion list in sync in models.py.
|
||||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
import inspect
|
||||
|
||||
expected = tuple(
|
||||
name
|
||||
for name in inspect.signature(ProwlerScan.__init__).parameters
|
||||
if name not in ("self", "provider")
|
||||
)
|
||||
assert Scan.get_scoping_scanner_arg_keys() == expected
|
||||
# Spot-check a few well-known filters survive the introspection.
|
||||
assert "checks" in expected
|
||||
assert "services" in expected
|
||||
assert "severities" in expected
|
||||
|
||||
def test_partial_scope_for_each_sdk_filter(self):
|
||||
for key in Scan.get_scoping_scanner_arg_keys():
|
||||
scan = Scan(scanner_args={key: ["x"]}, trigger=self._live_trigger())
|
||||
assert scan.is_full_scope() is False, f"{key} should mark scan as partial"
|
||||
|
||||
def test_imported_scan_is_never_full_scope(self):
|
||||
# Forward-defensive: any trigger outside LIVE_SCAN_TRIGGERS (e.g. a
|
||||
# future "imported" trigger) must never qualify, even with empty args.
|
||||
scan = Scan(scanner_args={}, trigger="imported")
|
||||
assert scan.is_full_scope() is False
|
||||
|
||||
def test_handles_none_scanner_args(self):
|
||||
scan = Scan(scanner_args=None, trigger=self._live_trigger())
|
||||
assert scan.is_full_scope() is True
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestResetEphemeralResourceFindingsCount:
|
||||
def _make_scan_summary(self, tenant_id, scan_id, resource):
|
||||
return ResourceScanSummary.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_id=resource.id,
|
||||
service=resource.service,
|
||||
region=resource.region,
|
||||
resource_type=resource.type,
|
||||
)
|
||||
|
||||
def test_resets_only_resources_missing_from_full_scope_scan(
|
||||
self, tenants_fixture, scans_fixture, providers_fixture, resources_fixture
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, scan2, *_ = scans_fixture
|
||||
resource1, resource2, resource3 = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource1.id).update(failed_findings_count=3)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
Resource.objects.filter(id=resource3.id).update(failed_findings_count=7)
|
||||
|
||||
# Only resource1 was scanned in scan1; resource2 is ephemeral.
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["reset"] == 1
|
||||
|
||||
resource1.refresh_from_db()
|
||||
resource2.refresh_from_db()
|
||||
resource3.refresh_from_db()
|
||||
|
||||
assert resource1.failed_findings_count == 3
|
||||
assert resource2.failed_findings_count == 0
|
||||
# Other provider's resource is never touched.
|
||||
assert resource3.failed_findings_count == 7
|
||||
|
||||
def test_skips_when_scan_not_completed(
|
||||
self, tenants_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Scan.objects.filter(id=scan1.id).update(state=StateChoices.EXECUTING)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "scan not completed"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 5
|
||||
|
||||
def test_skips_when_scan_has_scoping_filters(
|
||||
self, tenants_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
_, resource2, _ = resources_fixture
|
||||
|
||||
Scan.objects.filter(id=scan1.id).update(scanner_args={"checks": ["check1"]})
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "partial scan scope"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 5
|
||||
|
||||
def test_skips_when_scan_not_found(self, tenants_fixture):
|
||||
tenant, *_ = tenants_fixture
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(uuid.uuid4())
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "scan not found"
|
||||
|
||||
def test_skips_when_newer_scan_completed_for_same_provider(
|
||||
self, tenants_fixture, scans_fixture, providers_fixture, resources_fixture
|
||||
):
|
||||
# If a newer completed scan exists for the same provider, our
|
||||
# ResourceScanSummary set is stale relative to the resources' current
|
||||
# counts, and applying the diff would corrupt them.
|
||||
from datetime import timedelta
|
||||
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
provider, *_ = providers_fixture
|
||||
_, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
|
||||
# Create a newer COMPLETED scan for the same provider, with an
|
||||
# explicit completed_at strictly after scan1's so ordering is
|
||||
# deterministic regardless of clock resolution.
|
||||
newer_completed_at = scan1.completed_at + timedelta(minutes=5)
|
||||
Scan.objects.create(
|
||||
name="Newer Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
started_at=newer_completed_at,
|
||||
completed_at=newer_completed_at,
|
||||
)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "newer scan exists"
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 5
|
||||
|
||||
def test_does_not_touch_other_providers_resources(
|
||||
self, tenants_fixture, scans_fixture, providers_fixture, resources_fixture
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
_, _, resource3 = resources_fixture
|
||||
|
||||
# resource3 belongs to provider2 with failed_findings_count > 0 and is
|
||||
# not in scan1's summary. It MUST NOT be reset.
|
||||
Resource.objects.filter(id=resource3.id).update(failed_findings_count=9)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["reset"] == 0
|
||||
|
||||
resource3.refresh_from_db()
|
||||
assert resource3.failed_findings_count == 9
|
||||
|
||||
def test_resources_already_zero_are_not_rewritten(
|
||||
self, tenants_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
# Both resources already at 0, neither in summary -> nothing to update.
|
||||
Resource.objects.filter(id=resource1.id).update(failed_findings_count=0)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=0)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["reset"] == 0
|
||||
|
||||
def test_skips_when_summaries_missing_for_scan_with_resources(
|
||||
self, tenants_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
# Catastrophic guard: if a scan reports unique_resource_count > 0 but
|
||||
# no ResourceScanSummary rows are persisted (e.g. bulk_create silently
|
||||
# failed), the anti-join would classify EVERY resource as ephemeral
|
||||
# and zero their counts. The gate must skip and preserve the data.
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Scan.objects.filter(id=scan1.id).update(unique_resource_count=10)
|
||||
Resource.objects.filter(id=resource1.id).update(failed_findings_count=3)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "summaries missing"
|
||||
|
||||
resource1.refresh_from_db()
|
||||
resource2.refresh_from_db()
|
||||
assert resource1.failed_findings_count == 3
|
||||
assert resource2.failed_findings_count == 5
|
||||
|
||||
def test_ignores_sibling_scan_with_null_completed_at(
|
||||
self, tenants_fixture, scans_fixture, providers_fixture, resources_fixture
|
||||
):
|
||||
# Postgres orders NULL first in DESC; a sibling COMPLETED scan with a
|
||||
# missing completed_at must not be treated as the latest scan and
|
||||
# cause us to incorrectly skip the reset.
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
provider, *_ = providers_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=5)
|
||||
self._make_scan_summary(tenant.id, scan1.id, resource1)
|
||||
|
||||
Scan.objects.create(
|
||||
name="Ghost Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
started_at=scan1.completed_at,
|
||||
completed_at=None,
|
||||
)
|
||||
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["reset"] == 1
|
||||
|
||||
resource2.refresh_from_db()
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
def test_batches_updates_when_many_ephemeral_resources(
|
||||
self, tenants_fixture, scans_fixture, resources_fixture
|
||||
):
|
||||
# Forces multiple batches to confirm the chunked UPDATE path executes
|
||||
# cleanly and the count is the sum across batches.
|
||||
tenant, *_ = tenants_fixture
|
||||
scan1, *_ = scans_fixture
|
||||
resource1, resource2, _ = resources_fixture
|
||||
|
||||
Resource.objects.filter(id=resource1.id).update(failed_findings_count=2)
|
||||
Resource.objects.filter(id=resource2.id).update(failed_findings_count=4)
|
||||
|
||||
# No ResourceScanSummary -> both resource1 and resource2 are ephemeral.
|
||||
# Force a 1-row batch via the shared findings batch size knob.
|
||||
with patch("tasks.jobs.scan.DJANGO_FINDINGS_BATCH_SIZE", 1):
|
||||
result = reset_ephemeral_resource_findings_count(
|
||||
tenant_id=str(tenant.id), scan_id=str(scan1.id)
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["reset"] == 2
|
||||
|
||||
resource1.refresh_from_db()
|
||||
resource2.refresh_from_db()
|
||||
assert resource1.failed_findings_count == 0
|
||||
assert resource2.failed_findings_count == 0
|
||||
|
||||
@@ -842,6 +842,72 @@ class TestScanCompleteTasks:
|
||||
# Attack Paths task should be skipped when provider cannot run it
|
||||
mock_attack_paths_task.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"row_pre_existing",
|
||||
[True, False],
|
||||
ids=["row-pre-existing", "row-missing-fallback"],
|
||||
)
|
||||
@patch("tasks.tasks.aggregate_attack_surface_task.apply_async")
|
||||
@patch("tasks.tasks.chain")
|
||||
@patch("tasks.tasks.create_compliance_requirements_task.si")
|
||||
@patch("tasks.tasks.update_provider_compliance_scores_task.si")
|
||||
@patch("tasks.tasks.perform_scan_summary_task.si")
|
||||
@patch("tasks.tasks.generate_outputs_task.si")
|
||||
@patch("tasks.tasks.generate_compliance_reports_task.si")
|
||||
@patch("tasks.tasks.check_integrations_task.si")
|
||||
@patch("tasks.tasks.attack_paths_db_utils.set_attack_paths_scan_task_id")
|
||||
@patch("tasks.tasks.attack_paths_db_utils.create_attack_paths_scan")
|
||||
@patch("tasks.tasks.attack_paths_db_utils.retrieve_attack_paths_scan")
|
||||
@patch("tasks.tasks.perform_attack_paths_scan_task.apply_async")
|
||||
@patch("tasks.tasks.can_provider_run_attack_paths_scan", return_value=True)
|
||||
def test_scan_complete_dispatches_attack_paths_scan(
|
||||
self,
|
||||
_mock_can_run_attack_paths,
|
||||
mock_attack_paths_task,
|
||||
mock_retrieve,
|
||||
mock_create,
|
||||
mock_set_task_id,
|
||||
mock_check_integrations_task,
|
||||
mock_compliance_reports_task,
|
||||
mock_outputs_task,
|
||||
mock_scan_summary_task,
|
||||
mock_update_compliance_scores_task,
|
||||
mock_compliance_requirements_task,
|
||||
mock_chain,
|
||||
mock_attack_surface_task,
|
||||
row_pre_existing,
|
||||
):
|
||||
"""When a provider can run Attack Paths, dispatch must:
|
||||
1. Reuse the existing row or create one if missing.
|
||||
2. Call apply_async on the Attack Paths task.
|
||||
3. Persist the returned Celery task id on the row.
|
||||
"""
|
||||
existing_row = MagicMock(id="ap-scan-id")
|
||||
if row_pre_existing:
|
||||
mock_retrieve.return_value = existing_row
|
||||
else:
|
||||
mock_retrieve.return_value = None
|
||||
mock_create.return_value = existing_row
|
||||
|
||||
async_result = MagicMock(task_id="celery-task-id")
|
||||
mock_attack_paths_task.return_value = async_result
|
||||
|
||||
_perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id")
|
||||
|
||||
mock_retrieve.assert_called_once_with("tenant-id", "scan-id")
|
||||
if row_pre_existing:
|
||||
mock_create.assert_not_called()
|
||||
else:
|
||||
mock_create.assert_called_once_with("tenant-id", "scan-id", "provider-id")
|
||||
|
||||
mock_attack_paths_task.assert_called_once_with(
|
||||
kwargs={"tenant_id": "tenant-id", "scan_id": "scan-id"}
|
||||
)
|
||||
|
||||
mock_set_task_id.assert_called_once_with(
|
||||
"tenant-id", "ap-scan-id", "celery-task-id"
|
||||
)
|
||||
|
||||
|
||||
class TestAttackPathsTasks:
|
||||
@staticmethod
|
||||
|
||||
Generated
+5983
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,9 @@
|
||||
#!/bin/bash
|
||||
# Run Prowler against All AWS Accounts in an AWS Organization
|
||||
|
||||
# Activate Poetry Environment
|
||||
eval "$(poetry env activate)"
|
||||
# Activate uv-managed virtualenv
|
||||
# shellcheck disable=SC1091
|
||||
source .venv/bin/activate
|
||||
|
||||
# Show Prowler Version
|
||||
prowler -v
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
# AWS Inventory Connectivity Graph
|
||||
|
||||
A community-contributed tool that generates interactive connectivity graphs from Prowler AWS scans, visualizing relationships between AWS resources with zero additional API calls.
|
||||
|
||||
## Overview
|
||||
|
||||
This tool extends Prowler by producing two artifacts after a scan completes:
|
||||
|
||||
- **`<output>.inventory.json`** – Machine-readable graph (nodes + edges)
|
||||
- **`<output>.inventory.html`** – Interactive D3.js force-directed visualization
|
||||
|
||||
### Why?
|
||||
|
||||
Prowler's existing outputs (CSV, ASFF, OCSF, HTML) report individual check findings but provide no cross-service topology view. Security engineers need to understand **how** resources are connected—which Lambda functions sit inside which VPC, which IAM roles can be assumed by which services, which event sources trigger which functions—before they can reason about attack paths, blast-radius, or lateral-movement risk.
|
||||
|
||||
This tool fills that gap by building a connectivity graph from the service clients that are already loaded during a Prowler scan.
|
||||
|
||||
## Features
|
||||
|
||||
### Supported AWS Services
|
||||
|
||||
The tool currently extracts connectivity information from:
|
||||
|
||||
- **Lambda** – Functions, VPC/subnet/SG edges, event source mappings, layers, DLQ, KMS
|
||||
- **EC2** – Instances, security groups, subnet/VPC edges
|
||||
- **VPC** – VPCs, subnets, peering connections
|
||||
- **RDS** – DB instances, VPC/SG/cluster/KMS edges
|
||||
- **ELBv2** – ALB/NLB load balancers, SG and VPC edges
|
||||
- **S3** – Buckets, replication targets, logging buckets, KMS keys
|
||||
- **IAM** – Roles, trust-relationship edges (who can assume what)
|
||||
|
||||
### Edge Semantic Types
|
||||
|
||||
Edges are typed for downstream filtering and attack-path analysis:
|
||||
|
||||
- `network` – Resources share a network path (VPC/subnet/SG)
|
||||
- `iam` – IAM trust or permission relationship
|
||||
- `triggers` – One resource can invoke another (event source → Lambda)
|
||||
- `data_flow` – Data is written/read (Lambda → SQS dead-letter queue)
|
||||
- `depends_on` – Soft dependency (Lambda layer, subnet belongs to VPC)
|
||||
- `routes_to` – Traffic routing (LB → target)
|
||||
- `replicates_to` – S3 replication
|
||||
- `encrypts` – KMS key encrypts the resource
|
||||
- `logs_to` – Logging relationship
|
||||
|
||||
### Interactive HTML Graph Features
|
||||
|
||||
- Force-directed layout with drag-and-drop node pinning
|
||||
- Zoom / pan (mouse wheel + click-drag on background)
|
||||
- Per-service color-coded nodes with a legend
|
||||
- Hover tooltips showing ARN + all metadata properties
|
||||
- Service filter dropdown (show only Lambda, EC2, RDS, etc.)
|
||||
- Adjustable link-distance and charge-strength physics sliders
|
||||
- Edge labels on every arrow
|
||||
|
||||
## Installation
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.9.1 or higher
|
||||
- Prowler installed and configured (see [Prowler documentation](https://docs.prowler.com/))
|
||||
|
||||
### Setup
|
||||
|
||||
1. Clone or download this directory to your local machine
|
||||
2. Ensure Prowler is installed and working
|
||||
3. No additional dependencies required beyond Prowler's existing requirements
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
Run Prowler with your desired checks, then use the inventory graph script:
|
||||
|
||||
```bash
|
||||
# Run Prowler scan (example)
|
||||
prowler aws --output-formats csv
|
||||
|
||||
# Generate inventory graph from the scan
|
||||
python contrib/inventory-graph/inventory_graph.py --output-directory ./output
|
||||
```
|
||||
|
||||
### Command-Line Options
|
||||
|
||||
```bash
|
||||
python contrib/inventory-graph/inventory_graph.py [OPTIONS]
|
||||
|
||||
Options:
|
||||
--output-directory DIR Directory to save output files (default: ./output)
|
||||
--output-filename NAME Base filename without extension (default: prowler-inventory-<timestamp>)
|
||||
--help Show this help message and exit
|
||||
```
|
||||
|
||||
### Example Workflow
|
||||
|
||||
```bash
|
||||
# 1. Run a Prowler scan on your AWS account
|
||||
prowler aws --profile my-aws-profile --output-formats csv html
|
||||
|
||||
# 2. Generate the inventory graph
|
||||
python contrib/inventory-graph/inventory_graph.py \
|
||||
--output-directory ./output \
|
||||
--output-filename my-aws-inventory
|
||||
|
||||
# 3. Open the HTML file in your browser
|
||||
open output/my-aws-inventory.inventory.html
|
||||
```
|
||||
|
||||
### Integration with Prowler Scans
|
||||
|
||||
The tool reads from already-loaded AWS service clients in memory (via `sys.modules`). This means:
|
||||
|
||||
- **Zero extra AWS API calls** – Uses data already collected during the Prowler scan
|
||||
- **Graceful degradation** – Services not scanned are silently skipped
|
||||
- **Flexible** – Works with any subset of Prowler checks
|
||||
|
||||
## Output Files
|
||||
|
||||
### JSON Output (`*.inventory.json`)
|
||||
|
||||
Machine-readable graph structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"generated_at": "2026-03-19T12:34:56Z",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "arn:aws:lambda:us-east-1:123456789012:function:my-function",
|
||||
"type": "lambda_function",
|
||||
"name": "my-function",
|
||||
"service": "lambda",
|
||||
"region": "us-east-1",
|
||||
"account_id": "123456789012",
|
||||
"properties": {
|
||||
"runtime": "python3.9",
|
||||
"vpc_id": "vpc-abc123"
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source_id": "arn:aws:lambda:...",
|
||||
"target_id": "arn:aws:ec2:...:vpc/vpc-abc123",
|
||||
"edge_type": "network",
|
||||
"label": "in-vpc"
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"node_count": 42,
|
||||
"edge_count": 87
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### HTML Output (`*.inventory.html`)
|
||||
|
||||
Self-contained interactive visualization that opens in any modern browser. No server or build step required.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Design Decisions
|
||||
|
||||
| Decision | Rationale |
|
||||
|----------|-----------|
|
||||
| **Read from sys.modules** | Zero extra AWS API calls; services not scanned are silently skipped |
|
||||
| **Self-contained HTML** | D3.js v7 via CDN; no server, no build step; opens in any browser |
|
||||
| **One extractor per service** | Each extractor is independently testable; adding a new service = one new file + one line in the registry |
|
||||
| **Typed edges** | Semantic types allow downstream consumers (attack-path tools, Neo4j import) to filter by relationship class |
|
||||
|
||||
### Project Structure
|
||||
|
||||
```
|
||||
contrib/inventory-graph/
|
||||
├── README.md # This file
|
||||
├── inventory_graph.py # Main entry point script
|
||||
├── lib/
|
||||
│ ├── __init__.py
|
||||
│ ├── models.py # ResourceNode, ResourceEdge, ConnectivityGraph dataclasses
|
||||
│ ├── graph_builder.py # Reads loaded service clients from sys.modules
|
||||
│ ├── inventory_output.py # write_json(), write_html()
|
||||
│ └── extractors/
|
||||
│ ├── __init__.py
|
||||
│ ├── lambda_extractor.py # Lambda functions → VPC/subnet/SG/event-sources/layers/DLQ/KMS
|
||||
│ ├── ec2_extractor.py # EC2 instances + security groups → subnet/VPC
|
||||
│ ├── vpc_extractor.py # VPCs, subnets, peering connections
|
||||
│ ├── rds_extractor.py # RDS instances → VPC/SG/cluster/KMS
|
||||
│ ├── elbv2_extractor.py # ALB/NLB load balancers → SG/VPC
|
||||
│ ├── s3_extractor.py # S3 buckets → replication targets/logging buckets/KMS keys
|
||||
│ └── iam_extractor.py # IAM roles + trust-relationship edges
|
||||
└── examples/
|
||||
└── sample_output.html # Example output (optional)
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Smoke Test (No AWS Credentials Needed)
|
||||
|
||||
```python
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Wire a fake Lambda client
|
||||
mock_module = MagicMock()
|
||||
mock_fn = MagicMock()
|
||||
mock_fn.arn = "arn:aws:lambda:us-east-1:123:function:test"
|
||||
mock_fn.name = "test"
|
||||
mock_fn.region = "us-east-1"
|
||||
mock_fn.vpc_id = "vpc-abc"
|
||||
mock_fn.security_groups = ["sg-111"]
|
||||
mock_fn.subnet_ids = {"subnet-aaa"}
|
||||
mock_fn.environment = None
|
||||
mock_fn.kms_key_arn = None
|
||||
mock_fn.layers = []
|
||||
mock_fn.dead_letter_config = None
|
||||
mock_fn.event_source_mappings = []
|
||||
mock_module.awslambda_client.functions = {mock_fn.arn: mock_fn}
|
||||
mock_module.awslambda_client.audited_account = "123"
|
||||
sys.modules["prowler.providers.aws.services.awslambda.awslambda_client"] = mock_module
|
||||
|
||||
from contrib.inventory_graph.lib.graph_builder import build_graph
|
||||
from contrib.inventory_graph.lib.inventory_output import write_json, write_html
|
||||
|
||||
graph = build_graph()
|
||||
write_json(graph, "/tmp/test.inventory.json")
|
||||
write_html(graph, "/tmp/test.inventory.html")
|
||||
# Open /tmp/test.inventory.html in a browser
|
||||
```
|
||||
|
||||
## Extending
|
||||
|
||||
### Adding a New Service
|
||||
|
||||
1. Create a new extractor file in `lib/extractors/` (e.g., `dynamodb_extractor.py`)
|
||||
2. Implement the `extract(client)` function that returns `(nodes, edges)`
|
||||
3. Register it in `lib/graph_builder.py` in the `_SERVICE_REGISTRY` tuple
|
||||
|
||||
Example extractor template:
|
||||
|
||||
```python
|
||||
from typing import List, Tuple
|
||||
from prowler.lib.outputs.inventory.models import ResourceNode, ResourceEdge
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""Extract DynamoDB tables and their relationships."""
|
||||
nodes = []
|
||||
edges = []
|
||||
|
||||
for table in client.tables:
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=table.arn,
|
||||
type="dynamodb_table",
|
||||
name=table.name,
|
||||
service="dynamodb",
|
||||
region=table.region,
|
||||
account_id=client.audited_account,
|
||||
properties={"billing_mode": table.billing_mode}
|
||||
)
|
||||
)
|
||||
|
||||
# Add edges for KMS encryption, streams, etc.
|
||||
if table.kms_key_arn:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=table.kms_key_arn,
|
||||
target_id=table.arn,
|
||||
edge_type="encrypts",
|
||||
label="encrypts"
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### No nodes discovered
|
||||
|
||||
**Problem:** The tool reports "no nodes discovered" after running.
|
||||
|
||||
**Solution:** Ensure you've run a Prowler scan first. The tool reads from in-memory service clients loaded during the scan. If no services were scanned, no nodes will be discovered.
|
||||
|
||||
### Missing services in the graph
|
||||
|
||||
**Problem:** Some AWS services are not appearing in the graph.
|
||||
|
||||
**Solution:** The tool only includes services that have been scanned by Prowler. Run Prowler with the services you want to include, or run without service filters to scan all available services.
|
||||
|
||||
### HTML file doesn't display properly
|
||||
|
||||
**Problem:** The HTML visualization doesn't load or shows errors.
|
||||
|
||||
**Solution:**
|
||||
- Ensure you're opening the file in a modern browser (Chrome, Firefox, Safari, Edge)
|
||||
- Check your browser's console for JavaScript errors
|
||||
- Verify the file was generated completely (check file size > 0)
|
||||
- The HTML requires internet access to load D3.js from CDN
|
||||
|
||||
## Roadmap
|
||||
|
||||
Potential future enhancements:
|
||||
|
||||
- [ ] Support for additional AWS services (DynamoDB, SQS, SNS, etc.)
|
||||
- [ ] Export to Neo4j / Cartography format
|
||||
- [ ] Attack path analysis integration
|
||||
- [ ] Multi-account/multi-region aggregation
|
||||
- [ ] Custom edge type filtering in HTML UI
|
||||
- [ ] Graph diff between two scans
|
||||
|
||||
## Contributing
|
||||
|
||||
This is a community contribution. If you'd like to enhance it:
|
||||
|
||||
1. Fork the Prowler repository
|
||||
2. Make your changes in `contrib/inventory-graph/`
|
||||
3. Test thoroughly
|
||||
4. Submit a pull request with a clear description
|
||||
|
||||
## License
|
||||
|
||||
This tool is part of the Prowler project and is licensed under the Apache License 2.0.
|
||||
|
||||
## Credits
|
||||
|
||||
- **Author:** [@sandiyochristan](https://github.com/sandiyochristan)
|
||||
- **Related PR:** [#10382](https://github.com/prowler-cloud/prowler/pull/10382)
|
||||
- **Prowler Project:** [prowler-cloud/prowler](https://github.com/prowler-cloud/prowler)
|
||||
|
||||
## Support
|
||||
|
||||
For issues or questions:
|
||||
|
||||
- Open an issue in the [Prowler repository](https://github.com/prowler-cloud/prowler/issues)
|
||||
- Join the [Prowler Community Slack](https://goto.prowler.com/slack)
|
||||
- Tag your issue with `contrib:inventory-graph`
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example: Generate AWS Inventory Graph with Mock Data
|
||||
|
||||
This example demonstrates how to use the inventory graph tool with mock AWS data.
|
||||
No AWS credentials required.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from lib.graph_builder import build_graph
|
||||
from lib.inventory_output import write_json, write_html
|
||||
|
||||
|
||||
def create_mock_lambda_client():
|
||||
"""Create a mock Lambda client with sample data."""
|
||||
mock_module = MagicMock()
|
||||
|
||||
# Create a mock Lambda function
|
||||
mock_fn = MagicMock()
|
||||
mock_fn.arn = "arn:aws:lambda:us-east-1:123456789012:function:my-test-function"
|
||||
mock_fn.name = "my-test-function"
|
||||
mock_fn.region = "us-east-1"
|
||||
mock_fn.vpc_id = "vpc-abc123"
|
||||
mock_fn.security_groups = ["sg-111222"]
|
||||
mock_fn.subnet_ids = {"subnet-aaa111", "subnet-bbb222"}
|
||||
mock_fn.environment = {"Variables": {"ENV": "production"}}
|
||||
mock_fn.kms_key_arn = (
|
||||
"arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
|
||||
)
|
||||
mock_fn.layers = []
|
||||
mock_fn.dead_letter_config = None
|
||||
mock_fn.event_source_mappings = []
|
||||
|
||||
mock_module.awslambda_client.functions = {mock_fn.arn: mock_fn}
|
||||
mock_module.awslambda_client.audited_account = "123456789012"
|
||||
|
||||
return mock_module
|
||||
|
||||
|
||||
def create_mock_ec2_client():
|
||||
"""Create a mock EC2 client with sample data."""
|
||||
mock_module = MagicMock()
|
||||
|
||||
# Create a mock EC2 instance
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.arn = (
|
||||
"arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef0"
|
||||
)
|
||||
mock_instance.id = "i-1234567890abcdef0"
|
||||
mock_instance.region = "us-east-1"
|
||||
mock_instance.vpc_id = "vpc-abc123"
|
||||
mock_instance.subnet_id = "subnet-aaa111"
|
||||
mock_instance.security_groups = [MagicMock(id="sg-111222")]
|
||||
mock_instance.state = "running"
|
||||
mock_instance.type = "t3.micro"
|
||||
mock_instance.tags = [{"Key": "Name", "Value": "test-instance"}]
|
||||
|
||||
# Create a mock security group
|
||||
mock_sg = MagicMock()
|
||||
mock_sg.arn = "arn:aws:ec2:us-east-1:123456789012:security-group/sg-111222"
|
||||
mock_sg.id = "sg-111222"
|
||||
mock_sg.name = "test-security-group"
|
||||
mock_sg.region = "us-east-1"
|
||||
mock_sg.vpc_id = "vpc-abc123"
|
||||
|
||||
mock_module.ec2_client.instances = [mock_instance]
|
||||
mock_module.ec2_client.security_groups = [mock_sg]
|
||||
mock_module.ec2_client.audited_account = "123456789012"
|
||||
|
||||
return mock_module
|
||||
|
||||
|
||||
def create_mock_vpc_client():
|
||||
"""Create a mock VPC client with sample data."""
|
||||
mock_module = MagicMock()
|
||||
|
||||
# Create a mock VPC
|
||||
mock_vpc = MagicMock()
|
||||
mock_vpc.arn = "arn:aws:ec2:us-east-1:123456789012:vpc/vpc-abc123"
|
||||
mock_vpc.id = "vpc-abc123"
|
||||
mock_vpc.region = "us-east-1"
|
||||
mock_vpc.cidr_block = "10.0.0.0/16"
|
||||
mock_vpc.tags = [{"Key": "Name", "Value": "test-vpc"}]
|
||||
|
||||
# Create mock subnets
|
||||
mock_subnet1 = MagicMock()
|
||||
mock_subnet1.arn = "arn:aws:ec2:us-east-1:123456789012:subnet/subnet-aaa111"
|
||||
mock_subnet1.id = "subnet-aaa111"
|
||||
mock_subnet1.region = "us-east-1"
|
||||
mock_subnet1.vpc_id = "vpc-abc123"
|
||||
mock_subnet1.cidr_block = "10.0.1.0/24"
|
||||
mock_subnet1.availability_zone = "us-east-1a"
|
||||
|
||||
mock_subnet2 = MagicMock()
|
||||
mock_subnet2.arn = "arn:aws:ec2:us-east-1:123456789012:subnet/subnet-bbb222"
|
||||
mock_subnet2.id = "subnet-bbb222"
|
||||
mock_subnet2.region = "us-east-1"
|
||||
mock_subnet2.vpc_id = "vpc-abc123"
|
||||
mock_subnet2.cidr_block = "10.0.2.0/24"
|
||||
mock_subnet2.availability_zone = "us-east-1b"
|
||||
|
||||
mock_module.vpc_client.vpcs = [mock_vpc]
|
||||
mock_module.vpc_client.subnets = [mock_subnet1, mock_subnet2]
|
||||
mock_module.vpc_client.vpc_peering_connections = []
|
||||
mock_module.vpc_client.audited_account = "123456789012"
|
||||
|
||||
return mock_module
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to demonstrate the inventory graph generation."""
|
||||
print("=" * 70)
|
||||
print("AWS Inventory Graph - Mock Data Example")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
# Create mock clients and inject them into sys.modules
|
||||
print("Creating mock AWS service clients...")
|
||||
sys.modules["prowler.providers.aws.services.awslambda.awslambda_client"] = (
|
||||
create_mock_lambda_client()
|
||||
)
|
||||
sys.modules["prowler.providers.aws.services.ec2.ec2_client"] = (
|
||||
create_mock_ec2_client()
|
||||
)
|
||||
sys.modules["prowler.providers.aws.services.vpc.vpc_client"] = (
|
||||
create_mock_vpc_client()
|
||||
)
|
||||
print("✓ Mock clients created")
|
||||
print()
|
||||
|
||||
# Build the graph
|
||||
print("Building connectivity graph...")
|
||||
graph = build_graph()
|
||||
print(f"✓ Graph built: {len(graph.nodes)} nodes, {len(graph.edges)} edges")
|
||||
print()
|
||||
|
||||
# Display discovered nodes
|
||||
print("Discovered nodes:")
|
||||
for node in graph.nodes:
|
||||
print(f" - {node.type}: {node.name} ({node.region})")
|
||||
print()
|
||||
|
||||
# Display discovered edges
|
||||
print("Discovered edges:")
|
||||
for edge in graph.edges:
|
||||
source_node = next((n for n in graph.nodes if n.id == edge.source_id), None)
|
||||
target_node = next((n for n in graph.nodes if n.id == edge.target_id), None)
|
||||
source_name = source_node.name if source_node else edge.source_id
|
||||
target_name = target_node.name if target_node else edge.target_id
|
||||
print(f" - {source_name} --[{edge.edge_type}]--> {target_name}")
|
||||
print()
|
||||
|
||||
# Write outputs
|
||||
output_dir = Path(__file__).parent
|
||||
json_path = output_dir / "example_output.inventory.json"
|
||||
html_path = output_dir / "example_output.inventory.html"
|
||||
|
||||
print("Writing output files...")
|
||||
write_json(graph, str(json_path))
|
||||
write_html(graph, str(html_path))
|
||||
print(f"✓ JSON written to: {json_path}")
|
||||
print(f"✓ HTML written to: {html_path}")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("✓ Example complete!")
|
||||
print("=" * 70)
|
||||
print()
|
||||
print(f"Open the HTML file to view the interactive graph:")
|
||||
print(f" open {html_path}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+158
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
AWS Inventory Connectivity Graph Generator
|
||||
|
||||
A standalone tool that generates interactive connectivity graphs from Prowler AWS scans.
|
||||
This tool reads from already-loaded AWS service clients in memory and produces:
|
||||
- JSON graph (nodes + edges)
|
||||
- Interactive HTML visualization
|
||||
|
||||
Usage:
|
||||
python inventory_graph.py --output-directory ./output --output-filename my-inventory
|
||||
|
||||
For more information, see README.md
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# Add the contrib directory to the path so we can import the lib modules
|
||||
CONTRIB_DIR = Path(__file__).parent
|
||||
sys.path.insert(0, str(CONTRIB_DIR))
|
||||
|
||||
from lib.graph_builder import build_graph
|
||||
from lib.inventory_output import write_json, write_html
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""Parse command-line arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate AWS inventory connectivity graph from Prowler scan data",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Generate graph with default settings
|
||||
python inventory_graph.py
|
||||
|
||||
# Specify custom output directory and filename
|
||||
python inventory_graph.py --output-directory ./my-output --output-filename aws-inventory
|
||||
|
||||
# After running a Prowler scan
|
||||
prowler aws --profile my-profile
|
||||
python inventory_graph.py --output-directory ./output
|
||||
|
||||
For more information, see README.md
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output-directory",
|
||||
"-o",
|
||||
default="./output",
|
||||
help="Directory to save output files (default: ./output)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output-filename",
|
||||
"-f",
|
||||
default=None,
|
||||
help="Base filename without extension (default: prowler-inventory-<timestamp>)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
"-v",
|
||||
action="store_true",
|
||||
help="Enable verbose output",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the inventory graph generator."""
|
||||
args = parse_arguments()
|
||||
|
||||
# Set up output paths
|
||||
output_dir = Path(args.output_directory)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Generate filename with timestamp if not provided
|
||||
if args.output_filename:
|
||||
base_filename = args.output_filename
|
||||
else:
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
base_filename = f"prowler-inventory-{timestamp}"
|
||||
|
||||
json_path = output_dir / f"{base_filename}.inventory.json"
|
||||
html_path = output_dir / f"{base_filename}.inventory.html"
|
||||
|
||||
print("=" * 70)
|
||||
print("AWS Inventory Connectivity Graph Generator")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
# Build the graph from loaded service clients
|
||||
if args.verbose:
|
||||
print("Building connectivity graph from loaded AWS service clients...")
|
||||
|
||||
graph = build_graph()
|
||||
|
||||
# Check if any nodes were discovered
|
||||
if not graph.nodes:
|
||||
print("⚠️ WARNING: No nodes discovered!")
|
||||
print()
|
||||
print("This usually means:")
|
||||
print(" 1. No Prowler scan has been run yet in this Python session")
|
||||
print(" 2. No AWS service clients are loaded in memory")
|
||||
print()
|
||||
print("To fix this:")
|
||||
print(" 1. Run a Prowler scan first: prowler aws --output-formats csv")
|
||||
print(" 2. Then run this script in the same session")
|
||||
print()
|
||||
print(
|
||||
"Alternatively, integrate this tool directly into Prowler's output pipeline."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"✓ Discovered {len(graph.nodes)} nodes and {len(graph.edges)} edges")
|
||||
print()
|
||||
|
||||
# Write outputs
|
||||
if args.verbose:
|
||||
print(f"Writing JSON output to: {json_path}")
|
||||
write_json(graph, str(json_path))
|
||||
|
||||
if args.verbose:
|
||||
print(f"Writing HTML output to: {html_path}")
|
||||
write_html(graph, str(html_path))
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("✓ Graph generation complete!")
|
||||
print("=" * 70)
|
||||
print()
|
||||
print(f"📄 JSON: {json_path}")
|
||||
print(f"🌐 HTML: {html_path}")
|
||||
print()
|
||||
print(f"Open the HTML file in your browser to explore the interactive graph:")
|
||||
print(f" open {html_path}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nInterrupted by user. Exiting...")
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}", file=sys.stderr)
|
||||
if "--verbose" in sys.argv or "-v" in sys.argv:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,94 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract EC2 instance and security-group nodes with their edges.
|
||||
|
||||
Edges produced:
|
||||
- instance → security-group [network]
|
||||
- instance → subnet [network]
|
||||
- security-group → VPC [network]
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
# EC2 Instances
|
||||
for instance in client.instances:
|
||||
name = instance.id
|
||||
for tag in instance.tags or []:
|
||||
if tag.get("Key") == "Name":
|
||||
name = tag["Value"]
|
||||
break
|
||||
|
||||
props = {
|
||||
"instance_type": getattr(instance, "type", None),
|
||||
"state": getattr(instance, "state", None),
|
||||
"vpc_id": getattr(instance, "vpc_id", None),
|
||||
"subnet_id": getattr(instance, "subnet_id", None),
|
||||
"public_ip": getattr(instance, "public_ip_address", None),
|
||||
"private_ip": getattr(instance, "private_ip_address", None),
|
||||
}
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=instance.arn,
|
||||
type="ec2_instance",
|
||||
name=name,
|
||||
service="ec2",
|
||||
region=instance.region,
|
||||
account_id=client.audited_account,
|
||||
properties={k: v for k, v in props.items() if v is not None},
|
||||
)
|
||||
)
|
||||
|
||||
for sg_id in instance.security_groups or []:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=instance.arn,
|
||||
target_id=sg_id,
|
||||
edge_type="network",
|
||||
label="sg",
|
||||
)
|
||||
)
|
||||
|
||||
if instance.subnet_id:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=instance.arn,
|
||||
target_id=instance.subnet_id,
|
||||
edge_type="network",
|
||||
label="subnet",
|
||||
)
|
||||
)
|
||||
|
||||
# Security Groups
|
||||
for sg in client.security_groups.values():
|
||||
name = (
|
||||
sg.name if hasattr(sg, "name") else sg.id if hasattr(sg, "id") else sg.arn
|
||||
)
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=sg.arn,
|
||||
type="security_group",
|
||||
name=name,
|
||||
service="ec2",
|
||||
region=sg.region,
|
||||
account_id=client.audited_account,
|
||||
properties={"vpc_id": sg.vpc_id},
|
||||
)
|
||||
)
|
||||
|
||||
if sg.vpc_id:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=sg.arn,
|
||||
target_id=sg.vpc_id,
|
||||
edge_type="network",
|
||||
label="in-vpc",
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,60 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract ELBv2 (ALB/NLB) load balancer nodes and their edges.
|
||||
|
||||
Edges produced:
|
||||
- load_balancer → security-group [network]
|
||||
- load_balancer → VPC [network]
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
for lb in client.loadbalancersv2.values():
|
||||
props = {
|
||||
"type": getattr(lb, "type", None),
|
||||
"scheme": getattr(lb, "scheme", None),
|
||||
"dns_name": getattr(lb, "dns", None),
|
||||
"vpc_id": getattr(lb, "vpc_id", None),
|
||||
}
|
||||
|
||||
name = getattr(lb, "name", lb.arn.split("/")[-2] if "/" in lb.arn else lb.arn)
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=lb.arn,
|
||||
type="load_balancer",
|
||||
name=name,
|
||||
service="elbv2",
|
||||
region=lb.region,
|
||||
account_id=client.audited_account,
|
||||
properties={k: v for k, v in props.items() if v is not None},
|
||||
)
|
||||
)
|
||||
|
||||
for sg_id in lb.security_groups or []:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=lb.arn,
|
||||
target_id=sg_id,
|
||||
edge_type="network",
|
||||
label="sg",
|
||||
)
|
||||
)
|
||||
|
||||
vpc_id = getattr(lb, "vpc_id", None)
|
||||
if vpc_id:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=lb.arn,
|
||||
target_id=vpc_id,
|
||||
edge_type="network",
|
||||
label="in-vpc",
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,84 @@
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def _parse_trust_principals(assume_role_policy: Any) -> List[str]:
|
||||
"""
|
||||
Return a flat list of principal strings from an IAM assume-role policy document.
|
||||
The policy may be a dict already or a JSON string.
|
||||
"""
|
||||
if not assume_role_policy:
|
||||
return []
|
||||
|
||||
if isinstance(assume_role_policy, str):
|
||||
try:
|
||||
assume_role_policy = json.loads(assume_role_policy)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return []
|
||||
|
||||
principals = []
|
||||
for statement in assume_role_policy.get("Statement", []):
|
||||
principal = statement.get("Principal", {})
|
||||
if isinstance(principal, str):
|
||||
principals.append(principal)
|
||||
elif isinstance(principal, dict):
|
||||
for v in principal.values():
|
||||
if isinstance(v, list):
|
||||
principals.extend(v)
|
||||
else:
|
||||
principals.append(v)
|
||||
elif isinstance(principal, list):
|
||||
principals.extend(principal)
|
||||
|
||||
return principals
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract IAM role nodes and their trust-relationship edges.
|
||||
|
||||
Edges produced:
|
||||
- trusted-principal → role [iam] (who can assume this role)
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
for role in client.roles:
|
||||
props: Dict[str, Any] = {
|
||||
"path": getattr(role, "path", None),
|
||||
"create_date": str(getattr(role, "create_date", "") or ""),
|
||||
}
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=role.arn,
|
||||
type="iam_role",
|
||||
name=role.name,
|
||||
service="iam",
|
||||
region="global",
|
||||
account_id=client.audited_account,
|
||||
properties={k: v for k, v in props.items() if v},
|
||||
)
|
||||
)
|
||||
|
||||
# Trust-relationship edges: principal → role (principal CAN assume role)
|
||||
try:
|
||||
for principal in _parse_trust_principals(role.assume_role_policy):
|
||||
if principal and principal != "*":
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=principal,
|
||||
target_id=role.arn,
|
||||
edge_type="iam",
|
||||
label="can-assume",
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"inventory iam_extractor: could not parse trust policy for {role.arn}: {e}"
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,118 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract Lambda function nodes and their edges from an awslambda_client.
|
||||
|
||||
Edges produced:
|
||||
- lambda → VPC [network]
|
||||
- lambda → subnet [network]
|
||||
- lambda → sg [network]
|
||||
- lambda → event-source[triggers] (from EventSourceMapping)
|
||||
- lambda → layer ARN [depends_on]
|
||||
- lambda → DLQ target [data_flow]
|
||||
- lambda → KMS key [encrypts]
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
for fn in client.functions.values():
|
||||
props = {
|
||||
"runtime": fn.runtime,
|
||||
"vpc_id": fn.vpc_id,
|
||||
}
|
||||
if fn.environment:
|
||||
props["has_env_vars"] = True
|
||||
if fn.kms_key_arn:
|
||||
props["kms_key_arn"] = fn.kms_key_arn
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=fn.arn,
|
||||
type="lambda_function",
|
||||
name=fn.name,
|
||||
service="lambda",
|
||||
region=fn.region,
|
||||
account_id=client.audited_account,
|
||||
properties=props,
|
||||
)
|
||||
)
|
||||
|
||||
# Network edges → VPC, subnets, security groups
|
||||
if fn.vpc_id:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=fn.arn,
|
||||
target_id=fn.vpc_id,
|
||||
edge_type="network",
|
||||
label="in-vpc",
|
||||
)
|
||||
)
|
||||
for sg_id in fn.security_groups or []:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=fn.arn,
|
||||
target_id=sg_id,
|
||||
edge_type="network",
|
||||
label="sg",
|
||||
)
|
||||
)
|
||||
for subnet_id in fn.subnet_ids or set():
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=fn.arn,
|
||||
target_id=subnet_id,
|
||||
edge_type="network",
|
||||
label="subnet",
|
||||
)
|
||||
)
|
||||
|
||||
# Trigger edges from event source mappings
|
||||
for esm in getattr(fn, "event_source_mappings", []):
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=esm.event_source_arn,
|
||||
target_id=fn.arn,
|
||||
edge_type="triggers",
|
||||
label=f"esm:{esm.state}",
|
||||
)
|
||||
)
|
||||
|
||||
# Layer dependency edges
|
||||
for layer in getattr(fn, "layers", []):
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=fn.arn,
|
||||
target_id=layer.arn,
|
||||
edge_type="depends_on",
|
||||
label="layer",
|
||||
)
|
||||
)
|
||||
|
||||
# Dead-letter queue data-flow edge
|
||||
dlq = getattr(fn, "dead_letter_config", None)
|
||||
if dlq and dlq.target_arn:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=fn.arn,
|
||||
target_id=dlq.target_arn,
|
||||
edge_type="data_flow",
|
||||
label="dlq",
|
||||
)
|
||||
)
|
||||
|
||||
# KMS encryption edge
|
||||
if fn.kms_key_arn:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=fn.kms_key_arn,
|
||||
target_id=fn.arn,
|
||||
edge_type="encrypts",
|
||||
label="kms",
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,86 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract RDS DB instance nodes and their edges.
|
||||
|
||||
Edges produced:
|
||||
- db_instance → security-group [network]
|
||||
- db_instance → VPC [network]
|
||||
- db_instance → cluster [depends_on]
|
||||
- db_instance → KMS key [encrypts]
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
for db in client.db_instances.values():
|
||||
props = {
|
||||
"engine": getattr(db, "engine", None),
|
||||
"engine_version": getattr(db, "engine_version", None),
|
||||
"instance_class": getattr(db, "db_instance_class", None),
|
||||
"vpc_id": getattr(db, "vpc_id", None),
|
||||
"multi_az": getattr(db, "multi_az", None),
|
||||
"publicly_accessible": getattr(db, "publicly_accessible", None),
|
||||
"storage_encrypted": getattr(db, "storage_encrypted", None),
|
||||
}
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=db.arn,
|
||||
type="rds_instance",
|
||||
name=db.id,
|
||||
service="rds",
|
||||
region=db.region,
|
||||
account_id=client.audited_account,
|
||||
properties={k: v for k, v in props.items() if v is not None},
|
||||
)
|
||||
)
|
||||
|
||||
for sg in getattr(db, "security_groups", []):
|
||||
sg_id = sg if isinstance(sg, str) else getattr(sg, "id", str(sg))
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=db.arn,
|
||||
target_id=sg_id,
|
||||
edge_type="network",
|
||||
label="sg",
|
||||
)
|
||||
)
|
||||
|
||||
vpc_id = getattr(db, "vpc_id", None)
|
||||
if vpc_id:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=db.arn,
|
||||
target_id=vpc_id,
|
||||
edge_type="network",
|
||||
label="in-vpc",
|
||||
)
|
||||
)
|
||||
|
||||
cluster_arn = getattr(db, "cluster_arn", None)
|
||||
if cluster_arn:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=db.arn,
|
||||
target_id=cluster_arn,
|
||||
edge_type="depends_on",
|
||||
label="cluster-member",
|
||||
)
|
||||
)
|
||||
|
||||
kms_key_id = getattr(db, "kms_key_id", None)
|
||||
if kms_key_id:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=kms_key_id,
|
||||
target_id=db.arn,
|
||||
edge_type="encrypts",
|
||||
label="kms",
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,92 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract S3 bucket nodes and their edges.
|
||||
|
||||
Edges produced:
|
||||
- bucket → replication-target bucket [replicates_to]
|
||||
- bucket → KMS key [encrypts]
|
||||
- bucket → logging bucket [logs_to]
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
for bucket in client.buckets.values():
|
||||
encryption = getattr(bucket, "encryption", None)
|
||||
versioning = getattr(bucket, "versioning_enabled", None)
|
||||
logging = getattr(bucket, "logging", None)
|
||||
public = getattr(bucket, "public_access_block", None)
|
||||
|
||||
props = {}
|
||||
if versioning is not None:
|
||||
props["versioning"] = versioning
|
||||
if encryption:
|
||||
enc_type = getattr(encryption, "type", str(encryption))
|
||||
props["encryption"] = enc_type
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=bucket.arn,
|
||||
type="s3_bucket",
|
||||
name=bucket.name,
|
||||
service="s3",
|
||||
region=bucket.region,
|
||||
account_id=client.audited_account,
|
||||
properties=props,
|
||||
)
|
||||
)
|
||||
|
||||
# Replication edges
|
||||
for rule in getattr(bucket, "replication_rules", None) or []:
|
||||
dest_bucket = getattr(rule, "destination_bucket", None)
|
||||
if dest_bucket:
|
||||
dest_arn = (
|
||||
dest_bucket
|
||||
if dest_bucket.startswith("arn:")
|
||||
else f"arn:aws:s3:::{dest_bucket}"
|
||||
)
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=bucket.arn,
|
||||
target_id=dest_arn,
|
||||
edge_type="replicates_to",
|
||||
label="s3-replication",
|
||||
)
|
||||
)
|
||||
|
||||
# Logging edges
|
||||
if logging:
|
||||
target_bucket = getattr(logging, "target_bucket", None)
|
||||
if target_bucket:
|
||||
target_arn = (
|
||||
target_bucket
|
||||
if target_bucket.startswith("arn:")
|
||||
else f"arn:aws:s3:::{target_bucket}"
|
||||
)
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=bucket.arn,
|
||||
target_id=target_arn,
|
||||
edge_type="logs_to",
|
||||
label="access-logs",
|
||||
)
|
||||
)
|
||||
|
||||
# KMS encryption edges
|
||||
if encryption:
|
||||
kms_arn = getattr(encryption, "kms_master_key_id", None)
|
||||
if kms_arn:
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=kms_arn,
|
||||
target_id=bucket.arn,
|
||||
edge_type="encrypts",
|
||||
label="kms",
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,92 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from lib.models import ResourceEdge, ResourceNode
|
||||
|
||||
|
||||
def extract(client) -> Tuple[List[ResourceNode], List[ResourceEdge]]:
|
||||
"""
|
||||
Extract VPC and subnet nodes with their edges.
|
||||
|
||||
Edges produced:
|
||||
- subnet → VPC [depends_on]
|
||||
- peering connection between VPCs [network]
|
||||
"""
|
||||
nodes: List[ResourceNode] = []
|
||||
edges: List[ResourceEdge] = []
|
||||
|
||||
# VPCs
|
||||
for vpc in client.vpcs.values():
|
||||
name = vpc.id if hasattr(vpc, "id") else vpc.arn
|
||||
for tag in vpc.tags or []:
|
||||
if isinstance(tag, dict) and tag.get("Key") == "Name":
|
||||
name = tag["Value"]
|
||||
break
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=vpc.arn,
|
||||
type="vpc",
|
||||
name=name,
|
||||
service="vpc",
|
||||
region=vpc.region,
|
||||
account_id=client.audited_account,
|
||||
properties={
|
||||
"cidr_block": getattr(vpc, "cidr_block", None),
|
||||
"is_default": getattr(vpc, "is_default", None),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# VPC Subnets
|
||||
for subnet in client.vpc_subnets.values():
|
||||
name = subnet.id if hasattr(subnet, "id") else subnet.arn
|
||||
for tag in getattr(subnet, "tags", None) or []:
|
||||
if isinstance(tag, dict) and tag.get("Key") == "Name":
|
||||
name = tag["Value"]
|
||||
break
|
||||
|
||||
nodes.append(
|
||||
ResourceNode(
|
||||
id=subnet.arn,
|
||||
type="subnet",
|
||||
name=name,
|
||||
service="vpc",
|
||||
region=subnet.region,
|
||||
account_id=client.audited_account,
|
||||
properties={
|
||||
"vpc_id": getattr(subnet, "vpc_id", None),
|
||||
"cidr_block": getattr(subnet, "cidr_block", None),
|
||||
"availability_zone": getattr(subnet, "availability_zone", None),
|
||||
"public": getattr(subnet, "public", None),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
vpc_id = getattr(subnet, "vpc_id", None)
|
||||
if vpc_id:
|
||||
# Find the VPC ARN for this vpc_id
|
||||
vpc_arn = next(
|
||||
(v.arn for v in client.vpcs.values() if v.id == vpc_id),
|
||||
vpc_id,
|
||||
)
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=subnet.arn,
|
||||
target_id=vpc_arn,
|
||||
edge_type="depends_on",
|
||||
label="subnet-of",
|
||||
)
|
||||
)
|
||||
|
||||
# VPC Peering Connections
|
||||
for peering in getattr(client, "vpc_peering_connections", {}).values():
|
||||
edges.append(
|
||||
ResourceEdge(
|
||||
source_id=peering.arn,
|
||||
target_id=getattr(peering, "accepter_vpc_id", peering.arn),
|
||||
edge_type="network",
|
||||
label="vpc-peer",
|
||||
)
|
||||
)
|
||||
|
||||
return nodes, edges
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
graph_builder.py
|
||||
----------------
|
||||
Builds a ConnectivityGraph by reading already-loaded AWS service clients from
|
||||
sys.modules. Only services that were actually scanned (i.e. whose client
|
||||
module is already imported) contribute nodes and edges. Unknown / unloaded
|
||||
services are silently skipped, so the output degrades gracefully when only a
|
||||
subset of checks has been run.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from typing import Tuple
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from lib.models import ConnectivityGraph
|
||||
|
||||
# Registry: (sys.modules key, attribute name inside that module, extractor module path)
|
||||
_SERVICE_REGISTRY: Tuple[Tuple[str, str, str], ...] = (
|
||||
(
|
||||
"prowler.providers.aws.services.awslambda.awslambda_client",
|
||||
"awslambda_client",
|
||||
"lib.extractors.lambda_extractor",
|
||||
),
|
||||
(
|
||||
"prowler.providers.aws.services.ec2.ec2_client",
|
||||
"ec2_client",
|
||||
"lib.extractors.ec2_extractor",
|
||||
),
|
||||
(
|
||||
"prowler.providers.aws.services.vpc.vpc_client",
|
||||
"vpc_client",
|
||||
"lib.extractors.vpc_extractor",
|
||||
),
|
||||
(
|
||||
"prowler.providers.aws.services.rds.rds_client",
|
||||
"rds_client",
|
||||
"lib.extractors.rds_extractor",
|
||||
),
|
||||
(
|
||||
"prowler.providers.aws.services.elbv2.elbv2_client",
|
||||
"elbv2_client",
|
||||
"lib.extractors.elbv2_extractor",
|
||||
),
|
||||
(
|
||||
"prowler.providers.aws.services.s3.s3_client",
|
||||
"s3_client",
|
||||
"lib.extractors.s3_extractor",
|
||||
),
|
||||
(
|
||||
"prowler.providers.aws.services.iam.iam_client",
|
||||
"iam_client",
|
||||
"lib.extractors.iam_extractor",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_graph() -> ConnectivityGraph:
|
||||
"""
|
||||
Iterate over every registered service, check whether its client module is
|
||||
already loaded, and call the corresponding extractor.
|
||||
|
||||
Returns a ConnectivityGraph with all discovered nodes and edges.
|
||||
Duplicate node IDs are silently deduplicated (first occurrence wins).
|
||||
"""
|
||||
graph = ConnectivityGraph()
|
||||
seen_node_ids: set = set()
|
||||
|
||||
for client_module_key, client_attr, extractor_module_key in _SERVICE_REGISTRY:
|
||||
client_module = sys.modules.get(client_module_key)
|
||||
if client_module is None:
|
||||
continue
|
||||
|
||||
service_client = getattr(client_module, client_attr, None)
|
||||
if service_client is None:
|
||||
continue
|
||||
|
||||
extractor_module = sys.modules.get(extractor_module_key)
|
||||
if extractor_module is None:
|
||||
try:
|
||||
import importlib
|
||||
|
||||
extractor_module = importlib.import_module(extractor_module_key)
|
||||
except ImportError as e:
|
||||
logger.debug(
|
||||
f"inventory graph_builder: cannot import extractor {extractor_module_key}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
nodes, edges = extractor_module.extract(service_client)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"inventory graph_builder: extractor {extractor_module_key} failed: "
|
||||
f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}]: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
for node in nodes:
|
||||
if node.id not in seen_node_ids:
|
||||
graph.add_node(node)
|
||||
seen_node_ids.add(node.id)
|
||||
|
||||
for edge in edges:
|
||||
graph.add_edge(edge)
|
||||
|
||||
return graph
|
||||
@@ -0,0 +1,502 @@
|
||||
"""
|
||||
inventory_output.py
|
||||
-------------------
|
||||
Writes the ConnectivityGraph produced by graph_builder to two files:
|
||||
|
||||
<output_path>.inventory.json – machine-readable graph (nodes + edges)
|
||||
<output_path>.inventory.html – interactive D3.js force-directed graph
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from prowler.lib.logger import logger
|
||||
from lib.models import ConnectivityGraph
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(graph: ConnectivityGraph, file_path: str) -> None:
|
||||
"""Serialise the graph to a JSON file."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
||||
data = {
|
||||
"generated_at": datetime.utcnow().isoformat() + "Z",
|
||||
"nodes": [asdict(n) for n in graph.nodes],
|
||||
"edges": [asdict(e) for e in graph.edges],
|
||||
"stats": {
|
||||
"node_count": len(graph.nodes),
|
||||
"edge_count": len(graph.edges),
|
||||
},
|
||||
}
|
||||
with open(file_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, indent=2, default=str)
|
||||
logger.info(f"Inventory graph JSON written to {file_path}")
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"inventory_output.write_json: {e.__class__.__name__}[{e.__traceback__.tb_lineno}]: {e}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTML output (self-contained, D3.js CDN)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Colour palette per node type
|
||||
_NODE_COLOURS = {
|
||||
"lambda_function": "#f59e0b",
|
||||
"ec2_instance": "#3b82f6",
|
||||
"security_group": "#6366f1",
|
||||
"vpc": "#10b981",
|
||||
"subnet": "#34d399",
|
||||
"rds_instance": "#ef4444",
|
||||
"load_balancer": "#8b5cf6",
|
||||
"s3_bucket": "#06b6d4",
|
||||
"iam_role": "#f97316",
|
||||
"default": "#94a3b8",
|
||||
}
|
||||
|
||||
# Edge stroke colours per edge type
|
||||
_EDGE_COLOURS = {
|
||||
"network": "#64748b",
|
||||
"iam": "#f97316",
|
||||
"triggers": "#a855f7",
|
||||
"data_flow": "#0ea5e9",
|
||||
"depends_on": "#94a3b8",
|
||||
"routes_to": "#22c55e",
|
||||
"replicates_to": "#ec4899",
|
||||
"encrypts": "#eab308",
|
||||
"logs_to": "#78716c",
|
||||
}
|
||||
|
||||
_HTML_TEMPLATE = """\
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>Prowler – AWS Connectivity Graph</title>
|
||||
<script src="https://d3js.org/d3.v7.min.js"></script>
|
||||
<style>
|
||||
*, *::before, *::after {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
margin: 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
background: #0f172a;
|
||||
color: #e2e8f0;
|
||||
}}
|
||||
#header {{
|
||||
padding: 12px 20px;
|
||||
background: #1e293b;
|
||||
border-bottom: 1px solid #334155;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
}}
|
||||
#header h1 {{ margin: 0; font-size: 18px; font-weight: 700; }}
|
||||
#header .stats {{ font-size: 13px; color: #94a3b8; }}
|
||||
#controls {{
|
||||
padding: 8px 20px;
|
||||
background: #1e293b;
|
||||
border-bottom: 1px solid #334155;
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}}
|
||||
#controls label {{ font-size: 12px; color: #94a3b8; }}
|
||||
#controls select, #controls input[type=range] {{
|
||||
background: #0f172a;
|
||||
color: #e2e8f0;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 4px;
|
||||
padding: 3px 6px;
|
||||
font-size: 12px;
|
||||
}}
|
||||
#graph-container {{ width: 100%; height: calc(100vh - 100px); position: relative; }}
|
||||
svg {{ width: 100%; height: 100%; }}
|
||||
.node circle {{
|
||||
stroke: #1e293b;
|
||||
stroke-width: 1.5px;
|
||||
cursor: pointer;
|
||||
transition: r 0.15s;
|
||||
}}
|
||||
.node circle:hover {{ stroke-width: 3px; }}
|
||||
.node text {{
|
||||
font-size: 10px;
|
||||
fill: #e2e8f0;
|
||||
pointer-events: none;
|
||||
text-shadow: 0 0 4px #0f172a;
|
||||
}}
|
||||
.link {{
|
||||
stroke-opacity: 0.6;
|
||||
stroke-width: 1.5px;
|
||||
}}
|
||||
.link-label {{
|
||||
font-size: 8px;
|
||||
fill: #94a3b8;
|
||||
pointer-events: none;
|
||||
}}
|
||||
#tooltip {{
|
||||
position: fixed;
|
||||
background: #1e293b;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 6px;
|
||||
padding: 10px 14px;
|
||||
font-size: 12px;
|
||||
pointer-events: none;
|
||||
max-width: 320px;
|
||||
word-break: break-all;
|
||||
z-index: 9999;
|
||||
display: none;
|
||||
}}
|
||||
#tooltip strong {{ color: #f8fafc; }}
|
||||
#tooltip .prop {{ color: #94a3b8; margin-top: 4px; }}
|
||||
#legend {{
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
right: 10px;
|
||||
background: rgba(30,41,59,0.9);
|
||||
border: 1px solid #334155;
|
||||
border-radius: 6px;
|
||||
padding: 10px 14px;
|
||||
font-size: 11px;
|
||||
}}
|
||||
#legend h3 {{ margin: 0 0 6px; font-size: 12px; }}
|
||||
.legend-row {{ display: flex; align-items: center; gap: 6px; margin: 3px 0; }}
|
||||
.legend-dot {{ width: 12px; height: 12px; border-radius: 50%; flex-shrink: 0; }}
|
||||
.legend-line {{ width: 20px; height: 2px; flex-shrink: 0; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="header">
|
||||
<h1>🔗 AWS Connectivity Graph</h1>
|
||||
<span class="stats" id="stat-label">Generated: {generated_at}</span>
|
||||
</div>
|
||||
<div id="controls">
|
||||
<label>Filter service:
|
||||
<select id="filter-service">
|
||||
<option value="">All services</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>Link distance:
|
||||
<input type="range" id="link-distance" min="40" max="300" value="120"/>
|
||||
</label>
|
||||
<label>Charge strength:
|
||||
<input type="range" id="charge-strength" min="-800" max="-20" value="-250"/>
|
||||
</label>
|
||||
<span class="stats" id="visible-count"></span>
|
||||
</div>
|
||||
<div id="graph-container">
|
||||
<svg id="graph-svg"></svg>
|
||||
<div id="tooltip"></div>
|
||||
<div id="legend">
|
||||
<h3>Node types</h3>
|
||||
{legend_nodes_html}
|
||||
<h3 style="margin-top:8px">Edge types</h3>
|
||||
{legend_edges_html}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const RAW_NODES = {nodes_json};
|
||||
const RAW_EDGES = {edges_json};
|
||||
const NODE_COLOURS = {node_colours_json};
|
||||
const EDGE_COLOURS = {edge_colours_json};
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────
|
||||
function nodeColour(d) {{
|
||||
return NODE_COLOURS[d.type] || NODE_COLOURS["default"];
|
||||
}}
|
||||
function edgeColour(d) {{
|
||||
return EDGE_COLOURS[d.edge_type] || "#94a3b8";
|
||||
}}
|
||||
function nodeRadius(d) {{
|
||||
const base = {{
|
||||
lambda_function: 9, ec2_instance: 10, vpc: 14, subnet: 8,
|
||||
security_group: 7, rds_instance: 11, load_balancer: 12,
|
||||
s3_bucket: 9, iam_role: 9
|
||||
}};
|
||||
return base[d.type] || 8;
|
||||
}}
|
||||
|
||||
// ── filter controls ───────────────────────────────────────────────────────
|
||||
const services = [...new Set(RAW_NODES.map(n => n.service))].sort();
|
||||
const sel = document.getElementById("filter-service");
|
||||
services.forEach(s => {{
|
||||
const o = document.createElement("option");
|
||||
o.value = s; o.textContent = s;
|
||||
sel.appendChild(o);
|
||||
}});
|
||||
|
||||
// ── D3 setup ──────────────────────────────────────────────────────────────
|
||||
const svg = d3.select("#graph-svg");
|
||||
const container = svg.append("g");
|
||||
|
||||
// zoom
|
||||
svg.call(
|
||||
d3.zoom().scaleExtent([0.05, 8])
|
||||
.on("zoom", e => container.attr("transform", e.transform))
|
||||
);
|
||||
|
||||
// arrowhead marker
|
||||
const defs = svg.append("defs");
|
||||
defs.append("marker")
|
||||
.attr("id", "arrow")
|
||||
.attr("viewBox", "0 -5 10 10")
|
||||
.attr("refX", 20).attr("refY", 0)
|
||||
.attr("markerWidth", 6).attr("markerHeight", 6)
|
||||
.attr("orient", "auto")
|
||||
.append("path")
|
||||
.attr("d", "M0,-5L10,0L0,5")
|
||||
.attr("fill", "#94a3b8");
|
||||
|
||||
// tooltip
|
||||
const tooltip = document.getElementById("tooltip");
|
||||
|
||||
// ── simulation ────────────────────────────────────────────────────────────
|
||||
let simulation, linkSel, nodeSel, labelSel;
|
||||
|
||||
function buildGraph(nodeFilter) {{
|
||||
// Determine which nodes to show
|
||||
const visibleNodes = nodeFilter
|
||||
? RAW_NODES.filter(n => n.service === nodeFilter)
|
||||
: RAW_NODES;
|
||||
const visibleIds = new Set(visibleNodes.map(n => n.id));
|
||||
|
||||
// Only show edges where BOTH endpoints are visible
|
||||
const visibleEdges = RAW_EDGES.filter(
|
||||
e => visibleIds.has(e.source_id) && visibleIds.has(e.target_id)
|
||||
);
|
||||
|
||||
document.getElementById("visible-count").textContent =
|
||||
`Showing ${{visibleNodes.length}} nodes · ${{visibleEdges.length}} edges`;
|
||||
|
||||
container.selectAll("*").remove();
|
||||
|
||||
if (simulation) simulation.stop();
|
||||
|
||||
const nodes = visibleNodes.map(n => ({{ ...n }}));
|
||||
const nodeIndex = Object.fromEntries(nodes.map(n => [n.id, n]));
|
||||
|
||||
const links = visibleEdges.map(e => ({{
|
||||
...e,
|
||||
source: nodeIndex[e.source_id] || e.source_id,
|
||||
target: nodeIndex[e.target_id] || e.target_id,
|
||||
}}));
|
||||
|
||||
const dist = +document.getElementById("link-distance").value;
|
||||
const charge = +document.getElementById("charge-strength").value;
|
||||
|
||||
simulation = d3.forceSimulation(nodes)
|
||||
.force("link", d3.forceLink(links).id(d => d.id).distance(dist))
|
||||
.force("charge", d3.forceManyBody().strength(charge))
|
||||
.force("center", d3.forceCenter(
|
||||
document.getElementById("graph-container").clientWidth / 2,
|
||||
document.getElementById("graph-container").clientHeight / 2
|
||||
))
|
||||
.force("collision", d3.forceCollide().radius(d => nodeRadius(d) + 6));
|
||||
|
||||
// Edges
|
||||
linkSel = container.append("g").attr("class", "links")
|
||||
.selectAll("line")
|
||||
.data(links)
|
||||
.join("line")
|
||||
.attr("class", "link")
|
||||
.attr("stroke", edgeColour)
|
||||
.attr("marker-end", "url(#arrow)");
|
||||
|
||||
// Edge labels
|
||||
labelSel = container.append("g").attr("class", "link-labels")
|
||||
.selectAll("text")
|
||||
.data(links)
|
||||
.join("text")
|
||||
.attr("class", "link-label")
|
||||
.text(d => d.label || "");
|
||||
|
||||
// Nodes
|
||||
nodeSel = container.append("g").attr("class", "nodes")
|
||||
.selectAll("g")
|
||||
.data(nodes)
|
||||
.join("g")
|
||||
.attr("class", "node")
|
||||
.call(
|
||||
d3.drag()
|
||||
.on("start", (event, d) => {{
|
||||
if (!event.active) simulation.alphaTarget(0.3).restart();
|
||||
d.fx = d.x; d.fy = d.y;
|
||||
}})
|
||||
.on("drag", (event, d) => {{ d.fx = event.x; d.fy = event.y; }})
|
||||
.on("end", (event, d) => {{
|
||||
if (!event.active) simulation.alphaTarget(0);
|
||||
d.fx = null; d.fy = null;
|
||||
}})
|
||||
)
|
||||
.on("mouseover", (event, d) => {{
|
||||
const props = Object.entries(d.properties || {{}})
|
||||
.map(([k, v]) => `<div class="prop"><b>${{k}}</b>: ${{v}}</div>`)
|
||||
.join("");
|
||||
tooltip.innerHTML = `
|
||||
<strong>${{d.name}}</strong>
|
||||
<div class="prop"><b>type</b>: ${{d.type}}</div>
|
||||
<div class="prop"><b>service</b>: ${{d.service}}</div>
|
||||
<div class="prop"><b>region</b>: ${{d.region}}</div>
|
||||
<div class="prop"><b>account</b>: ${{d.account_id}}</div>
|
||||
<div class="prop" style="word-break:break-all"><b>arn</b>: ${{d.id}}</div>
|
||||
${{props}}
|
||||
`;
|
||||
tooltip.style.display = "block";
|
||||
tooltip.style.left = (event.clientX + 12) + "px";
|
||||
tooltip.style.top = (event.clientY - 10) + "px";
|
||||
}})
|
||||
.on("mousemove", event => {{
|
||||
tooltip.style.left = (event.clientX + 12) + "px";
|
||||
tooltip.style.top = (event.clientY - 10) + "px";
|
||||
}})
|
||||
.on("mouseout", () => {{ tooltip.style.display = "none"; }});
|
||||
|
||||
nodeSel.append("circle")
|
||||
.attr("r", nodeRadius)
|
||||
.attr("fill", nodeColour);
|
||||
|
||||
nodeSel.append("text")
|
||||
.attr("dx", d => nodeRadius(d) + 3)
|
||||
.attr("dy", "0.35em")
|
||||
.text(d => d.name.length > 24 ? d.name.slice(0, 22) + "…" : d.name);
|
||||
|
||||
simulation.on("tick", () => {{
|
||||
linkSel
|
||||
.attr("x1", d => d.source.x)
|
||||
.attr("y1", d => d.source.y)
|
||||
.attr("x2", d => d.target.x)
|
||||
.attr("y2", d => d.target.y);
|
||||
|
||||
labelSel
|
||||
.attr("x", d => (d.source.x + d.target.x) / 2)
|
||||
.attr("y", d => (d.source.y + d.target.y) / 2);
|
||||
|
||||
nodeSel.attr("transform", d => `translate(${{d.x}},${{d.y}})`);
|
||||
}});
|
||||
}}
|
||||
|
||||
// Initial render
|
||||
buildGraph(null);
|
||||
|
||||
// Filter change
|
||||
sel.addEventListener("change", () => buildGraph(sel.value || null));
|
||||
|
||||
// Simulation control sliders — restart on change
|
||||
document.getElementById("link-distance").addEventListener("input", () => buildGraph(sel.value || null));
|
||||
document.getElementById("charge-strength").addEventListener("input", () => buildGraph(sel.value || null));
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def _build_legend_html(colours: dict, shape: str) -> str:
|
||||
rows = []
|
||||
for key, colour in sorted(colours.items()):
|
||||
if shape == "dot":
|
||||
rows.append(
|
||||
f'<div class="legend-row">'
|
||||
f'<div class="legend-dot" style="background:{colour}"></div>'
|
||||
f"<span>{key}</span></div>"
|
||||
)
|
||||
else:
|
||||
rows.append(
|
||||
f'<div class="legend-row">'
|
||||
f'<div class="legend-line" style="background:{colour}"></div>'
|
||||
f"<span>{key}</span></div>"
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def write_html(graph: ConnectivityGraph, file_path: str) -> None:
|
||||
"""Render the graph as a self-contained interactive HTML page."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
||||
|
||||
nodes_json = json.dumps(
|
||||
[
|
||||
{
|
||||
"id": n.id,
|
||||
"type": n.type,
|
||||
"name": n.name,
|
||||
"service": n.service,
|
||||
"region": n.region,
|
||||
"account_id": n.account_id,
|
||||
"properties": n.properties,
|
||||
}
|
||||
for n in graph.nodes
|
||||
],
|
||||
indent=None,
|
||||
default=str,
|
||||
)
|
||||
edges_json = json.dumps(
|
||||
[
|
||||
{
|
||||
"source_id": e.source_id,
|
||||
"target_id": e.target_id,
|
||||
"edge_type": e.edge_type,
|
||||
"label": e.label or "",
|
||||
}
|
||||
for e in graph.edges
|
||||
],
|
||||
indent=None,
|
||||
default=str,
|
||||
)
|
||||
|
||||
html = _HTML_TEMPLATE.format(
|
||||
generated_at=datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC"),
|
||||
nodes_json=nodes_json,
|
||||
edges_json=edges_json,
|
||||
node_colours_json=json.dumps(_NODE_COLOURS),
|
||||
edge_colours_json=json.dumps(_EDGE_COLOURS),
|
||||
legend_nodes_html=_build_legend_html(_NODE_COLOURS, "dot"),
|
||||
legend_edges_html=_build_legend_html(_EDGE_COLOURS, "line"),
|
||||
)
|
||||
|
||||
with open(file_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(html)
|
||||
|
||||
logger.info(f"Inventory graph HTML written to {file_path}")
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"inventory_output.write_html: {e.__class__.__name__}[{e.__traceback__.tb_lineno}]: {e}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Convenience entry-point called from __main__.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_inventory_outputs(output_path: str) -> None:
|
||||
"""
|
||||
Build the connectivity graph from currently-loaded service clients and write
|
||||
both JSON and HTML outputs.
|
||||
|
||||
Args:
|
||||
output_path: base file path WITHOUT extension, e.g.
|
||||
"output/prowler-output-20240101120000".
|
||||
The function appends .inventory.json and .inventory.html.
|
||||
"""
|
||||
from lib.graph_builder import build_graph
|
||||
|
||||
graph = build_graph()
|
||||
|
||||
if not graph.nodes:
|
||||
logger.warning(
|
||||
"Inventory graph: no nodes discovered. "
|
||||
"Make sure at least one AWS service was scanned before generating the inventory."
|
||||
)
|
||||
|
||||
write_json(graph, f"{output_path}.inventory.json")
|
||||
write_html(graph, f"{output_path}.inventory.html")
|
||||
@@ -0,0 +1,71 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResourceNode:
|
||||
"""
|
||||
Represents a single AWS resource as a node in the connectivity graph.
|
||||
|
||||
id : globally unique identifier — always the resource ARN
|
||||
type : coarse resource type used for grouping/colour, e.g. "lambda_function"
|
||||
name : human-readable label shown on the graph
|
||||
service : AWS service name, e.g. "lambda", "ec2", "rds"
|
||||
region : AWS region the resource lives in
|
||||
account_id: AWS account ID
|
||||
properties: additional resource-specific metadata (runtime, vpc_id, etc.)
|
||||
"""
|
||||
|
||||
id: str
|
||||
type: str
|
||||
name: str
|
||||
service: str
|
||||
region: str
|
||||
account_id: str
|
||||
properties: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResourceEdge:
|
||||
"""
|
||||
Represents a directional relationship between two resource nodes.
|
||||
|
||||
source_id : ARN of the source node
|
||||
target_id : ARN of the target node
|
||||
edge_type : semantic type of the relationship, e.g.:
|
||||
"network" – resources share a network path (VPC/subnet/SG)
|
||||
"iam" – IAM trust or permission relationship
|
||||
"triggers" – one resource can invoke another (event source → Lambda)
|
||||
"data_flow" – data is written/read (Lambda → SQS dead-letter queue)
|
||||
"depends_on" – soft dependency (Lambda layer, subnet belongs to VPC)
|
||||
"routes_to" – traffic routing (LB → target)
|
||||
"encrypts" – KMS key encrypts the resource
|
||||
label : optional short label rendered on the edge in the HTML graph
|
||||
"""
|
||||
|
||||
source_id: str
|
||||
target_id: str
|
||||
edge_type: str
|
||||
label: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConnectivityGraph:
|
||||
"""
|
||||
Container for the full inventory connectivity graph.
|
||||
|
||||
nodes: all discovered resource nodes
|
||||
edges: all discovered edges between nodes
|
||||
"""
|
||||
|
||||
nodes: List[ResourceNode] = field(default_factory=list)
|
||||
edges: List[ResourceEdge] = field(default_factory=list)
|
||||
|
||||
def add_node(self, node: ResourceNode) -> None:
|
||||
self.nodes.append(node)
|
||||
|
||||
def add_edge(self, edge: ResourceEdge) -> None:
|
||||
self.edges.append(edge)
|
||||
|
||||
def node_ids(self) -> set:
|
||||
return {n.id for n in self.nodes}
|
||||
@@ -73,7 +73,7 @@ secrets:
|
||||
DJANGO_SECRETS_ENCRYPTION_KEY:
|
||||
DJANGO_BROKER_VISIBILITY_TIMEOUT: 86400
|
||||
|
||||
releaseConfigRoot: /home/prowler/.cache/pypoetry/virtualenvs/prowler-api-NnJNioq7-py3.12/lib/python3.12/site-packages/
|
||||
releaseConfigRoot: /home/prowler/.venv/lib/python3.12/site-packages/
|
||||
releaseConfigPath: prowler/config/config.yaml
|
||||
|
||||
mainConfig:
|
||||
|
||||
+7
-1
@@ -48,10 +48,16 @@ services:
|
||||
- path: .env
|
||||
required: false
|
||||
ports:
|
||||
- ${UI_PORT:-3000}:${UI_PORT:-3000}
|
||||
- ${UI_PORT:-3000}:3000
|
||||
depends_on:
|
||||
mcp-server:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
postgres:
|
||||
image: postgres:16.3-alpine3.20
|
||||
|
||||
+1
-1
@@ -467,7 +467,7 @@ Effective headers and section titles enhance document readability and structure,
|
||||
|
||||
* **Example:**
|
||||
* How to Clone and Install Prowler from GitHub (header: Title case)
|
||||
* How to install poetry dependencies (subheading: Sentence case)
|
||||
* How to install uv dependencies (subheading: Sentence case)
|
||||
5. **Using Keywords in Headers**
|
||||
Headers should include relevant keywords to improve document searchability:
|
||||
* **Good:** Scanning AWS Accounts in Parallel
|
||||
|
||||
+2
-2
@@ -10,10 +10,10 @@ This repository contains the Prowler Open Source documentation powered by [Mintl
|
||||
|
||||
## Local Development
|
||||
|
||||
Install the [Mintlify CLI](https://www.npmjs.com/package/mint) to preview documentation changes locally:
|
||||
Install a reviewed version of the [Mintlify CLI](https://www.npmjs.com/package/mint) to preview documentation changes locally:
|
||||
|
||||
```bash
|
||||
npm i -g mint
|
||||
npm install --global mint@4.2.560
|
||||
```
|
||||
|
||||
Run the following command at the root of your documentation (where `mint.json` is located):
|
||||
|
||||
@@ -73,6 +73,58 @@ The best reference to understand how to implement a new service is following the
|
||||
- AWS API calls are wrapped in try/except blocks, with specific handling for `ClientError` and generic exceptions, always logging errors.
|
||||
- If ARN is not present for some resource, it can be constructed using string interpolation, always including partition, service, region, account, and resource ID.
|
||||
- Tags and additional attributes that cannot be retrieved from the default call, should be collected and stored for each resource using dedicated methods and threading using the resource object list as iterator.
|
||||
- When accessing dictionary values from AWS API responses, always use `.get()` with a default value instead of direct dictionary access (e.g., `response.get("Policies", {})` instead of `response["Policies"]`). AWS API responses may not always include all keys, and direct access can cause `KeyError` exceptions that break the entire scan for that service.
|
||||
|
||||
### Extending an Existing Service with New Attributes
|
||||
|
||||
When adding a new check that requires data not yet collected by an existing service, you need to extend the service by adding new attributes to its resource models and updating the data collection methods. This is a common contributor task that follows a consistent pattern:
|
||||
|
||||
1. **Identify the missing data**: Determine which AWS API call provides the data you need and whether it's already being called by the service.
|
||||
|
||||
2. **Add new attributes to the resource model**: Extend the Pydantic `BaseModel` class for the resource with the new fields. Use `Optional` types with `None` as the default value to maintain backward compatibility with existing checks.
|
||||
|
||||
3. **Update the data collection method**: Modify the existing method that fetches resource details to also extract and store the new attributes. If no existing method fetches the data, add a new method and call it in the constructor using `self.__threading_call__` if possible.
|
||||
|
||||
4. **Use safe dictionary access**: When extracting values from API responses, always use `.get()` with appropriate defaults to prevent `KeyError` exceptions when the API doesn't return certain fields.
|
||||
|
||||
#### Example: Adding DKIM Status to SES Identities
|
||||
|
||||
```python
|
||||
# Step 1 & 2: Add new fields to the resource model
|
||||
class Identity(BaseModel):
|
||||
name: str
|
||||
arn: str
|
||||
region: str
|
||||
type: Optional[str]
|
||||
policy: Optional[dict] = None
|
||||
tags: Optional[list] = []
|
||||
# New attributes for DKIM check
|
||||
dkim_status: Optional[str] = None
|
||||
dkim_signing_attributes_origin: Optional[str] = None
|
||||
|
||||
# Step 3: Update the data collection method
|
||||
def _get_email_identities(self, identity):
|
||||
try:
|
||||
regional_client = self.regional_clients[identity.region]
|
||||
identity_attributes = regional_client.get_email_identity(
|
||||
EmailIdentity=identity.name
|
||||
)
|
||||
# Step 4: Use .get() for safe dictionary access
|
||||
for content_key, content_value in identity_attributes.get("Policies", {}).items():
|
||||
identity.policy = loads(content)
|
||||
identity.tags = identity_attributes.get("Tags", [])
|
||||
# Extract new DKIM attributes
|
||||
identity.dkim_status = identity_attributes.get("DkimStatus")
|
||||
identity.dkim_signing_attributes_origin = (
|
||||
identity_attributes.get("DkimSigningAttributesOrigin")
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
```
|
||||
|
||||
5. **Update the service tests**: Add the new attributes to the test mock data and assertions to verify correct data extraction.
|
||||
|
||||
## Specific Patterns in AWS Checks
|
||||
|
||||
|
||||
@@ -215,3 +215,6 @@ Also is important to keep all code examples as short as possible, including the
|
||||
| e5 | M365 and Azure Entra checks enabled by or dependent on an E5 license (e.g., advanced threat protection, audit, DLP, and eDiscovery) |
|
||||
| privilege-escalation | Detects IAM policies or permissions that allow identities to elevate their privileges beyond their intended scope, potentially gaining administrator or higher-level access through specific action combinations |
|
||||
| ec2-imdsv1 | Identifies EC2 instances using Instance Metadata Service version 1 (IMDSv1), which is vulnerable to SSRF attacks and should be replaced with IMDSv2 for enhanced security |
|
||||
| vercel-hobby-plan | Vercel checks whose audited feature is available on the Hobby plan (and therefore also on Pro and Enterprise plans) |
|
||||
| vercel-pro-plan | Vercel checks whose audited feature requires a Pro plan or higher, including features also available on Enterprise or via supported paid add-ons for Pro plans |
|
||||
| vercel-enterprise-plan | Vercel checks whose audited feature requires the Enterprise plan |
|
||||
|
||||
@@ -20,21 +20,35 @@ The most common high level steps to create a new check are:
|
||||
3. Create a check-specific folder. The path should follow this pattern: `prowler/providers/<provider>/services/<service>/<check_name_want_to_implement>`. Adhere to the [Naming Format for Checks](#naming-format-for-checks).
|
||||
4. Populate the folder with files as specified in [File Creation](#file-creation).
|
||||
5. Run the check locally to ensure it works as expected. For checking you can use the CLI in the next way:
|
||||
- To ensure the check has been detected by Prowler: `poetry run python prowler-cli.py <provider> --list-checks | grep <check_name>`.
|
||||
- To run the check, to find possible issues: `poetry run python prowler-cli.py <provider> --log-level ERROR --verbose --check <check_name>`.
|
||||
- To ensure the check has been detected by Prowler: `uv run python prowler-cli.py <provider> --list-checks | grep <check_name>`.
|
||||
- To run the check, to find possible issues: `uv run python prowler-cli.py <provider> --log-level ERROR --verbose --check <check_name>`.
|
||||
6. Create comprehensive tests for the check that cover multiple scenarios including both PASS (compliant) and FAIL (non-compliant) cases. For detailed information about test structure and implementation guidelines, refer to the [Testing](/developer-guide/unit-testing) documentation.
|
||||
7. If the check and its corresponding tests are working as expected, you can submit a PR to Prowler.
|
||||
|
||||
### Naming Format for Checks
|
||||
|
||||
Checks must be named following the format: `service_subservice_resource_action`.
|
||||
If you already know the check name when creating a request or implementing a check, use a descriptive identifier with lowercase letters and underscores only.
|
||||
|
||||
Recommended patterns:
|
||||
|
||||
- `<service>_<resource>_<best_practice>`
|
||||
|
||||
The name components are:
|
||||
|
||||
- `service` – The main service being audited (e.g., ec2, entra, iam, etc.)
|
||||
- `subservice` – An individual component or subset of functionality within the service that is being audited. This may correspond to a shortened version of the class attribute accessed within the check. If there is no subservice, just omit.
|
||||
- `resource` – The specific resource type being evaluated (e.g., instance, policy, role, etc.)
|
||||
- `action` – The security aspect or configuration being checked (e.g., public, encrypted, enabled, etc.)
|
||||
- `service` – The main service or product area being audited (e.g., ec2, entra, iam, bedrock).
|
||||
- `resource` – The resource, feature, or configuration being evaluated. It can be a single word or a compound phrase joined with underscores (e.g., instance, policy, guardrail, sensitive_information_filter).
|
||||
- `best_practice` – The expected secure state or best practice being checked (e.g., enabled, encrypted, restricted, configured, not_publicly_accessible).
|
||||
|
||||
Additional guidance:
|
||||
|
||||
- Use underscores only. Do not use hyphens.
|
||||
- Keep the name specific enough to describe the behavior of the check.
|
||||
- The first segment should match the service or product area whenever possible.
|
||||
|
||||
Examples:
|
||||
|
||||
- `s3_bucket_versioning_enabled`
|
||||
- `bedrock_guardrail_sensitive_information_filter_enabled`
|
||||
|
||||
### File Creation
|
||||
|
||||
@@ -387,7 +401,7 @@ Provides both code examples and best practice recommendations for addressing the
|
||||
|
||||
#### Categories
|
||||
|
||||
One or more functional groupings used for execution filtering (e.g., `internet-exposed`). You can define new categories just by adding to this field.
|
||||
One or more functional groupings used for execution filtering (e.g., `internet-exposed`). Categories must match the predefined values enforced by `CheckMetadata`; adding a new category requires updating the validator and the metadata documentation.
|
||||
|
||||
For the complete list of available categories, see [Categories Guidelines](/developer-guide/check-metadata-guidelines#categories-guidelines).
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ This includes the [AGENTS.md](https://github.com/prowler-cloud/prowler/blob/mast
|
||||
<Steps>
|
||||
<Step title="Install Mintlify CLI">
|
||||
```bash
|
||||
npm i -g mint
|
||||
npm install --global mint@4.2.560
|
||||
```
|
||||
For detailed instructions, check the [Mintlify documentation](https://www.mintlify.com/docs/installation).
|
||||
</Step>
|
||||
|
||||
@@ -80,7 +80,7 @@ Before proceeding, ensure the following:
|
||||
|
||||
- Git is installed.
|
||||
- Python 3.10 or higher is installed.
|
||||
- `poetry` is installed to manage dependencies.
|
||||
- `uv` is installed to manage dependencies.
|
||||
|
||||
### Forking the Prowler Repository
|
||||
|
||||
@@ -97,28 +97,21 @@ cd prowler
|
||||
|
||||
### Dependency Management and Environment Isolation
|
||||
|
||||
To prevent conflicts between environments, we recommend using `poetry`, a Python dependency management solution. Install it by following the [instructions](https://python-poetry.org/docs/#installation).
|
||||
To prevent conflicts between environments, we recommend using [`uv`](https://docs.astral.sh/uv/), a fast Python package and project manager. Install it by following the [official instructions](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
|
||||
### Installing Dependencies
|
||||
|
||||
To install all required dependencies, including those needed for development, run:
|
||||
|
||||
```
|
||||
poetry install --with dev
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
```
|
||||
|
||||
<Warning>
|
||||
Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`.
|
||||
If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment.
|
||||
In case you have any doubts, consult the [Poetry environment activation guide](https://python-poetry.org/docs/managing-environments/#activating-the-environment).
|
||||
|
||||
</Warning>
|
||||
|
||||
|
||||
### Pre-Commit Hooks
|
||||
|
||||
This repository uses Git pre-commit hooks managed by the [prek](https://prek.j178.dev/) tool, it is installed with `poetry install --with dev`. Next, run the following command in the root of this repository:
|
||||
This repository uses Git pre-commit hooks managed by the [prek](https://prek.j178.dev/) tool, it is installed with `uv sync`. Next, run the following command in the root of this repository:
|
||||
|
||||
```shell
|
||||
prek install
|
||||
@@ -134,16 +127,32 @@ prek installed at `.git/hooks/pre-commit`
|
||||
If pre-commit hooks were previously installed, run `prek install --overwrite` to replace the existing hook. Otherwise, both tools will run on each commit.
|
||||
</Warning>
|
||||
|
||||
#### Enable TruffleHog as a Pre-Push Hook
|
||||
|
||||
By default, only `pre-commit` hooks are installed. To enable [`TruffleHog`](https://github.com/trufflesecurity/trufflehog) secret scanning on every push, install the `pre-push` hook type explicitly:
|
||||
|
||||
```shell
|
||||
prek install --hook-type pre-push
|
||||
```
|
||||
|
||||
Successful installation should produce the following output:
|
||||
|
||||
```shell
|
||||
prek installed at `.git/hooks/pre-push`
|
||||
```
|
||||
|
||||
Once installed, TruffleHog runs before each push and blocks the operation when verified secrets are detected.
|
||||
|
||||
### Code Quality and Security Checks
|
||||
|
||||
Before merging pull requests, several automated checks and utilities ensure code security and updated dependencies:
|
||||
|
||||
<Note>
|
||||
These should have been already installed if `poetry install --with dev` was already run.
|
||||
These should have been already installed if `uv sync` was already run.
|
||||
|
||||
</Note>
|
||||
- [`bandit`](https://pypi.org/project/bandit/) for code security review.
|
||||
- [`safety`](https://pypi.org/project/safety/) and [`dependabot`](https://github.com/features/security) for dependencies.
|
||||
- [`osv-scanner`](https://github.com/google/osv-scanner) and [`dependabot`](https://github.com/features/security) for dependencies.
|
||||
- [`hadolint`](https://github.com/hadolint/hadolint) and [`dockle`](https://github.com/goodwithtech/dockle) for container security.
|
||||
- [`Snyk`](https://docs.snyk.io/integrations/snyk-container-integrations/container-security-with-docker-hub-integration) for container security in Docker Hub.
|
||||
- [`clair`](https://github.com/quay/clair) for container security in Amazon ECR.
|
||||
@@ -167,7 +176,7 @@ These resources help ensure that AI-assisted contributions maintain consistency
|
||||
|
||||
All dependencies are listed in the `pyproject.toml` file.
|
||||
|
||||
The SDK keeps direct dependencies pinned to exact versions, while `poetry.lock` records the full resolved dependency tree and the artifact hashes for every package. Use `poetry install` from the lock file instead of ad-hoc `pip` installs when you need a reproducible environment.
|
||||
The SDK keeps direct dependencies pinned to exact versions, while `uv.lock` records the full resolved dependency tree and the artifact hashes for every package. Use `uv sync` from the lock file instead of ad-hoc `pip` installs when you need a reproducible environment.
|
||||
|
||||
For proper code documentation, refer to the following and follow the code documentation practices presented there: [Google Python Style Guide - Comments and Docstrings](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings).
|
||||
|
||||
@@ -193,8 +202,8 @@ prowler/
|
||||
├── contrib/ # Community-contributed scripts or modules
|
||||
├── kubernetes/ # Kubernetes deployment files
|
||||
├── .github/ # GitHub-related files (workflows, issue templates, etc.)
|
||||
├── pyproject.toml # Python project configuration (Poetry)
|
||||
├── poetry.lock # Poetry lock file
|
||||
├── pyproject.toml # Python project configuration (uv)
|
||||
├── uv.lock # uv lock file
|
||||
├── README.md # Project overview and getting started
|
||||
├── Makefile # Common development commands
|
||||
├── Dockerfile # SDK Docker container
|
||||
|
||||
@@ -1003,7 +1003,7 @@ class ProwlerArgumentParser:
|
||||
formatter_class=RawTextHelpFormatter,
|
||||
usage="prowler [-h] [--version] {aws,azure,gcp,kubernetes,m365,github,nhn,dashboard,iac,your_provider} ...",
|
||||
epilog="""
|
||||
Available Cloud Providers:
|
||||
Available Providers:
|
||||
{aws,azure,gcp,kubernetes,m365,github,iac,nhn,your_provider}
|
||||
aws AWS Provider
|
||||
azure Azure Provider
|
||||
@@ -1277,10 +1277,12 @@ Dependencies ensure that your provider's required libraries are available when P
|
||||
**File:** `pyproject.toml`
|
||||
|
||||
```toml
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.10,<3.13"
|
||||
# ... other dependencies
|
||||
your-sdk-library = "^1.0.0" # Add your SDK dependency
|
||||
[project]
|
||||
requires-python = ">=3.10,<3.13"
|
||||
dependencies = [
|
||||
# ... other dependencies
|
||||
"your-sdk-library>=1.0.0,<2.0.0", # Add your SDK dependency
|
||||
]
|
||||
```
|
||||
|
||||
#### Step 18: Create Tests
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user