mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-09 21:04:53 +00:00
Compare commits
265 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5487372c99 | |||
| ea34d34876 | |||
| f3d0f55d52 | |||
| 50492e2d6f | |||
| 7b190f01c8 | |||
| abebbeea20 | |||
| a835843cc7 | |||
| a7f4f44e7b | |||
| 2a31bfc3e6 | |||
| 1a4cfd81c5 | |||
| c0559e7f10 | |||
| 706742e6dc | |||
| baaf56ea5e | |||
| cb01769237 | |||
| 4c802620c4 | |||
| 4fa8d5465e | |||
| 31b9619627 | |||
| d4a1bc10e9 | |||
| a1848747a3 | |||
| 4c0a3f477f | |||
| bc443eef22 | |||
| 298ad3382f | |||
| bfcbe0a9c4 | |||
| 37aa290d1c | |||
| 5cd7fe4f96 | |||
| 0234f038f0 | |||
| 5ca6e31f45 | |||
| 21df5c58b1 | |||
| 964548a677 | |||
| 5d34577b0b | |||
| 855e74add0 | |||
| 40b7cb3991 | |||
| c1cfda530f | |||
| 9894ac7bc3 | |||
| 8ae076f22c | |||
| 27fb8518cd | |||
| 375de5dc87 | |||
| 3410fc927a | |||
| dcf91ef252 | |||
| bf4fd8fabd | |||
| 3d65208fd3 | |||
| 7d3ed62e90 | |||
| 5f92989492 | |||
| 6befa78978 | |||
| 78af0c24fe | |||
| 1bb547e5e1 | |||
| 1f39b01fb2 | |||
| fb0ef391f2 | |||
| f2e6a3264d | |||
| 9bd4e4b65c | |||
| 68ffb2b219 | |||
| 739be07077 | |||
| 0abbb7fc59 | |||
| 0b4393776c | |||
| 4dd5baadf6 | |||
| 934d995661 | |||
| ccdc01ed7b | |||
| d84099e87a | |||
| cf55f7eb43 | |||
| 9293c7b58d | |||
| a883bb30d4 | |||
| e476bbde2d | |||
| 7f3dcdf02f | |||
| 132e79df89 | |||
| b2ed9ee221 | |||
| def2d3d188 | |||
| 1090ed59b7 | |||
| 67e4b1a082 | |||
| 7478ec9420 | |||
| a30b6623ed | |||
| 15bc317ec4 | |||
| 1536102784 | |||
| 1b99550572 | |||
| 6dfa135755 | |||
| 80482da1cb | |||
| 9cedbd3582 | |||
| c3d1c5c5f7 | |||
| 1fd6c51af6 | |||
| adbfc0bcd1 | |||
| 8f041f6f52 | |||
| 1b0e12ec51 | |||
| 759f7b84d6 | |||
| 0b26c1a39c | |||
| fc7fbddfe7 | |||
| 500b395125 | |||
| a1961d6d5f | |||
| a7e988c361 | |||
| 02cdcb29db | |||
| 6e0d7866cd | |||
| 4b71f37c91 | |||
| cdfbe5b2e3 | |||
| 1b6a459df4 | |||
| 73c0305dc4 | |||
| 0e01e67257 | |||
| 1ad329f9cf | |||
| d03d1d2393 | |||
| 832516be2a | |||
| 34727a7237 | |||
| 4216a3e23a | |||
| a59192e6f5 | |||
| 592bc6f6a8 | |||
| 962ebac8e4 | |||
| 2c5d47a8cd | |||
| bcaa6ac488 | |||
| 71683f3093 | |||
| 2357af912d | |||
| 7971b40f49 | |||
| e585ae45bd | |||
| 4d9921a9b7 | |||
| 19b602c381 | |||
| 4c3e741af7 | |||
| 8affbf44ce | |||
| 16798e293d | |||
| 1194d34396 | |||
| 98277689f5 | |||
| 0ddd7fbd69 | |||
| 22b233f206 | |||
| aa759ab6b7 | |||
| 369d6cecc1 | |||
| d23c2f3b53 | |||
| 786059bfb2 | |||
| 703a33108c | |||
| 7c6d658154 | |||
| 21d7d08b4b | |||
| f314725f4d | |||
| 02f43a7ad6 | |||
| 0dd8981ee4 | |||
| 269e51259d | |||
| f4afdf0541 | |||
| 652cb69216 | |||
| 921f49a0de | |||
| 6cb770fcc8 | |||
| 86449fb99d | |||
| 40dd0e640b | |||
| 8db3a89669 | |||
| c802dc8a36 | |||
| 3ab9a4efa5 | |||
| 36b8aa1b79 | |||
| e821e07d7d | |||
| 228fe6d579 | |||
| 578186aa40 | |||
| 4608e45c8a | |||
| 5987651aee | |||
| 85800f2ddd | |||
| 4fb5272362 | |||
| 85d38b5f71 | |||
| 59dcdb87c4 | |||
| 9297453b8a | |||
| dd37f4ee1f | |||
| 20f36f7c84 | |||
| ec4d27746f | |||
| 7076900fb1 | |||
| 5d90352a0f | |||
| a981dc64a7 | |||
| d2086cad3f | |||
| 380b89cfb6 | |||
| 13b04d339b | |||
| be3c5fb3c1 | |||
| 1de01bcb78 | |||
| 13d983450c | |||
| 8b368e1343 | |||
| c76a9baa20 | |||
| 30e2813e02 | |||
| 0f874c6ffd | |||
| 2242689295 | |||
| e252058af4 | |||
| 37e6c9761f | |||
| ebe666bec7 | |||
| 7df2703db1 | |||
| 67234210ba | |||
| 15ca69942d | |||
| df76efc197 | |||
| 3441ad7f70 | |||
| 059b71d34b | |||
| 013809919c | |||
| 368d9c1519 | |||
| fb6da427f8 | |||
| 65fd3335d3 | |||
| d6288be472 | |||
| 0cddb71d1c | |||
| af2930130c | |||
| b668770480 | |||
| f31c5717e9 | |||
| 4788dcade2 | |||
| 22a6cc9e73 | |||
| 06bb382f8e | |||
| d4ece2b43e | |||
| b97d68fbd5 | |||
| ca79300440 | |||
| 7a0e107617 | |||
| 6d3fcec5da | |||
| ce1cf51d37 | |||
| 3554859a5c | |||
| 80d62f355f | |||
| 0df24eeff6 | |||
| d1fc482832 | |||
| ffb1bb89e1 | |||
| d877bea0e3 | |||
| 2304bf0093 | |||
| 2ca74102a9 | |||
| 6ae129fcc0 | |||
| e9731f53ad | |||
| db2f92e6d5 | |||
| f4b0f8fa22 | |||
| dff5541e11 | |||
| 927be17fb7 | |||
| c27cb28a2a | |||
| 94ee24071a | |||
| 1093f6c99b | |||
| 48060c47ba | |||
| 72acc2119d | |||
| b1ebea4a7e | |||
| 001057644e | |||
| 1456def7d4 | |||
| 12d475e7af | |||
| 43bd1083e0 | |||
| bbd4ce7565 | |||
| 97a085bf21 | |||
| 29a2f8fac8 | |||
| a24869fc26 | |||
| 72c94db1cf | |||
| 4ef7bbdb7c | |||
| f2c5d2ec87 | |||
| 61a62fd6e0 | |||
| 39911e3ab7 | |||
| bcce8d6236 | |||
| 570c86948e | |||
| 548389d79f | |||
| fc3066bc60 | |||
| ac6dd03fb8 | |||
| d3a1df3473 | |||
| 858dfc2a00 | |||
| 6b0ba79652 | |||
| 390bbdd1a6 | |||
| 8d48c26c1e | |||
| 98b9449e14 | |||
| 3406c5ec64 | |||
| 4346401a0a | |||
| dcec79d259 | |||
| 2a9c538aff | |||
| bf1b53bbd2 | |||
| 94a2ea1e8f | |||
| f7194b32de | |||
| 6ffe4e95bf | |||
| 577aa14acc | |||
| 19c752c127 | |||
| f2d35f5885 | |||
| 536e90f2a5 | |||
| 276a5d66bd | |||
| 489c6c1073 | |||
| b08b072288 | |||
| ca29e354b6 | |||
| 85a3927950 | |||
| 04fe3f65e0 | |||
| 297c9d0734 | |||
| a2a1a73749 | |||
| 08fbe17e29 | |||
| d920f78059 | |||
| 12bf3d5e70 | |||
| 4002c28b5d | |||
| 2439f54280 | |||
| b0e59156e6 | |||
| f013bd4a53 | |||
| 6ad15f900f | |||
| 1784bf38ab |
@@ -0,0 +1,22 @@
|
||||
# Prowler worktree automation for worktrunk (wt CLI).
|
||||
# Runs automatically on `wt switch --create`.
|
||||
|
||||
# Block 1: setup + copy gitignored env files (.envrc, ui/.env.local)
|
||||
# from the primary worktree - patterns selected via .worktreeinclude.
|
||||
[[pre-start]]
|
||||
skills = "./skills/setup.sh --claude"
|
||||
envs = "wt step copy-ignored"
|
||||
|
||||
# Block 2: install Python deps (uv manages the venv on `uv sync`).
|
||||
[[pre-start]]
|
||||
deps = "uv sync"
|
||||
|
||||
# Block 3: reminder - last visible output before `wt switch` returns.
|
||||
# Hooks can't mutate the parent shell, so venv activation is manual.
|
||||
[[pre-start]]
|
||||
reminder = "echo '>> Reminder: activate the venv in this shell with: source .venv/bin/activate'"
|
||||
|
||||
# Background: pnpm install runs while you start working.
|
||||
# Tail logs via `wt config state logs`.
|
||||
[post-start]
|
||||
ui = "cd ui && pnpm install"
|
||||
@@ -145,7 +145,7 @@ SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.24.5
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.27.1
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
+12
-11
@@ -1,14 +1,15 @@
|
||||
# SDK
|
||||
/* @prowler-cloud/sdk
|
||||
/prowler/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation
|
||||
/tests/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation
|
||||
/dashboard/ @prowler-cloud/sdk
|
||||
/docs/ @prowler-cloud/sdk
|
||||
/examples/ @prowler-cloud/sdk
|
||||
/util/ @prowler-cloud/sdk
|
||||
/contrib/ @prowler-cloud/sdk
|
||||
/permissions/ @prowler-cloud/sdk
|
||||
/codecov.yml @prowler-cloud/sdk @prowler-cloud/api
|
||||
/* @prowler-cloud/detection-remediation
|
||||
/prowler/ @prowler-cloud/detection-remediation
|
||||
/prowler/compliance/ @prowler-cloud/compliance
|
||||
/tests/ @prowler-cloud/detection-remediation
|
||||
/dashboard/ @prowler-cloud/detection-remediation
|
||||
/docs/ @prowler-cloud/detection-remediation
|
||||
/examples/ @prowler-cloud/detection-remediation
|
||||
/util/ @prowler-cloud/detection-remediation
|
||||
/contrib/ @prowler-cloud/detection-remediation
|
||||
/permissions/ @prowler-cloud/detection-remediation
|
||||
/codecov.yml @prowler-cloud/detection-remediation @prowler-cloud/api
|
||||
|
||||
# API
|
||||
/api/ @prowler-cloud/api
|
||||
@@ -17,7 +18,7 @@
|
||||
/ui/ @prowler-cloud/ui
|
||||
|
||||
# AI
|
||||
/mcp_server/ @prowler-cloud/ai
|
||||
/mcp_server/ @prowler-cloud/detection-remediation
|
||||
|
||||
# Platform
|
||||
/.github/ @prowler-cloud/platform
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: [prowler-cloud]
|
||||
# patreon: # Replace with a single Patreon username
|
||||
# open_collective: # Replace with a single Open Collective username
|
||||
# ko_fi: # Replace with a single Ko-fi username
|
||||
# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
# liberapay: # Replace with a single Liberapay username
|
||||
# issuehunt: # Replace with a single IssueHunt username
|
||||
# lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
||||
# polar: # Replace with a single Polar username
|
||||
# buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
|
||||
# thanks_dev: # Replace with a single thanks.dev username
|
||||
# custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
||||
@@ -0,0 +1,143 @@
|
||||
name: "🔎 New Check Request"
|
||||
description: Request a new Prowler security check
|
||||
title: "[New Check]: "
|
||||
labels: ["feature-request", "status/needs-triage"]
|
||||
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: search
|
||||
attributes:
|
||||
label: Existing check search
|
||||
description: Confirm this check does not already exist before opening a new request.
|
||||
options:
|
||||
- label: I have searched existing issues, Prowler Hub, and the public roadmap, and this check does not already exist.
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Use this form to describe the security condition that Prowler should evaluate.
|
||||
|
||||
The most useful inputs for [Prowler Studio](https://github.com/prowler-cloud/prowler-studio) are:
|
||||
- What should be detected
|
||||
- What PASS and FAIL mean
|
||||
- Vendor docs, API references, SDK methods, CLI commands, or reference code
|
||||
|
||||
- type: dropdown
|
||||
id: provider
|
||||
attributes:
|
||||
label: Provider
|
||||
description: Cloud or platform this check targets.
|
||||
options:
|
||||
- AWS
|
||||
- Azure
|
||||
- GCP
|
||||
- Kubernetes
|
||||
- GitHub
|
||||
- Microsoft 365
|
||||
- OCI
|
||||
- Alibaba Cloud
|
||||
- Cloudflare
|
||||
- MongoDB Atlas
|
||||
- Google Workspace
|
||||
- OpenStack
|
||||
- Vercel
|
||||
- NHN
|
||||
- Other / New provider
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: other_provider_name
|
||||
attributes:
|
||||
label: New provider name
|
||||
description: Only fill this if you selected "Other / New provider" above.
|
||||
placeholder: "NewProviderName"
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: service_name
|
||||
attributes:
|
||||
label: Service or product area
|
||||
description: Optional. Main service, product, or feature to audit.
|
||||
placeholder: "s3, bedrock, entra, repository, apiserver"
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: suggested_check_name
|
||||
attributes:
|
||||
label: Suggested check name
|
||||
description: Optional. Use `snake_case` following `<service>_<resource>_<best_practice>`, with lowercase letters and underscores only.
|
||||
placeholder: "bedrock_guardrail_sensitive_information_filter_enabled"
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: context
|
||||
attributes:
|
||||
label: Context and goal
|
||||
description: Describe the security problem, why it matters, and what this new check should help detect.
|
||||
placeholder: |-
|
||||
- Security condition to validate:
|
||||
- Why it matters:
|
||||
- Resource, feature, or configuration involved:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected_behavior
|
||||
attributes:
|
||||
label: Expected behavior
|
||||
description: Explain what the check should evaluate and what PASS, FAIL, or MANUAL should mean.
|
||||
placeholder: |-
|
||||
- Resource or scope to evaluate:
|
||||
- PASS when:
|
||||
- FAIL when:
|
||||
- MANUAL when (if applicable):
|
||||
- Exclusions, thresholds, or edge cases:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: references
|
||||
attributes:
|
||||
label: References
|
||||
description: Add vendor docs, API references, SDK methods, CLI commands, endpoint docs, sample payloads, or similar reference material.
|
||||
placeholder: |-
|
||||
- Product or service documentation:
|
||||
- API or SDK reference:
|
||||
- CLI command or endpoint documentation:
|
||||
- Sample payload or response:
|
||||
- Security advisory or benchmark:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: severity
|
||||
attributes:
|
||||
label: Suggested severity
|
||||
description: Your best estimate. Reviewers will confirm during triage.
|
||||
options:
|
||||
- Critical
|
||||
- High
|
||||
- Medium
|
||||
- Low
|
||||
- Informational
|
||||
- Not sure
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: implementation_notes
|
||||
attributes:
|
||||
label: Additional implementation notes
|
||||
description: Optional. Add permissions, unsupported regions, config knobs, product limitations, or anything else that may affect implementation.
|
||||
placeholder: |-
|
||||
- Required permissions or scopes:
|
||||
- Region, tenant, or subscription limitations:
|
||||
- Configurable behavior or thresholds:
|
||||
- Other constraints:
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,169 @@
|
||||
name: 'OSV-Scanner'
|
||||
description: 'Install osv-scanner and scan a lockfile, failing on HIGH/CRITICAL/UNKNOWN severity findings. Posts/updates a PR comment with findings on pull_request events (requires pull-requests: write).'
|
||||
author: 'Prowler'
|
||||
|
||||
inputs:
|
||||
lockfile:
|
||||
description: 'Path to the lockfile to scan, relative to the repository root (e.g. uv.lock, api/uv.lock, ui/pnpm-lock.yaml).'
|
||||
required: true
|
||||
severity-levels:
|
||||
description: 'Comma-separated severity levels that fail the scan. Default: HIGH,CRITICAL,UNKNOWN.'
|
||||
required: false
|
||||
default: 'HIGH,CRITICAL,UNKNOWN'
|
||||
version:
|
||||
description: 'osv-scanner release tag to install. When overriding, you MUST also override binary-sha256.'
|
||||
required: false
|
||||
default: 'v2.3.8'
|
||||
binary-sha256:
|
||||
description: 'Expected SHA256 of osv-scanner_linux_amd64 for the given version. Default tracks v2.3.8. See https://github.com/google/osv-scanner/releases/download/<version>/osv-scanner_SHA256SUMS.'
|
||||
required: false
|
||||
default: 'bc98e15319ed0d515e3f9235287ba53cdc5535d576d24fd573978ecfe9ab92dc'
|
||||
post-pr-comment:
|
||||
description: 'Post or update a PR comment with the scan report. Only effective on pull_request events. Requires pull-requests: write permission on the caller job.'
|
||||
required: false
|
||||
default: 'true'
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Install osv-scanner
|
||||
shell: bash
|
||||
env:
|
||||
OSV_SCANNER_VERSION: ${{ inputs.version }}
|
||||
# Download the binary AND the published SHA256SUMS file, then verify the
|
||||
# binary checksum against the upstream-signed manifest. Aborts on mismatch.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if command -v osv-scanner >/dev/null 2>&1; then
|
||||
INSTALLED="$(osv-scanner --version 2>&1 | awk '/scanner version/ {print $NF; exit}')"
|
||||
if [ "v${INSTALLED}" = "${OSV_SCANNER_VERSION}" ]; then
|
||||
echo "osv-scanner ${OSV_SCANNER_VERSION} already installed."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
BASE="https://github.com/google/osv-scanner/releases/download/${OSV_SCANNER_VERSION}"
|
||||
BIN_NAME="osv-scanner_linux_amd64"
|
||||
curl -fSL --retry 3 "${BASE}/${BIN_NAME}" -o "${RUNNER_TEMP}/${BIN_NAME}"
|
||||
curl -fSL --retry 3 "${BASE}/osv-scanner_SHA256SUMS" -o "${RUNNER_TEMP}/osv-scanner_SHA256SUMS"
|
||||
(cd "${RUNNER_TEMP}" && sha256sum --check --ignore-missing osv-scanner_SHA256SUMS)
|
||||
chmod +x "${RUNNER_TEMP}/${BIN_NAME}"
|
||||
sudo mv "${RUNNER_TEMP}/${BIN_NAME}" /usr/local/bin/osv-scanner
|
||||
rm -f "${RUNNER_TEMP}/osv-scanner_SHA256SUMS"
|
||||
osv-scanner --version
|
||||
|
||||
- name: Run osv-scanner
|
||||
id: scan
|
||||
shell: bash
|
||||
working-directory: ${{ github.workspace }}
|
||||
env:
|
||||
OSV_LOCKFILE: ${{ inputs.lockfile }}
|
||||
OSV_SEVERITY_LEVELS: ${{ inputs.severity-levels }}
|
||||
OSV_REPORT_FILE: ${{ runner.temp }}/osv-scanner-findings.json
|
||||
# Per-vulnerability ignores (reason + expiry) live in osv-scanner.toml at the repo root, if present.
|
||||
# Severity filter is enforced in the wrapper via OSV_SEVERITY_LEVELS.
|
||||
# `continue-on-error: true` lets the PR-comment step run even when findings exist;
|
||||
# the gate step below re-fails the job from the wrapper exit code.
|
||||
continue-on-error: true
|
||||
run: ./.github/scripts/osv-scan.sh --lockfile="${OSV_LOCKFILE}"
|
||||
|
||||
- name: Post osv-scanner report on PR
|
||||
if: >-
|
||||
always()
|
||||
&& inputs.post-pr-comment == 'true'
|
||||
&& github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.head.repo.full_name == github.repository
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
||||
env:
|
||||
OSV_REPORT_FILE: ${{ runner.temp }}/osv-scanner-findings.json
|
||||
OSV_LOCKFILE: ${{ inputs.lockfile }}
|
||||
OSV_SEVERITY_LEVELS: ${{ inputs.severity-levels }}
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const lockfile = process.env.OSV_LOCKFILE;
|
||||
const severityLevels = process.env.OSV_SEVERITY_LEVELS;
|
||||
const reportFile = process.env.OSV_REPORT_FILE;
|
||||
const marker = `<!-- osv-scanner-report:${lockfile} -->`;
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
let findings = [];
|
||||
if (fs.existsSync(reportFile)) {
|
||||
try {
|
||||
findings = JSON.parse(fs.readFileSync(reportFile, 'utf8'));
|
||||
} catch (err) {
|
||||
core.warning(`Could not parse ${reportFile}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body?.includes(marker));
|
||||
|
||||
if (findings.length === 0) {
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
core.info(`Deleted stale osv-scanner comment for ${lockfile}.`);
|
||||
} else {
|
||||
core.info(`No findings and no stale comment for ${lockfile}.`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const sevIcon = (s) => ({
|
||||
CRITICAL: '🔴', HIGH: '🟠', MEDIUM: '🟡', LOW: '🟢', UNKNOWN: '⚪',
|
||||
}[s] || '⚪');
|
||||
const escape = (s) => String(s ?? '').replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
||||
const rows = findings.map(f =>
|
||||
`| ${sevIcon(f.severity)} ${f.severity}${f.score ? ` (${f.score})` : ''} | \`${escape(f.id)}\` | \`${escape(f.ecosystem)}/${escape(f.package)}\` | \`${escape(f.version)}\` | ${escape(f.summary || '(no summary)')} |`
|
||||
);
|
||||
|
||||
const body = [
|
||||
marker,
|
||||
`## 🔒 osv-scanner: ${findings.length} finding(s) in \`${lockfile}\``,
|
||||
'',
|
||||
`Severity gate: \`${severityLevels}\``,
|
||||
'',
|
||||
'| Severity | ID | Package | Version | Summary |',
|
||||
'|----------|----|---------|---------|---------|',
|
||||
...rows,
|
||||
'',
|
||||
`To accept a finding, add an \`[[IgnoredVulns]]\` entry to \`osv-scanner.toml\` at the repo root with a reason and \`ignoreUntil\`.`,
|
||||
'',
|
||||
`<sub>[View run](${runUrl})</sub>`,
|
||||
].join('\n');
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
core.info(`Updated osv-scanner comment for ${lockfile}.`);
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
core.info(`Posted new osv-scanner comment for ${lockfile}.`);
|
||||
}
|
||||
|
||||
- name: Enforce osv-scanner severity gate
|
||||
shell: bash
|
||||
env:
|
||||
SCAN_OUTCOME: ${{ steps.scan.outcome }}
|
||||
run: |
|
||||
if [ "${SCAN_OUTCOME}" != "success" ]; then
|
||||
echo "osv-scanner gate: scan reported findings (outcome=${SCAN_OUTCOME})" >&2
|
||||
exit 1
|
||||
fi
|
||||
+24
-47
@@ -1,5 +1,5 @@
|
||||
name: 'Setup Python with Poetry'
|
||||
description: 'Setup Python environment with Poetry and install dependencies'
|
||||
name: 'Setup Python with uv'
|
||||
description: 'Setup Python environment with uv and install dependencies'
|
||||
author: 'Prowler'
|
||||
|
||||
inputs:
|
||||
@@ -7,23 +7,15 @@ inputs:
|
||||
description: 'Python version to use'
|
||||
required: true
|
||||
working-directory:
|
||||
description: 'Working directory for Poetry'
|
||||
description: 'Working directory for uv'
|
||||
required: false
|
||||
default: '.'
|
||||
poetry-version:
|
||||
description: 'Poetry version to install'
|
||||
uv-version:
|
||||
description: 'uv version to install'
|
||||
required: false
|
||||
default: '2.3.4'
|
||||
default: '0.11.14'
|
||||
install-dependencies:
|
||||
description: 'Install Python dependencies with Poetry'
|
||||
required: false
|
||||
default: 'true'
|
||||
update-lock:
|
||||
description: 'Run `poetry lock` during setup. Only enable when a prior step mutates pyproject.toml (e.g. API `@master` VCS rewrite). Default: false.'
|
||||
required: false
|
||||
default: 'false'
|
||||
enable-cache:
|
||||
description: 'Whether to enable Poetry dependency caching via actions/setup-python'
|
||||
description: 'Install Python dependencies with uv'
|
||||
required: false
|
||||
default: 'true'
|
||||
|
||||
@@ -47,67 +39,52 @@ runs:
|
||||
sed -i "s|\(git+https://github.com/prowler-cloud/prowler[^@]*\)@master|\1@$BRANCH_NAME|g" pyproject.toml
|
||||
fi
|
||||
|
||||
- name: Install poetry
|
||||
shell: bash
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pipx install poetry==${INPUTS_POETRY_VERSION}
|
||||
env:
|
||||
INPUTS_POETRY_VERSION: ${{ inputs.poetry-version }}
|
||||
|
||||
- name: Update poetry.lock with latest Prowler commit
|
||||
- name: Update uv.lock with latest Prowler commit
|
||||
if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / {
|
||||
s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/
|
||||
}' poetry.lock
|
||||
echo "Updated resolved_reference:"
|
||||
grep -A2 -B2 "resolved_reference" poetry.lock
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
grep "prowler-cloud/prowler" uv.lock
|
||||
|
||||
- name: Update SDK resolved_reference to latest commit (prowler repo on push)
|
||||
- name: Update uv.lock SDK commit (prowler repo on push)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / {
|
||||
s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/
|
||||
}' poetry.lock
|
||||
echo "Updated resolved_reference:"
|
||||
grep -A2 -B2 "resolved_reference" poetry.lock
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
grep "prowler-cloud/prowler" uv.lock
|
||||
|
||||
- name: Update poetry.lock (prowler repo only)
|
||||
if: github.repository == 'prowler-cloud/prowler' && inputs.update-lock == 'true'
|
||||
- name: Install uv
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: poetry lock
|
||||
env:
|
||||
UV_VERSION: ${{ inputs.uv-version }}
|
||||
run: pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir "uv==${UV_VERSION}"
|
||||
|
||||
- name: Set up Python ${{ inputs.python-version }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
# Disable cache when callers skip dependency install: Poetry 2.3.4 creates
|
||||
# the venv in a path setup-python can't hash, breaking the post-step save-cache.
|
||||
cache: ${{ inputs.enable-cache == 'true' && 'poetry' || '' }}
|
||||
cache-dependency-path: ${{ inputs.working-directory }}/poetry.lock
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Python dependencies
|
||||
if: inputs.install-dependencies == 'true'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry install --no-root
|
||||
poetry run pip list
|
||||
uv sync --no-install-project
|
||||
uv run pip list
|
||||
|
||||
- name: Update Prowler Cloud API Client
|
||||
if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry remove prowler-cloud-api-client
|
||||
poetry add ./prowler-cloud-api-client
|
||||
uv remove prowler-cloud-api-client
|
||||
uv add ./prowler-cloud-api-client
|
||||
@@ -66,6 +66,18 @@ updates:
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "pre-commit"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "pre-commit"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
# - package-ecosystem: "pip"
|
||||
|
||||
@@ -72,6 +72,11 @@ provider/vercel:
|
||||
- any-glob-to-any-file: "prowler/providers/vercel/**"
|
||||
- any-glob-to-any-file: "tests/providers/vercel/**"
|
||||
|
||||
provider/okta:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/okta/**"
|
||||
- any-glob-to-any-file: "tests/providers/okta/**"
|
||||
|
||||
github_actions:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: ".github/workflows/*"
|
||||
@@ -109,6 +114,8 @@ mutelist:
|
||||
- any-glob-to-any-file: "tests/providers/googleworkspace/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/vercel/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/vercel/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/okta/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/okta/lib/mutelist/**"
|
||||
|
||||
integration/s3:
|
||||
- changed-files:
|
||||
|
||||
@@ -36,6 +36,7 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
#### UI
|
||||
- [ ] All issue/task requirements work as expected on the UI
|
||||
- [ ] If this PR adds or updates npm dependencies, include package-health evidence (maintenance, popularity, known vulnerabilities, license, release age) and explain why existing/native alternatives are insufficient.
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
|
||||
@@ -48,7 +49,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Performance test results (if applicable)
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required (e.g., specs, Poetry, etc.).
|
||||
- [ ] Check if version updates are required (e.g., specs, uv, etc.).
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
|
||||
|
||||
### License
|
||||
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run osv-scanner and fail when findings match the configured severity levels.
|
||||
#
|
||||
# Replaces `safety check --policy-file .safety-policy.yml`. Used by:
|
||||
# - .github/actions/osv-scanner/action.yml (composite action)
|
||||
# - .github/workflows/api-security.yml, sdk-security.yml, ui-security.yml
|
||||
#
|
||||
# Severity levels (comma-separated) are read from OSV_SEVERITY_LEVELS.
|
||||
# Default: HIGH,CRITICAL,UNKNOWN — preserves prior .safety-policy.yml policy
|
||||
# (ignore-cvss-severity-below: 7 + ignore-cvss-unknown-severity: False).
|
||||
# osv-scanner has no native CVSS threshold (google/osv-scanner#1400, closed
|
||||
# not-planned). Severity is derived from $group.max_severity (numeric CVSS
|
||||
# score string) which osv-scanner emits per group.
|
||||
#
|
||||
# CVSS v3 score → categorical label:
|
||||
# CRITICAL >= 9.0
|
||||
# HIGH >= 7.0
|
||||
# MEDIUM >= 4.0
|
||||
# LOW > 0.0
|
||||
# UNKNOWN no score available
|
||||
#
|
||||
# Per-vulnerability ignores (with reason + expiry) live in osv-scanner.toml at
|
||||
# the repo root, if it exists. Without that file, osv-scanner uses defaults.
|
||||
#
|
||||
# Usage:
|
||||
# osv-scan.sh [osv-scanner pass-through args...]
|
||||
# Examples:
|
||||
# osv-scan.sh --lockfile=uv.lock
|
||||
# osv-scan.sh --recursive .
|
||||
# OSV_SEVERITY_LEVELS=CRITICAL osv-scan.sh --lockfile=uv.lock
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel)"
|
||||
CONFIG="${ROOT}/osv-scanner.toml"
|
||||
SEVERITY_LEVELS="${OSV_SEVERITY_LEVELS:-HIGH,CRITICAL,UNKNOWN}"
|
||||
|
||||
for bin in osv-scanner jq; do
|
||||
if ! command -v "${bin}" >/dev/null 2>&1; then
|
||||
echo "error: ${bin} not found in PATH" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
SCAN_ARGS=()
|
||||
if [ -f "${CONFIG}" ]; then
|
||||
SCAN_ARGS+=(--config="${CONFIG}")
|
||||
fi
|
||||
|
||||
# Exit codes: 0=clean, 1=findings, 127=no supported files, 128=internal error.
|
||||
STDERR="$(mktemp)"
|
||||
trap 'rm -f "${STDERR}"' EXIT
|
||||
|
||||
set +e
|
||||
OUTPUT="$(osv-scanner scan source "${SCAN_ARGS[@]}" --format=json "$@" 2>"${STDERR}")"
|
||||
RC=$?
|
||||
set -e
|
||||
|
||||
case "${RC}" in
|
||||
0|1) ;;
|
||||
127) echo "osv-scanner: no supported lockfiles in scan target"; exit 0 ;;
|
||||
*)
|
||||
echo "osv-scanner: exited with code ${RC}" >&2
|
||||
[ -s "${STDERR}" ] && cat "${STDERR}" >&2
|
||||
exit "${RC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Build a JSON array of normalized severity levels for jq.
|
||||
SEVERITY_JSON="$(printf '%s' "${SEVERITY_LEVELS}" | jq -Rc '
|
||||
split(",") | map(ascii_upcase | sub("^\\s+"; "") | sub("\\s+$"; ""))
|
||||
')"
|
||||
|
||||
# Walk each vulnerability, look up its group's max_severity (numeric CVSS),
|
||||
# map to a categorical label, then filter by OSV_SEVERITY_LEVELS.
|
||||
FINDINGS="$(printf '%s' "${OUTPUT}" | jq --argjson sevs "${SEVERITY_JSON}" '
|
||||
[ .results[]?.packages[]?
|
||||
| . as $pkg
|
||||
| ($pkg.groups // []) as $groups
|
||||
| ($pkg.vulnerabilities // [])[]
|
||||
| . as $vuln
|
||||
| ([ $groups[] | select((.ids // []) | index($vuln.id)) ][0] // {}) as $group
|
||||
| (($group.max_severity // "") | tonumber? // null) as $score
|
||||
| (if $score == null then "UNKNOWN"
|
||||
elif $score >= 9.0 then "CRITICAL"
|
||||
elif $score >= 7.0 then "HIGH"
|
||||
elif $score >= 4.0 then "MEDIUM"
|
||||
elif $score > 0 then "LOW"
|
||||
else "UNKNOWN"
|
||||
end) as $label
|
||||
| {
|
||||
id: $vuln.id,
|
||||
severity: $label,
|
||||
score: $score,
|
||||
summary: ($vuln.summary // null),
|
||||
package: $pkg.package.name,
|
||||
version: $pkg.package.version,
|
||||
ecosystem: $pkg.package.ecosystem
|
||||
}
|
||||
| select(.severity as $s | $sevs | any(. == $s))
|
||||
]
|
||||
')"
|
||||
|
||||
COUNT="$(printf '%s' "${FINDINGS}" | jq 'length')"
|
||||
|
||||
# Write the findings JSON to OSV_REPORT_FILE so callers (e.g. the composite
|
||||
# action's PR-comment step) can consume the same data the gate decision uses.
|
||||
if [ -n "${OSV_REPORT_FILE:-}" ]; then
|
||||
printf '%s' "${FINDINGS}" > "${OSV_REPORT_FILE}"
|
||||
fi
|
||||
|
||||
if [ "${COUNT}" -gt 0 ]; then
|
||||
echo "osv-scanner: ${COUNT} finding(s) at severity ${SEVERITY_LEVELS}"
|
||||
printf '%s' "${FINDINGS}" | jq -r '
|
||||
.[] | " [\(.severity)\(if .score then " \(.score)" else "" end)] \(.id) \(.ecosystem)/\(.package)@\(.version) — \(.summary // "(no summary)")"
|
||||
'
|
||||
echo
|
||||
echo "To accept a finding, create osv-scanner.toml at the repo root with a reason and ignoreUntil."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "osv-scanner: no findings at severity levels: ${SEVERITY_LEVELS}"
|
||||
@@ -1,291 +0,0 @@
|
||||
name: 'API: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
current_api_version: ${{ steps.get_api_version.outputs.current_api_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get current API version
|
||||
id: get_api_version
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
|
||||
echo "current_api_version=${CURRENT_API_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next API minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
CURRENT_API_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_API_VERSION}"
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# For Prowler 5.17.0 -> API 1.18.0
|
||||
# For next master (Prowler 5.18.0) -> API 1.19.0
|
||||
NEXT_API_VERSION=1.$((MINOR_VERSION + 2)).0
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_VERSION=${NEXT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
echo "Next API minor version (for master): $NEXT_API_VERSION"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_API_VERSION: ${{ needs.detect-release-type.outputs.current_api_version }}
|
||||
|
||||
- name: Bump API versions in files for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next API minor version to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.NEXT_API_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.NEXT_API_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate first API patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
CURRENT_API_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_API_VERSION}"
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# For Prowler 5.17.0 release -> version branch v5.17 should have API 1.18.1
|
||||
FIRST_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).1
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_API_PATCH_VERSION=${FIRST_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
|
||||
echo "First API patch version (for ${VERSION_BRANCH}): $FIRST_API_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_API_VERSION: ${{ needs.detect-release-type.outputs.current_api_version }}
|
||||
|
||||
- name: Bump API versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${FIRST_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first API patch version to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.FIRST_API_PATCH_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.FIRST_API_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next API patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
PATCH_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION}
|
||||
CURRENT_API_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_API_VERSION}"
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# Extract current API patch to increment it
|
||||
if [[ $CURRENT_API_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
API_PATCH=${BASH_REMATCH[3]}
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# Keep same API minor (based on Prowler minor), increment patch
|
||||
NEXT_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).$((API_PATCH + 1))
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_PATCH_VERSION=${NEXT_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
echo "Next API patch version: $NEXT_API_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
else
|
||||
echo "::error::Invalid API version format: $CURRENT_API_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION: ${{ needs.detect-release-type.outputs.patch_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_API_VERSION: ${{ needs.detect-release-type.outputs.current_api_version }}
|
||||
|
||||
- name: Bump API versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next API patch version to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.NEXT_API_PATCH_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.NEXT_API_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -43,6 +43,7 @@ jobs:
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
api.github.com:443
|
||||
raw.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -63,26 +64,25 @@ jobs:
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Poetry check
|
||||
- name: uv lock check
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry check --lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Ruff lint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run ruff check . --exclude contrib
|
||||
run: uv run ruff check . --exclude contrib
|
||||
|
||||
- name: Ruff format
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run ruff format --check . --exclude contrib
|
||||
run: uv run ruff format --check . --exclude contrib
|
||||
|
||||
- name: Pylint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/
|
||||
run: uv run pylint --disable=W,C,R,E -j 0 -rn -sn src/
|
||||
|
||||
@@ -122,6 +122,7 @@ jobs:
|
||||
github.com:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
pypi.org:443
|
||||
registry-1.docker.io:443
|
||||
release-assets.githubusercontent.com:443
|
||||
@@ -132,11 +133,15 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin prowler SDK to latest master commit
|
||||
if: github.event_name == 'push'
|
||||
- name: Refresh prowler SDK pin to current branch tip
|
||||
run: |
|
||||
LATEST_SHA=$(git ls-remote https://github.com/prowler-cloud/prowler.git refs/heads/master | cut -f1)
|
||||
sed -i "s|prowler-cloud/prowler.git@master|prowler-cloud/prowler.git@${LATEST_SHA}|" api/pyproject.toml
|
||||
# api/pyproject.toml has `@master` on master and `@v5.X` on release
|
||||
# branches (set by prepare-release.yml). uv lock --upgrade-package
|
||||
# re-resolves whichever ref is present against the current branch tip
|
||||
# and writes the SHA into api/uv.lock. The Dockerfile runs
|
||||
# `uv sync --locked`, which is what actually drives the install.
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
(cd api && uv lock --upgrade-package prowler)
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
@@ -158,7 +163,7 @@ jobs:
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
@@ -179,6 +184,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
with:
|
||||
|
||||
@@ -5,10 +5,16 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -57,16 +63,7 @@ jobs:
|
||||
|
||||
api-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -86,6 +83,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
debian.map.fastlydns.net:80
|
||||
release-assets.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
@@ -119,23 +117,22 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build container for ${{ matrix.arch }}
|
||||
- name: Build container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: ${{ env.API_WORKING_DIR }}
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
|
||||
- name: Scan container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -5,10 +5,24 @@ on:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-tests.yml'
|
||||
- '.github/workflows/api-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-tests.yml'
|
||||
- '.github/workflows/api-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -25,6 +39,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
@@ -42,10 +57,13 @@ jobs:
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
github.com:443
|
||||
auth.safetycli.com:443
|
||||
pyup.io:443
|
||||
data.safetycli.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
raw.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -60,31 +78,34 @@ jobs:
|
||||
files: |
|
||||
api/**
|
||||
.github/workflows/api-security.yml
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Bandit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r .
|
||||
# Exclude .venv because uv places the project venv inside ./api; otherwise
|
||||
# bandit would recurse into installed third-party packages.
|
||||
run: uv run bandit -q -lll -x '*_test.py,./contrib/,./.venv/' -r .
|
||||
|
||||
- name: Safety
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run safety check --ignore 79023,79027,86217,71600
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
# TODO: 86217 because `alibabacloud-tea-openapi == 0.4.3` don't let us upgrade `cryptography >= 46.0.0`
|
||||
# TODO: 71600 CVE-2024-1135 false positive - fixed in gunicorn 22.0.0, project uses 23.0.0
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: api/uv.lock
|
||||
|
||||
- name: Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 .
|
||||
# Run even when osv-scanner reports findings so dead-code signal isn't masked by SCA failures.
|
||||
if: ${{ !cancelled() && steps.check-changes.outputs.any_changed == 'true' }}
|
||||
run: uv run vulture --exclude "contrib,tests,conftest.py,.venv" --min-confidence 100 .
|
||||
|
||||
@@ -87,6 +87,7 @@ jobs:
|
||||
files.pythonhosted.org:443
|
||||
cli.codecov.io:443
|
||||
keybase.io:443
|
||||
raw.githubusercontent.com:443
|
||||
ingest.codecov.io:443
|
||||
storage.googleapis.com:443
|
||||
o26192.ingest.us.sentry.io:443
|
||||
@@ -112,17 +113,16 @@ jobs:
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Run tests with pytest
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend
|
||||
run: uv run pytest --cov=./src/backend --cov-report=xml src/backend
|
||||
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -35,6 +35,7 @@ jobs:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
|
||||
- name: Check labels
|
||||
id: label_check
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
name: 'Release: Bump Versions'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: release-bump-versions-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
DOCS_FILE: docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-master:
|
||||
name: Bump versions on master (minor release)
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: master
|
||||
persist-credentials: false
|
||||
|
||||
- name: Compute next versions for master
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
|
||||
# SDK / UI / docs mirror the Prowler version directly.
|
||||
NEXT_SDK_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
|
||||
# API is an independent stream: 1.<prowler_minor + 1>.X
|
||||
# After Prowler 5.M.0 release, master moves on to next API minor: 1.(M+2).0
|
||||
NEXT_API_VERSION=1.$((MINOR_VERSION + 2)).0
|
||||
|
||||
# Read current versions to drive sed replacements.
|
||||
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
|
||||
CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' "${DOCS_FILE}")
|
||||
|
||||
echo "NEXT_SDK_VERSION=${NEXT_SDK_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_VERSION=${NEXT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Released Prowler version: $PROWLER_VERSION"
|
||||
echo "Next SDK/UI version (master): $NEXT_SDK_VERSION"
|
||||
echo "Next API version (master): $NEXT_API_VERSION (current: $CURRENT_API_VERSION)"
|
||||
echo "Docs target version: $PROWLER_VERSION (current: $CURRENT_DOCS_VERSION)"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Decide whether to bump docs on master
|
||||
id: docs_decision
|
||||
run: |
|
||||
# Skip docs bump if master is already at or ahead of the release version
|
||||
# (re-run, or patch shipped against an older minor line).
|
||||
HIGHEST=$(printf '%s\n%s\n' "${CURRENT_DOCS_VERSION}" "${PROWLER_VERSION}" | sort -V | tail -n1)
|
||||
if [[ "${CURRENT_DOCS_VERSION}" == "${PROWLER_VERSION}" || "${HIGHEST}" != "${PROWLER_VERSION}" ]]; then
|
||||
echo "skip=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "Skipping docs bump: current ($CURRENT_DOCS_VERSION) >= release ($PROWLER_VERSION)"
|
||||
else
|
||||
echo "skip=false" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Bump SDK version (pyproject.toml, config.py)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_SDK_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_SDK_VERSION}\"|" prowler/config/config.py
|
||||
|
||||
- name: Bump API version (api/pyproject.toml, specs/v1.yaml)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_SDK_VERSION}|" .env
|
||||
|
||||
- name: Bump docs versions (prowler-app.mdx)
|
||||
if: steps.docs_decision.outputs.skip == 'false'
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" "${DOCS_FILE}"
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" "${DOCS_FILE}"
|
||||
|
||||
- name: Show consolidated diff
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for next versions to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(release): Bump versions to v${{ env.NEXT_SDK_VERSION }}'
|
||||
branch: release-version-bump-to-v${{ env.NEXT_SDK_VERSION }}
|
||||
title: 'chore(release): Bump versions to v${{ env.NEXT_SDK_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler versions on master after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
| Area | File(s) | New version |
|
||||
| --- | --- | --- |
|
||||
| SDK | `pyproject.toml`, `prowler/config/config.py` | v${{ env.NEXT_SDK_VERSION }} |
|
||||
| API | `api/pyproject.toml`, `api/src/backend/api/specs/v1.yaml` | v${{ env.NEXT_API_VERSION }} |
|
||||
| UI | `.env` (`NEXT_PUBLIC_PROWLER_RELEASE_VERSION`) | v${{ env.NEXT_SDK_VERSION }} |
|
||||
| Docs | `docs/getting-started/installation/prowler-app.mdx` (`PROWLER_UI_VERSION`, `PROWLER_API_VERSION`) | v${{ env.PROWLER_VERSION }} (skipped if already at or ahead) |
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-minor-version-branch:
|
||||
name: Bump versions on version branch (minor release)
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Compute first patch versions for version branch
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# SDK / UI first patch mirrors Prowler version directly.
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
|
||||
# API on this branch stays on the 1.<MINOR+1>.X stream, starting at .1
|
||||
FIRST_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).1
|
||||
|
||||
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
|
||||
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_API_PATCH_VERSION=${FIRST_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Released Prowler version: $PROWLER_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
echo "First SDK/UI patch: $FIRST_PATCH_VERSION"
|
||||
echo "First API patch: $FIRST_API_PATCH_VERSION (current: $CURRENT_API_VERSION)"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Bump SDK version (pyproject.toml, config.py)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
|
||||
- name: Bump API version (api/pyproject.toml, specs/v1.yaml)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
|
||||
|
||||
- name: Show consolidated diff
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch versions to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(release): Bump versions to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: release-version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump versions to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler versions on `${{ env.VERSION_BRANCH }}` after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
| Area | File(s) | New version |
|
||||
| --- | --- | --- |
|
||||
| SDK | `pyproject.toml`, `prowler/config/config.py` | v${{ env.FIRST_PATCH_VERSION }} |
|
||||
| API | `api/pyproject.toml`, `api/src/backend/api/specs/v1.yaml` | v${{ env.FIRST_API_PATCH_VERSION }} |
|
||||
| UI | `.env` (`NEXT_PUBLIC_PROWLER_RELEASE_VERSION`) | v${{ env.FIRST_PATCH_VERSION }} |
|
||||
| Docs | (not touched on version branches) | — |
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version-branch:
|
||||
name: Bump versions on version branch (patch release)
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Compute next patch versions
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
PATCH_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION}
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# SDK / UI patch mirrors Prowler version directly.
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
|
||||
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
|
||||
|
||||
# API on this branch stays on 1.<MINOR+1>.X; bump its patch component.
|
||||
if [[ $CURRENT_API_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
API_PATCH=${BASH_REMATCH[3]}
|
||||
NEXT_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).$((API_PATCH + 1))
|
||||
else
|
||||
echo "::error::Invalid API version format: $CURRENT_API_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_PATCH_VERSION=${NEXT_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Released Prowler version: $PROWLER_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
echo "Next SDK/UI patch: $NEXT_PATCH_VERSION"
|
||||
echo "Next API patch: $NEXT_API_PATCH_VERSION (current: $CURRENT_API_VERSION)"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION: ${{ needs.detect-release-type.outputs.patch_version }}
|
||||
|
||||
- name: Bump SDK version (pyproject.toml, config.py)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
|
||||
- name: Bump API version (api/pyproject.toml, specs/v1.yaml)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
|
||||
|
||||
- name: Show consolidated diff
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch versions to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(release): Bump versions to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: release-version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump versions to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler versions on `${{ env.VERSION_BRANCH }}` after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
| Area | File(s) | New version |
|
||||
| --- | --- | --- |
|
||||
| SDK | `pyproject.toml`, `prowler/config/config.py` | v${{ env.NEXT_PATCH_VERSION }} |
|
||||
| API | `api/pyproject.toml`, `api/src/backend/api/specs/v1.yaml` | v${{ env.NEXT_API_PATCH_VERSION }} |
|
||||
| UI | `.env` (`NEXT_PUBLIC_PROWLER_RELEASE_VERSION`) | v${{ env.NEXT_PATCH_VERSION }} |
|
||||
| Docs | (not touched on version branches) | — |
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -4,8 +4,6 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v3'
|
||||
- 'v4.*'
|
||||
- 'v5.*'
|
||||
types:
|
||||
- 'opened'
|
||||
|
||||
@@ -43,14 +43,11 @@ jobs:
|
||||
|
||||
echo "Processing release tag: $RELEASE_TAG"
|
||||
|
||||
# Remove 'v' prefix if present (e.g., v3.2.0 -> 3.2.0)
|
||||
VERSION_ONLY="${RELEASE_TAG#v}"
|
||||
|
||||
# Check if it's a minor version (X.Y.0)
|
||||
if [[ "$VERSION_ONLY" =~ ^([0-9]+)\.([0-9]+)\.0$ ]]; then
|
||||
echo "Release $RELEASE_TAG (version $VERSION_ONLY) is a minor version. Proceeding to create backport label."
|
||||
|
||||
# Extract X.Y from X.Y.0 (e.g., 5.6 from 5.6.0)
|
||||
MAJOR="${BASH_REMATCH[1]}"
|
||||
MINOR="${BASH_REMATCH[2]}"
|
||||
TWO_DIGIT_VERSION="${MAJOR}.${MINOR}"
|
||||
@@ -62,7 +59,6 @@ jobs:
|
||||
echo "Label name: $LABEL_NAME"
|
||||
echo "Label description: $LABEL_DESC"
|
||||
|
||||
# Check if label already exists
|
||||
if gh label list --repo ${{ github.repository }} --limit 1000 | grep -q "^${LABEL_NAME}[[:space:]]"; then
|
||||
echo "Label '$LABEL_NAME' already exists."
|
||||
else
|
||||
|
||||
@@ -1,324 +0,0 @@
|
||||
name: 'Docs: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
current_docs_version: ${{ steps.get_docs_version.outputs.current_docs_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get current documentation version
|
||||
id: get_docs_version
|
||||
run: |
|
||||
CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' docs/getting-started/installation/prowler-app.mdx)
|
||||
echo "current_docs_version=${CURRENT_DOCS_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
CURRENT_DOCS_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION}"
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
echo "Current release version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION: ${{ needs.detect-release-type.outputs.current_docs_version }}
|
||||
|
||||
- name: Bump versions in documentation for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.BASE_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
- All `*.mdx` files with `<VersionBadge>` components
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
CURRENT_DOCS_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION}"
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION: ${{ needs.detect-release-type.outputs.current_docs_version }}
|
||||
|
||||
- name: Bump versions in documentation for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
PATCH_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION}
|
||||
CURRENT_DOCS_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION}"
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
echo "Current release version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION: ${{ needs.detect-release-type.outputs.patch_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION: ${{ needs.detect-release-type.outputs.current_docs_version }}
|
||||
|
||||
- name: Checkout master branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ env.BASE_BRANCH }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Bump versions in documentation for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.BASE_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ env.VERSION_BRANCH }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Bump versions in documentation for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -27,19 +27,23 @@ jobs:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
# We can't block as Trufflehog needs to verify secrets against vendors
|
||||
egress-policy: audit
|
||||
# allowed-endpoints: >
|
||||
# github.com:443
|
||||
# ghcr.io:443
|
||||
# pkg-containers.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# PRs only need the diff range; push to master/release walks the new range from event.before.
|
||||
# 50 is enough headroom for the longest realistic PR/push chain without paying for a full clone.
|
||||
fetch-depth: 50
|
||||
persist-credentials: false
|
||||
|
||||
- name: Scan for secrets with TruffleHog
|
||||
- name: Scan diff for secrets with TruffleHog
|
||||
# Action auto-injects --since-commit/--branch from event payload; passing them in extra_args produces duplicate flags.
|
||||
uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4
|
||||
with:
|
||||
extra_args: '--results=verified,unknown'
|
||||
extra_args: --results=verified,unknown
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
"Alan-TheGentleman"
|
||||
"alejandrobailo"
|
||||
"amitsharm"
|
||||
"andoniaf"
|
||||
# "andoniaf"
|
||||
"cesararroba"
|
||||
"danibarranqueroo"
|
||||
"HugoPBrito"
|
||||
|
||||
@@ -114,6 +114,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
files.pythonhosted.org:443
|
||||
@@ -152,7 +153,7 @@ jobs:
|
||||
org.opencontainers.image.created=${{ github.event_name == 'release' && github.event.release.published_at || github.event.head_commit.timestamp }}
|
||||
${{ github.event_name == 'release' && format('org.opencontainers.image.version={0}', env.RELEASE_TAG) || '' }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
@@ -171,6 +172,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
|
||||
|
||||
@@ -5,10 +5,16 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/**'
|
||||
- '.github/workflows/mcp-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/**'
|
||||
- '.github/workflows/mcp-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -56,16 +62,7 @@ jobs:
|
||||
|
||||
mcp-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -82,6 +79,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
files.pythonhosted.org:443
|
||||
@@ -112,23 +110,22 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build MCP container for ${{ matrix.arch }}
|
||||
- name: Build MCP container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: ${{ env.MCP_WORKING_DIR }}
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
|
||||
- name: Scan MCP container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan MCP container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -86,11 +86,33 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
# The MCP server version (mcp_server/pyproject.toml) is decoupled from the Prowler release
|
||||
# version: it only changes when MCP code changes. mcp-bump-version.yml normally keeps it in
|
||||
# sync with mcp_server/CHANGELOG.md (separate from the release bump-version.yml), but this
|
||||
# publish workflow still runs on every release.
|
||||
# Pre-flight PyPI check covers the legitimate "no MCP changes for this release" case (and any
|
||||
# workflow_dispatch re-runs) without failing with HTTP 400 (version exists).
|
||||
- name: Check if prowler-mcp version already exists on PyPI
|
||||
id: pypi-check
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: |
|
||||
MCP_VERSION=$(grep '^version' pyproject.toml | head -1 | sed -E 's/^version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/')
|
||||
echo "mcp_version=${MCP_VERSION}" >> "$GITHUB_OUTPUT"
|
||||
if curl -fsS "https://pypi.org/pypi/prowler-mcp/${MCP_VERSION}/json" >/dev/null 2>&1; then
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice title=Skipping prowler-mcp publish::Version ${MCP_VERSION} already exists on PyPI; bump mcp_server/pyproject.toml to publish a new release."
|
||||
else
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice title=Publishing prowler-mcp::Version ${MCP_VERSION} not on PyPI yet; proceeding."
|
||||
fi
|
||||
|
||||
- name: Build prowler-mcp package
|
||||
if: steps.pypi-check.outputs.skip != 'true'
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-mcp package to PyPI
|
||||
if: steps.pypi-check.outputs.skip != 'true'
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
with:
|
||||
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
name: 'Nightly: ARM64 Container Builds'
|
||||
|
||||
# Mitigation for amd64-only PR container-checks: build amd64+arm64 nightly against
|
||||
# master to keep arm-specific Dockerfile regressions caught quickly. Build only —
|
||||
# no push, no Trivy (weekly checks already cover that).
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 4 * * *'
|
||||
workflow_dispatch: {}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build-arm64:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- component: sdk
|
||||
context: .
|
||||
dockerfile: ./Dockerfile
|
||||
image_name: prowler
|
||||
- component: api
|
||||
context: ./api
|
||||
dockerfile: ./api/Dockerfile
|
||||
image_name: prowler-api
|
||||
- component: ui
|
||||
context: ./ui
|
||||
dockerfile: ./ui/Dockerfile
|
||||
image_name: prowler-ui
|
||||
target: prod
|
||||
build_args: |
|
||||
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
|
||||
- component: mcp
|
||||
context: ./mcp_server
|
||||
dockerfile: ./mcp_server/Dockerfile
|
||||
image_name: prowler-mcp
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build ${{ matrix.component }} container (linux/arm64)
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: ${{ matrix.context }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
target: ${{ matrix.target }}
|
||||
push: false
|
||||
load: false
|
||||
platforms: linux/arm64
|
||||
tags: ${{ matrix.image_name }}:nightly-arm64
|
||||
build-args: ${{ matrix.build_args }}
|
||||
cache-from: type=gha,scope=arm64
|
||||
cache-to: type=gha,mode=min,scope=arm64
|
||||
|
||||
notify-failure:
|
||||
needs: build-arm64
|
||||
if: failure() && github.event_name == 'schedule'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Notify Slack on failure
|
||||
uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
|
||||
with:
|
||||
method: chat.postMessage
|
||||
token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload: |
|
||||
channel: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
text: ":rotating_light: Nightly arm64 container build failed for prowler — <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|view run>"
|
||||
errors: true
|
||||
@@ -41,10 +41,15 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
@@ -54,7 +59,7 @@ jobs:
|
||||
ui/**
|
||||
prowler/**
|
||||
mcp_server/**
|
||||
poetry.lock
|
||||
uv.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
@@ -79,9 +84,9 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (poetry.lock, pyproject.toml)
|
||||
# Check root-level dependency files (uv.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(poetry\.lock|pyproject\.toml)$" || true)
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
|
||||
@@ -20,7 +20,13 @@ permissions: {}
|
||||
|
||||
jobs:
|
||||
check-compliance-mapping:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'no-compliance-check') == false
|
||||
if: >-
|
||||
github.event.pull_request.state == 'open' &&
|
||||
contains(github.event.pull_request.labels.*.name, 'no-compliance-check') == false &&
|
||||
(
|
||||
(github.event.action != 'labeled' && github.event.action != 'unlabeled')
|
||||
|| github.event.label.name == 'no-compliance-check'
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
@@ -39,10 +45,15 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
|
||||
@@ -36,8 +36,14 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
|
||||
@@ -40,12 +40,11 @@ jobs:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Setup Python with uv
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: '3.12'
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
@@ -54,7 +53,7 @@ jobs:
|
||||
|
||||
- name: Parse version and determine branch
|
||||
run: |
|
||||
# Validate version format (reusing pattern from sdk-bump-version.yml)
|
||||
# Validate version format (reusing pattern from bump-version.yml)
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
@@ -300,17 +299,6 @@ jobs:
|
||||
fi
|
||||
echo "✓ api/pyproject.toml prowler dependency: $CURRENT_PROWLER_REF"
|
||||
|
||||
- name: Verify API version in api/src/backend/api/v1/views.py
|
||||
if: ${{ env.HAS_API_CHANGES == 'true' }}
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep 'spectacular_settings.VERSION = ' api/src/backend/api/v1/views.py | sed -E 's/.*spectacular_settings.VERSION = "([^"]+)".*/\1/' | tr -d '[:space:]')
|
||||
API_VERSION_TRIMMED=$(echo "$API_VERSION" | tr -d '[:space:]')
|
||||
if [ "$CURRENT_API_VERSION" != "$API_VERSION_TRIMMED" ]; then
|
||||
echo "ERROR: API version mismatch in views.py (expected: '$API_VERSION_TRIMMED', found: '$CURRENT_API_VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Verify API version in api/src/backend/api/specs/v1.yaml
|
||||
if: ${{ env.HAS_API_CHANGES == 'true' }}
|
||||
run: |
|
||||
@@ -339,10 +327,11 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Update poetry lock file
|
||||
echo "Updating poetry.lock file..."
|
||||
# Update uv lock file
|
||||
echo "Updating uv.lock file..."
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
cd api
|
||||
poetry lock
|
||||
uv lock
|
||||
cd ..
|
||||
|
||||
echo "✓ Prepared prowler dependency update to: $UPDATED_PROWLER_REF"
|
||||
@@ -357,7 +346,7 @@ jobs:
|
||||
base: ${{ env.BRANCH_NAME }}
|
||||
add-paths: |
|
||||
api/pyproject.toml
|
||||
api/poetry.lock
|
||||
api/uv.lock
|
||||
title: "chore(api): Update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}"
|
||||
body: |
|
||||
### Description
|
||||
@@ -366,7 +355,7 @@ jobs:
|
||||
|
||||
**Changes:**
|
||||
- Updates `api/pyproject.toml` prowler dependency from `@master` to `@${{ env.BRANCH_NAME }}`
|
||||
- Updates `api/poetry.lock` file with resolved dependencies
|
||||
- Updates `api/uv.lock` file with resolved dependencies
|
||||
|
||||
This PR should be merged into the `${{ env.BRANCH_NAME }}` release branch.
|
||||
|
||||
|
||||
@@ -1,247 +0,0 @@
|
||||
name: 'SDK: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Bump versions in files for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next minor version to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler version to v${{ env.NEXT_MINOR_VERSION }} after releasing v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Bump versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch version to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
PATCH_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION}
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION: ${{ needs.detect-release-type.outputs.patch_version }}
|
||||
|
||||
- name: Bump versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch version to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler version to v${{ env.NEXT_PATCH_VERSION }} after releasing v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -5,6 +5,9 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'tests/providers/**/*_test.py'
|
||||
- '.github/workflows/sdk-check-duplicate-test-names.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
@@ -71,24 +71,26 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Check Poetry lock file
|
||||
- name: Check uv lock file
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry check --lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Lint with flake8
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api,skills
|
||||
run: uv run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude .venv,contrib,ui,api,skills,mcp_server
|
||||
|
||||
- name: Check format with black
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run black --exclude "api|ui|skills" --check .
|
||||
# mcp_server has its own pyproject and uses ruff format, exclude it so SDK black
|
||||
# does not fight ruff over rules it never formatted.
|
||||
run: uv run black --exclude "\.venv|api|ui|skills|mcp_server" --check .
|
||||
|
||||
- name: Lint with pylint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
|
||||
run: uv run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
|
||||
|
||||
@@ -3,9 +3,7 @@ name: 'SDK: Container Build and Push'
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'v3' # For v3-latest
|
||||
- 'v4.6' # For v4-latest
|
||||
- 'master' # For latest
|
||||
- 'master'
|
||||
paths-ignore:
|
||||
- '.github/**'
|
||||
- '!.github/workflows/sdk-container-build-push.yml'
|
||||
@@ -56,7 +54,6 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }}
|
||||
prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }}
|
||||
latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }}
|
||||
stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }}
|
||||
permissions:
|
||||
@@ -76,48 +73,19 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Inject poetry-bumpversion plugin
|
||||
run: pipx inject poetry poetry-bumpversion
|
||||
|
||||
- name: Get Prowler version and set tags
|
||||
id: get-prowler-version
|
||||
run: |
|
||||
PROWLER_VERSION="$(poetry version -s 2>/dev/null)"
|
||||
PROWLER_VERSION="$(grep -E '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]')"
|
||||
echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Extract major version
|
||||
PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}"
|
||||
echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Set version-specific tags
|
||||
case ${PROWLER_VERSION_MAJOR} in
|
||||
3)
|
||||
echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable"
|
||||
;;
|
||||
4)
|
||||
echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable"
|
||||
;;
|
||||
5)
|
||||
echo "latest_tag=latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v5 detected - tags: latest, stable"
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [[ "${PROWLER_VERSION_MAJOR}" != "5" ]]; then
|
||||
echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}"
|
||||
exit 1
|
||||
fi
|
||||
echo "latest_tag=latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=stable" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
notify-release-started:
|
||||
if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
@@ -181,6 +149,7 @@ jobs:
|
||||
public.ecr.aws:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
auth.docker.io:443
|
||||
debian.map.fastlydns.net:80
|
||||
github.com:443
|
||||
@@ -228,7 +197,7 @@ jobs:
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
@@ -248,6 +217,7 @@ jobs:
|
||||
auth.docker.io:443
|
||||
public.ecr.aws:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
@@ -386,39 +356,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
dispatch-v3-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && needs.setup.outputs.prowler_version_major == '3' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Calculate short SHA
|
||||
id: short-sha
|
||||
run: echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Dispatch v3 deployment (latest)
|
||||
if: github.event_name == 'push'
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
event-type: dispatch
|
||||
client-payload: '{"version":"v3-latest","tag":"${{ steps.short-sha.outputs.short_sha }}"}'
|
||||
|
||||
- name: Dispatch v3 deployment (release)
|
||||
if: github.event_name == 'release'
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
event-type: dispatch
|
||||
client-payload: '{"version":"release","tag":"${{ needs.setup.outputs.prowler_version }}"}'
|
||||
|
||||
@@ -5,10 +5,22 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'Dockerfile*'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'Dockerfile*'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -56,16 +68,7 @@ jobs:
|
||||
|
||||
sdk-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -82,6 +85,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
api.github.com:443
|
||||
mirror.gcr.io:443
|
||||
check.trivy.dev:443
|
||||
@@ -132,23 +136,22 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build SDK container for ${{ matrix.arch }}
|
||||
- name: Build SDK container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
|
||||
- name: Scan SDK container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan SDK container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -75,15 +75,14 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Setup Python with uv
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Build Prowler package
|
||||
run: poetry build
|
||||
run: uv build
|
||||
|
||||
- name: Publish Prowler package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
@@ -112,12 +111,11 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Setup Python with uv
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
enable-cache: 'false'
|
||||
|
||||
- name: Install toml package
|
||||
run: pip install toml
|
||||
@@ -128,7 +126,7 @@ jobs:
|
||||
python util/replicate_pypi_package.py
|
||||
|
||||
- name: Build prowler-cloud package
|
||||
run: poetry build
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-cloud package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
|
||||
@@ -5,10 +5,30 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'tests/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-tests.yml'
|
||||
- '.github/workflows/sdk-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'tests/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-tests.yml'
|
||||
- '.github/workflows/sdk-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -23,6 +43,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -33,10 +54,12 @@ jobs:
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
github.com:443
|
||||
auth.safetycli.com:443
|
||||
pyup.io:443
|
||||
data.safetycli.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -71,20 +94,23 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Security scan with Bandit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run bandit -q -lll -x '*_test.py,./contrib/,./api/,./ui' -r .
|
||||
run: uv run bandit -q -lll -x '*_test.py,./.venv/,./contrib/,./api/,./ui' -r .
|
||||
|
||||
- name: Security scan with Safety
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run safety check -r pyproject.toml
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: uv.lock
|
||||
|
||||
- name: Dead code detection with Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run vulture --exclude "contrib,api,ui" --min-confidence 100 .
|
||||
# Run even when osv-scanner reports findings so dead-code signal isn't masked by SCA failures.
|
||||
if: ${{ !cancelled() && steps.check-changes.outputs.any_changed == 'true' }}
|
||||
run: uv run vulture --exclude ".venv,contrib,api,ui" --min-confidence 100 .
|
||||
|
||||
@@ -46,6 +46,7 @@ jobs:
|
||||
schema.ocsf.io:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
o26192.ingest.us.sentry.io:443
|
||||
management.azure.com:443
|
||||
@@ -92,9 +93,9 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Setup Python with uv
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
uses: ./.github/actions/setup-python-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
@@ -107,7 +108,7 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/aws/**
|
||||
./tests/**/aws/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Resolve AWS services under test
|
||||
if: steps.changed-aws.outputs.any_changed == 'true'
|
||||
@@ -209,11 +210,11 @@ jobs:
|
||||
echo "AWS service_paths='${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}'"
|
||||
|
||||
if [ "${STEPS_AWS_SERVICES_OUTPUTS_RUN_ALL}" = "true" ]; then
|
||||
poetry run pytest -p no:randomly -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws
|
||||
uv run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws
|
||||
elif [ -z "${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}" ]; then
|
||||
echo "No AWS service paths detected; skipping AWS tests."
|
||||
else
|
||||
poetry run pytest -p no:randomly -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml ${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}
|
||||
uv run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml ${STEPS_AWS_SERVICES_OUTPUTS_SERVICE_PATHS}
|
||||
fi
|
||||
env:
|
||||
STEPS_AWS_SERVICES_OUTPUTS_RUN_ALL: ${{ steps.aws-services.outputs.run_all }}
|
||||
@@ -237,11 +238,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/azure/**
|
||||
./tests/**/azure/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Azure tests
|
||||
if: steps.changed-azure.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure
|
||||
|
||||
- name: Upload Azure coverage to Codecov
|
||||
if: steps.changed-azure.outputs.any_changed == 'true'
|
||||
@@ -261,11 +262,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/gcp/**
|
||||
./tests/**/gcp/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run GCP tests
|
||||
if: steps.changed-gcp.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp
|
||||
|
||||
- name: Upload GCP coverage to Codecov
|
||||
if: steps.changed-gcp.outputs.any_changed == 'true'
|
||||
@@ -285,11 +286,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/kubernetes/**
|
||||
./tests/**/kubernetes/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Kubernetes tests
|
||||
if: steps.changed-kubernetes.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes
|
||||
|
||||
- name: Upload Kubernetes coverage to Codecov
|
||||
if: steps.changed-kubernetes.outputs.any_changed == 'true'
|
||||
@@ -309,11 +310,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/github/**
|
||||
./tests/**/github/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run GitHub tests
|
||||
if: steps.changed-github.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github
|
||||
|
||||
- name: Upload GitHub coverage to Codecov
|
||||
if: steps.changed-github.outputs.any_changed == 'true'
|
||||
@@ -324,6 +325,30 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-github
|
||||
files: ./github_coverage.xml
|
||||
|
||||
# Okta Provider
|
||||
- name: Check if Okta files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-okta
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/okta/**
|
||||
./tests/**/okta/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run Okta tests
|
||||
if: steps.changed-okta.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/okta --cov-report=xml:okta_coverage.xml tests/providers/okta
|
||||
|
||||
- name: Upload Okta coverage to Codecov
|
||||
if: steps.changed-okta.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-okta
|
||||
files: ./okta_coverage.xml
|
||||
|
||||
# NHN Provider
|
||||
- name: Check if NHN files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -333,11 +358,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/nhn/**
|
||||
./tests/**/nhn/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run NHN tests
|
||||
if: steps.changed-nhn.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn
|
||||
|
||||
- name: Upload NHN coverage to Codecov
|
||||
if: steps.changed-nhn.outputs.any_changed == 'true'
|
||||
@@ -357,11 +382,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/m365/**
|
||||
./tests/**/m365/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run M365 tests
|
||||
if: steps.changed-m365.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365
|
||||
|
||||
- name: Upload M365 coverage to Codecov
|
||||
if: steps.changed-m365.outputs.any_changed == 'true'
|
||||
@@ -381,11 +406,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/iac/**
|
||||
./tests/**/iac/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run IaC tests
|
||||
if: steps.changed-iac.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac
|
||||
|
||||
- name: Upload IaC coverage to Codecov
|
||||
if: steps.changed-iac.outputs.any_changed == 'true'
|
||||
@@ -405,11 +430,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/mongodbatlas/**
|
||||
./tests/**/mongodbatlas/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run MongoDB Atlas tests
|
||||
if: steps.changed-mongodbatlas.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodbatlas_coverage.xml tests/providers/mongodbatlas
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodbatlas_coverage.xml tests/providers/mongodbatlas
|
||||
|
||||
- name: Upload MongoDB Atlas coverage to Codecov
|
||||
if: steps.changed-mongodbatlas.outputs.any_changed == 'true'
|
||||
@@ -429,11 +454,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/oraclecloud/**
|
||||
./tests/**/oraclecloud/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run OCI tests
|
||||
if: steps.changed-oraclecloud.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oraclecloud_coverage.xml tests/providers/oraclecloud
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oraclecloud_coverage.xml tests/providers/oraclecloud
|
||||
|
||||
- name: Upload OCI coverage to Codecov
|
||||
if: steps.changed-oraclecloud.outputs.any_changed == 'true'
|
||||
@@ -453,11 +478,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/openstack/**
|
||||
./tests/**/openstack/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run OpenStack tests
|
||||
if: steps.changed-openstack.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/openstack --cov-report=xml:openstack_coverage.xml tests/providers/openstack
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/openstack --cov-report=xml:openstack_coverage.xml tests/providers/openstack
|
||||
|
||||
- name: Upload OpenStack coverage to Codecov
|
||||
if: steps.changed-openstack.outputs.any_changed == 'true'
|
||||
@@ -477,11 +502,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/googleworkspace/**
|
||||
./tests/**/googleworkspace/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Google Workspace tests
|
||||
if: steps.changed-googleworkspace.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/googleworkspace --cov-report=xml:googleworkspace_coverage.xml tests/providers/googleworkspace
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/googleworkspace --cov-report=xml:googleworkspace_coverage.xml tests/providers/googleworkspace
|
||||
|
||||
- name: Upload Google Workspace coverage to Codecov
|
||||
if: steps.changed-googleworkspace.outputs.any_changed == 'true'
|
||||
@@ -501,11 +526,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/**/vercel/**
|
||||
./tests/**/vercel/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Vercel tests
|
||||
if: steps.changed-vercel.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/vercel --cov-report=xml:vercel_coverage.xml tests/providers/vercel
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/vercel --cov-report=xml:vercel_coverage.xml tests/providers/vercel
|
||||
|
||||
- name: Upload Vercel coverage to Codecov
|
||||
if: steps.changed-vercel.outputs.any_changed == 'true'
|
||||
@@ -525,11 +550,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/lib/**
|
||||
./tests/lib/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Lib tests
|
||||
if: steps.changed-lib.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib
|
||||
run: uv run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib
|
||||
|
||||
- name: Upload Lib coverage to Codecov
|
||||
if: steps.changed-lib.outputs.any_changed == 'true'
|
||||
@@ -549,11 +574,11 @@ jobs:
|
||||
files: |
|
||||
./prowler/config/**
|
||||
./tests/config/**
|
||||
./poetry.lock
|
||||
./uv.lock
|
||||
|
||||
- name: Run Config tests
|
||||
if: steps.changed-config.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config
|
||||
run: uv run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config
|
||||
|
||||
- name: Upload Config coverage to Codecov
|
||||
if: steps.changed-config.outputs.any_changed == 'true'
|
||||
|
||||
@@ -1,253 +0,0 @@
|
||||
name: 'UI: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Bump UI version in .env for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next minor version to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.NEXT_MINOR_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Bump UI version in .env for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch version to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION}
|
||||
MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION}
|
||||
PATCH_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION}
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
env:
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }}
|
||||
NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION: ${{ needs.detect-release-type.outputs.patch_version }}
|
||||
|
||||
- name: Bump UI version in .env for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch version to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.NEXT_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -116,6 +116,7 @@ jobs:
|
||||
allowed-endpoints: >
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
auth.docker.io:443
|
||||
registry.npmjs.org:443
|
||||
dl-cdn.alpinelinux.org:443
|
||||
@@ -151,7 +152,7 @@ jobs:
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }},scope=${{ matrix.arch }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
@@ -172,6 +173,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
|
||||
@@ -5,10 +5,16 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/**'
|
||||
- '.github/workflows/ui-container-checks.yml'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/**'
|
||||
- '.github/workflows/ui-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -57,16 +63,7 @@ jobs:
|
||||
|
||||
ui-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -83,6 +80,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
registry.npmjs.org:443
|
||||
dl-cdn.alpinelinux.org:443
|
||||
fonts.googleapis.com:443
|
||||
@@ -114,7 +112,7 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build UI container for ${{ matrix.arch }}
|
||||
- name: Build UI container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
@@ -122,18 +120,17 @@ jobs:
|
||||
target: prod
|
||||
push: false
|
||||
load: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=${{ github.event_name == 'pull_request' && 'min' || 'max' }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
|
||||
|
||||
- name: Scan UI container with Trivy for ${{ matrix.arch }}
|
||||
- name: Scan UI container with Trivy
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -15,6 +15,10 @@ on:
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
@@ -126,6 +130,12 @@ jobs:
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
|
||||
- name: Build API image from current code
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
# which lags behind PR changes; build locally so E2E exercises the API image
|
||||
# produced by this PR.
|
||||
run: docker build -t prowlercloud/prowler-api:latest ./api
|
||||
|
||||
- name: Start API services
|
||||
run: |
|
||||
export PROWLER_API_VERSION=latest
|
||||
@@ -154,7 +164,7 @@ jobs:
|
||||
for fixture in api/fixtures/dev/*.json; do
|
||||
if [ -f "$fixture" ]; then
|
||||
echo "Loading $fixture"
|
||||
poetry run python manage.py loaddata "$fixture" --database admin
|
||||
uv run python manage.py loaddata "$fixture" --database admin
|
||||
fi
|
||||
done
|
||||
'
|
||||
@@ -266,7 +276,7 @@ jobs:
|
||||
with:
|
||||
name: playwright-report
|
||||
path: ui/playwright-report/
|
||||
retention-days: 30
|
||||
retention-days: 7
|
||||
|
||||
- name: Cleanup services
|
||||
if: always()
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
name: 'UI: Security'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/package.json'
|
||||
- 'ui/pnpm-lock.yaml'
|
||||
- '.github/workflows/ui-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/package.json'
|
||||
- 'ui/pnpm-lock.yaml'
|
||||
- '.github/workflows/ui-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Check for UI dependency changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
ui/package.json
|
||||
ui/pnpm-lock.yaml
|
||||
.github/workflows/ui-security.yml
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: ui/pnpm-lock.yaml
|
||||
@@ -1,14 +1,14 @@
|
||||
name: 'UI: Tests'
|
||||
name: "UI: Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
- "master"
|
||||
- "v5.*"
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
- "master"
|
||||
- "v5.*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -16,7 +16,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: '24.13.0'
|
||||
NODE_VERSION: "24.13.0"
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -42,6 +42,9 @@ jobs:
|
||||
fonts.gstatic.com:443
|
||||
api.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
cdn.playwright.dev:443
|
||||
objects.githubusercontent.com:443
|
||||
playwright.download.prss.microsoft.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -129,11 +132,15 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run healthcheck
|
||||
|
||||
- name: Run pnpm audit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run audit
|
||||
|
||||
- name: Run unit tests (all - critical paths changed)
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.critical-changes.outputs.any_changed == 'true'
|
||||
run: |
|
||||
echo "Critical paths changed - running ALL unit tests"
|
||||
pnpm run test:run
|
||||
pnpm run test:unit
|
||||
|
||||
- name: Run unit tests (related to changes only)
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.critical-changes.outputs.any_changed != 'true' && steps.changed-source.outputs.all_changed_files != ''
|
||||
@@ -142,7 +149,7 @@ jobs:
|
||||
echo "${STEPS_CHANGED_SOURCE_OUTPUTS_ALL_CHANGED_FILES}"
|
||||
# Convert space-separated to vitest related format (remove ui/ prefix for relative paths)
|
||||
CHANGED_FILES=$(echo "${STEPS_CHANGED_SOURCE_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | sed 's|^ui/||' | tr '\n' ' ')
|
||||
pnpm exec vitest related $CHANGED_FILES --run
|
||||
pnpm exec vitest related $CHANGED_FILES --run --project unit
|
||||
env:
|
||||
STEPS_CHANGED_SOURCE_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-source.outputs.all_changed_files }}
|
||||
|
||||
@@ -150,7 +157,25 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.critical-changes.outputs.any_changed != 'true' && steps.changed-source.outputs.all_changed_files == ''
|
||||
run: |
|
||||
echo "Only test files changed - running ALL unit tests"
|
||||
pnpm run test:run
|
||||
pnpm run test:unit
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: playwright-cache
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-playwright-chromium-
|
||||
|
||||
- name: Install Playwright Chromium browser
|
||||
if: steps.check-changes.outputs.any_changed == 'true' && steps.playwright-cache.outputs.cache-hit != 'true'
|
||||
run: pnpm exec playwright install chromium
|
||||
|
||||
- name: Run browser tests
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run test:browser
|
||||
|
||||
- name: Build application
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
+2
-4
@@ -1,21 +1,19 @@
|
||||
rules:
|
||||
secrets-outside-env:
|
||||
ignore:
|
||||
- api-bump-version.yml
|
||||
- api-container-build-push.yml
|
||||
- api-tests.yml
|
||||
- backport.yml
|
||||
- docs-bump-version.yml
|
||||
- bump-version.yml
|
||||
- issue-triage.lock.yml
|
||||
- mcp-container-build-push.yml
|
||||
- nightly-arm64-container-builds.yml
|
||||
- pr-merged.yml
|
||||
- prepare-release.yml
|
||||
- sdk-bump-version.yml
|
||||
- sdk-container-build-push.yml
|
||||
- sdk-refresh-aws-services-regions.yml
|
||||
- sdk-refresh-oci-regions.yml
|
||||
- sdk-tests.yml
|
||||
- ui-bump-version.yml
|
||||
- ui-container-build-push.yml
|
||||
- ui-e2e-tests-v2.yml
|
||||
superfluous-actions:
|
||||
|
||||
@@ -151,6 +151,8 @@ node_modules
|
||||
|
||||
# Persistent data
|
||||
_data/
|
||||
/openspec/
|
||||
/.gitmodules
|
||||
|
||||
# AI Instructions (generated by skills/setup.sh from AGENTS.md)
|
||||
CLAUDE.md
|
||||
|
||||
+93
-67
@@ -1,149 +1,175 @@
|
||||
# Priority tiers (lower = runs first, same priority = concurrent):
|
||||
# P0 — fast file fixers
|
||||
# P10 — validators and guards
|
||||
# P20 — auto-formatters
|
||||
# P30 — linters
|
||||
# P40 — security scanners
|
||||
# P50 — dependency validation
|
||||
|
||||
default_install_hook_types: [pre-commit]
|
||||
|
||||
repos:
|
||||
## GENERAL
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.6.0
|
||||
## GENERAL (prek built-in — no external repo needed)
|
||||
- repo: builtin
|
||||
hooks:
|
||||
- id: check-merge-conflict
|
||||
priority: 10
|
||||
- id: check-yaml
|
||||
args: ["--unsafe"]
|
||||
exclude: prowler/config/llm_config.yaml
|
||||
args: ["--allow-multiple-documents"]
|
||||
exclude: (prowler/config/llm_config.yaml|contrib/)
|
||||
priority: 10
|
||||
- id: check-json
|
||||
priority: 10
|
||||
- id: end-of-file-fixer
|
||||
priority: 0
|
||||
- id: trailing-whitespace
|
||||
priority: 0
|
||||
- id: no-commit-to-branch
|
||||
priority: 10
|
||||
- id: pretty-format-json
|
||||
args: ["--autofix", --no-sort-keys, --no-ensure-ascii]
|
||||
priority: 10
|
||||
|
||||
## TOML
|
||||
- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
|
||||
rev: v2.13.0
|
||||
rev: v2.16.0
|
||||
hooks:
|
||||
- id: pretty-format-toml
|
||||
args: [--autofix]
|
||||
files: pyproject.toml
|
||||
priority: 20
|
||||
|
||||
## GITHUB ACTIONS
|
||||
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
||||
rev: v1.6.0
|
||||
rev: v1.24.1
|
||||
hooks:
|
||||
- id: zizmor
|
||||
files: ^\.github/
|
||||
# zizmor only audits workflows, composite actions and dependabot
|
||||
# config; broader paths trip exit 3 ("no audit was performed").
|
||||
files: ^\.github/(workflows|actions)/.+\.ya?ml$|^\.github/dependabot\.ya?ml$
|
||||
priority: 30
|
||||
|
||||
## BASH
|
||||
- repo: https://github.com/koalaman/shellcheck-precommit
|
||||
rev: v0.10.0
|
||||
rev: v0.11.0
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
exclude: contrib
|
||||
priority: 30
|
||||
|
||||
## PYTHON
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/)
|
||||
- repo: https://github.com/myint/autoflake
|
||||
rev: v2.3.1
|
||||
rev: v2.3.3
|
||||
hooks:
|
||||
- id: autoflake
|
||||
exclude: ^skills/
|
||||
args:
|
||||
[
|
||||
"--in-place",
|
||||
"--remove-all-unused-imports",
|
||||
"--remove-unused-variable",
|
||||
]
|
||||
name: "SDK - autoflake"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variable"]
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.13.2
|
||||
rev: 8.0.1
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: ^skills/
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.4.2
|
||||
rev: 26.3.1
|
||||
hooks:
|
||||
- id: black
|
||||
exclude: ^skills/
|
||||
name: "SDK - black"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 7.0.0
|
||||
rev: 7.3.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
exclude: (contrib|^skills/)
|
||||
name: "SDK - flake8"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
priority: 30
|
||||
|
||||
- repo: https://github.com/python-poetry/poetry
|
||||
rev: 2.3.4
|
||||
## PYTHON — API + MCP Server (ruff)
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.11
|
||||
hooks:
|
||||
- id: poetry-check
|
||||
name: API - poetry-check
|
||||
args: ["--directory=./api"]
|
||||
pass_filenames: false
|
||||
- id: ruff
|
||||
name: "API + MCP - ruff check"
|
||||
files: { glob: ["{api,mcp_server}/**/*.py"] }
|
||||
args: ["--fix"]
|
||||
priority: 30
|
||||
- id: ruff-format
|
||||
name: "API + MCP - ruff format"
|
||||
files: { glob: ["{api,mcp_server}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- id: poetry-lock
|
||||
name: API - poetry-lock
|
||||
args: ["--directory=./api"]
|
||||
## PYTHON — uv (API + SDK)
|
||||
- repo: https://github.com/astral-sh/uv-pre-commit
|
||||
rev: 0.11.14
|
||||
hooks:
|
||||
- id: uv-lock
|
||||
name: API - uv-lock
|
||||
args: ["--check", "--project=./api"]
|
||||
files: { glob: ["api/{pyproject.toml,uv.lock}"] }
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
- id: poetry-check
|
||||
name: SDK - poetry-check
|
||||
args: ["--directory=./"]
|
||||
pass_filenames: false
|
||||
|
||||
- id: poetry-lock
|
||||
name: SDK - poetry-lock
|
||||
args: ["--directory=./"]
|
||||
- id: uv-lock
|
||||
name: SDK - uv-lock
|
||||
args: ["--check", "--project=./"]
|
||||
files: { glob: ["{pyproject.toml,uv.lock}"] }
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
## CONTAINERS
|
||||
- repo: https://github.com/hadolint/hadolint
|
||||
rev: v2.13.0-beta
|
||||
rev: v2.14.0
|
||||
hooks:
|
||||
- id: hadolint
|
||||
args: ["--ignore=DL3013"]
|
||||
priority: 30
|
||||
|
||||
## LOCAL HOOKS
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: pylint
|
||||
name: pylint
|
||||
entry: bash -c 'pylint --disable=W,C,R,E -j 0 -rn -sn prowler/'
|
||||
name: "SDK - pylint"
|
||||
entry: pylint --disable=W,C,R,E -j 0 -rn -sn
|
||||
language: system
|
||||
files: '.*\.py'
|
||||
types: [python]
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
priority: 30
|
||||
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data.
|
||||
entry: bash -c 'trufflehog --no-update git file://. --only-verified --fail'
|
||||
entry: bash -c 'trufflehog --no-update git file://. --since-commit HEAD --only-verified --fail'
|
||||
# For running trufflehog in docker, use the following entry instead:
|
||||
# entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --only-verified --fail'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
stages: ["pre-commit", "pre-push"]
|
||||
priority: 40
|
||||
|
||||
- id: bandit
|
||||
name: bandit
|
||||
description: "Bandit is a tool for finding common security issues in Python code"
|
||||
entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/,./skills/' -r .'
|
||||
entry: bandit -q -lll
|
||||
language: system
|
||||
types: [python]
|
||||
files: '.*\.py'
|
||||
|
||||
- id: safety
|
||||
name: safety
|
||||
description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities"
|
||||
# TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
# TODO: 86217 because `alibabacloud-tea-openapi == 0.4.3` don't let us upgrade `cryptography >= 46.0.0`
|
||||
# TODO: 71600 CVE-2024-1135 false positive - fixed in gunicorn 22.0.0, project uses 23.0.0
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745,79023,79027,86217,71600'
|
||||
language: system
|
||||
exclude: { glob: ["{contrib,skills}/**", "**/.venv/**", "**/*_test.py"] }
|
||||
priority: 40
|
||||
|
||||
- id: vulture
|
||||
name: vulture
|
||||
description: "Vulture finds unused code in Python programs."
|
||||
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/,skills/" --min-confidence 100 .'
|
||||
entry: vulture --min-confidence 100
|
||||
language: system
|
||||
types: [python]
|
||||
files: '.*\.py'
|
||||
|
||||
- id: ui-checks
|
||||
name: UI - Husky Pre-commit
|
||||
description: "Run UI pre-commit checks (Claude Code validation + healthcheck)"
|
||||
entry: bash -c 'cd ui && .husky/pre-commit'
|
||||
language: system
|
||||
files: '^ui/.*\.(ts|tsx|js|jsx|json|css)$'
|
||||
pass_filenames: false
|
||||
verbose: true
|
||||
priority: 40
|
||||
|
||||
+2
-6
@@ -11,15 +11,11 @@ build:
|
||||
python: "3.11"
|
||||
jobs:
|
||||
post_create_environment:
|
||||
# Install poetry
|
||||
# https://python-poetry.org/docs/#installing-manually
|
||||
- python -m pip install poetry==2.3.4
|
||||
- python -m pip install uv==0.11.14
|
||||
post_install:
|
||||
# Install dependencies with 'docs' dependency group
|
||||
# https://python-poetry.org/docs/managing-dependencies/#dependency-groups
|
||||
# VIRTUAL_ENV needs to be set manually for now.
|
||||
# See https://github.com/readthedocs/readthedocs.org/pull/11152/
|
||||
- VIRTUAL_ENV=${READTHEDOCS_VIRTUALENV_PATH} python -m poetry install --only=docs
|
||||
- VIRTUAL_ENV=${READTHEDOCS_VIRTUALENV_PATH} uv sync --group docs --no-install-project
|
||||
|
||||
mkdocs:
|
||||
configuration: mkdocs.yml
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
.envrc
|
||||
ui/.env.local
|
||||
@@ -15,7 +15,7 @@ Use these skills for detailed patterns on-demand:
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) |
|
||||
| `nextjs-15` | App Router, Server Actions, streaming | [SKILL.md](skills/nextjs-15/SKILL.md) |
|
||||
| `nextjs-16` | App Router, Server Actions, proxy.ts, streaming | [SKILL.md](skills/nextjs-16/SKILL.md) |
|
||||
| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) |
|
||||
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
|
||||
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
|
||||
@@ -60,11 +60,14 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding a compliance output formatter (per-provider class + table dispatcher) | `prowler-compliance` |
|
||||
| Adding indexes or constraints to database tables | `django-migration-psql` |
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-15` |
|
||||
| App Router / Server Actions | `nextjs-16` |
|
||||
| Auditing check-to-requirement mappings as a cloud auditor | `prowler-compliance` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Configuring MCP servers in agentic workflows | `gh-aw` |
|
||||
@@ -78,6 +81,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating new checks | `prowler-sdk-check` |
|
||||
| Creating new skills | `skill-creator` |
|
||||
| Creating or reviewing Django migrations | `django-migration-psql` |
|
||||
| Creating/modifying Prowler UI components | `prowler-ui` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
@@ -85,6 +89,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Debugging gh-aw compilation errors | `gh-aw` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| Fixing bug | `tdd` |
|
||||
| Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs) | `prowler-compliance` |
|
||||
| General Prowler development questions | `prowler` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Implementing feature | `tdd` |
|
||||
@@ -102,6 +107,8 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Testing hooks or utilities | `vitest` |
|
||||
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
|
||||
@@ -129,6 +136,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Writing React components | `react-19` |
|
||||
| Writing TypeScript types/interfaces | `typescript` |
|
||||
| Writing Vitest tests | `vitest` |
|
||||
| Writing data backfill or data migration | `django-migration-psql` |
|
||||
| Writing documentation | `prowler-docs` |
|
||||
| Writing unit tests for UI | `vitest` |
|
||||
|
||||
@@ -140,9 +148,9 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure,
|
||||
|
||||
| Component | Location | Tech Stack |
|
||||
|-----------|----------|------------|
|
||||
| SDK | `prowler/` | Python 3.10+, Poetry 2.3+ |
|
||||
| SDK | `prowler/` | Python 3.10+, uv |
|
||||
| API | `api/` | Django 5.1, DRF, Celery |
|
||||
| UI | `ui/` | Next.js 15, React 19, Tailwind 4 |
|
||||
| UI | `ui/` | Next.js 16, React 19, Tailwind 4 |
|
||||
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
|
||||
| Dashboard | `dashboard/` | Dash, Plotly |
|
||||
|
||||
@@ -152,13 +160,13 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure,
|
||||
|
||||
```bash
|
||||
# Setup
|
||||
poetry install --with dev
|
||||
poetry run prek install
|
||||
uv sync
|
||||
uv run prek install
|
||||
|
||||
# Code quality
|
||||
poetry run make lint
|
||||
poetry run make format
|
||||
poetry run prek run --all-files
|
||||
uv run make lint
|
||||
uv run make format
|
||||
uv run prek run --all-files
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+29
-6
@@ -1,11 +1,34 @@
|
||||
# Do you want to learn on how to...
|
||||
|
||||
- Contribute with your code or fixes to Prowler
|
||||
- Create a new check for a provider
|
||||
- Create a new security compliance framework
|
||||
- Add a custom output format
|
||||
- Add a new integration
|
||||
- Contribute with documentation
|
||||
- [Contribute with your code or fixes to Prowler](https://docs.prowler.com/developer-guide/introduction)
|
||||
- [Create a new provider](https://docs.prowler.com/developer-guide/provider)
|
||||
- [Create a new service](https://docs.prowler.com/developer-guide/services)
|
||||
- [Create a new check for a provider](https://docs.prowler.com/developer-guide/checks)
|
||||
- [Create a new security compliance framework](https://docs.prowler.com/developer-guide/security-compliance-framework)
|
||||
- [Add a custom output format](https://docs.prowler.com/developer-guide/outputs)
|
||||
- [Add a new integration](https://docs.prowler.com/developer-guide/integrations)
|
||||
- [Contribute with documentation](https://docs.prowler.com/developer-guide/documentation)
|
||||
- [Write unit tests](https://docs.prowler.com/developer-guide/unit-testing)
|
||||
- [Write integration tests](https://docs.prowler.com/developer-guide/integration-testing)
|
||||
- [Write end-to-end tests](https://docs.prowler.com/developer-guide/end2end-testing)
|
||||
- [Debug Prowler](https://docs.prowler.com/developer-guide/debugging)
|
||||
- [Configure checks](https://docs.prowler.com/developer-guide/configurable-checks)
|
||||
- [Rename checks](https://docs.prowler.com/developer-guide/renaming-checks)
|
||||
- [Follow the check metadata guidelines](https://docs.prowler.com/developer-guide/check-metadata-guidelines)
|
||||
- [Extend the MCP server](https://docs.prowler.com/developer-guide/mcp-server)
|
||||
- [Extend Lighthouse AI](https://docs.prowler.com/developer-guide/lighthouse-architecture)
|
||||
- [Add AI skills](https://docs.prowler.com/developer-guide/ai-skills)
|
||||
|
||||
Provider-specific developer notes:
|
||||
|
||||
- [AWS](https://docs.prowler.com/developer-guide/aws-details)
|
||||
- [Azure](https://docs.prowler.com/developer-guide/azure-details)
|
||||
- [Google Cloud](https://docs.prowler.com/developer-guide/gcp-details)
|
||||
- [Alibaba Cloud](https://docs.prowler.com/developer-guide/alibabacloud-details)
|
||||
- [Kubernetes](https://docs.prowler.com/developer-guide/kubernetes-details)
|
||||
- [Microsoft 365](https://docs.prowler.com/developer-guide/m365-details)
|
||||
- [GitHub](https://docs.prowler.com/developer-guide/github-details)
|
||||
- [LLM](https://docs.prowler.com/developer-guide/llm-details)
|
||||
|
||||
Want some swag as appreciation for your contribution?
|
||||
|
||||
|
||||
+30
-11
@@ -6,9 +6,12 @@ LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.69.2
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
ENV ZIZMOR_VERSION=${ZIZMOR_VERSION}
|
||||
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
wget libicu72 libunwind8 libssl3 libcurl4 ca-certificates apt-transport-https gnupg \
|
||||
@@ -48,6 +51,22 @@ RUN ARCH=$(uname -m) && \
|
||||
mkdir -p /tmp/.cache/trivy && \
|
||||
chmod 777 /tmp/.cache/trivy
|
||||
|
||||
# Install zizmor for GitHub Actions workflow scanning
|
||||
RUN ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "x86_64" ]; then \
|
||||
ZIZMOR_ARCH="x86_64-unknown-linux-gnu" ; \
|
||||
elif [ "$ARCH" = "aarch64" ]; then \
|
||||
ZIZMOR_ARCH="aarch64-unknown-linux-gnu" ; \
|
||||
else \
|
||||
echo "Unsupported architecture for zizmor: $ARCH" && exit 1 ; \
|
||||
fi && \
|
||||
wget --progress=dot:giga "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/zizmor-${ZIZMOR_ARCH}.tar.gz" -O /tmp/zizmor.tar.gz && \
|
||||
mkdir -p /tmp/zizmor-extract && \
|
||||
tar zxf /tmp/zizmor.tar.gz -C /tmp/zizmor-extract && \
|
||||
mv /tmp/zizmor-extract/zizmor /usr/local/bin/zizmor && \
|
||||
chmod +x /usr/local/bin/zizmor && \
|
||||
rm -rf /tmp/zizmor.tar.gz /tmp/zizmor-extract
|
||||
|
||||
# Add prowler user
|
||||
RUN addgroup --gid 1000 prowler && \
|
||||
adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler
|
||||
@@ -57,28 +76,28 @@ USER prowler
|
||||
WORKDIR /home/prowler
|
||||
|
||||
# Copy necessary files
|
||||
COPY prowler/ /home/prowler/prowler/
|
||||
COPY dashboard/ /home/prowler/dashboard/
|
||||
COPY pyproject.toml /home/prowler
|
||||
COPY README.md /home/prowler/
|
||||
COPY prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
COPY --chown=prowler:prowler prowler/ /home/prowler/prowler/
|
||||
COPY --chown=prowler:prowler dashboard/ /home/prowler/dashboard/
|
||||
COPY --chown=prowler:prowler pyproject.toml uv.lock /home/prowler/
|
||||
COPY --chown=prowler:prowler README.md /home/prowler/
|
||||
COPY --chown=prowler:prowler prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
# Install Python dependencies
|
||||
ENV HOME='/home/prowler'
|
||||
ENV PATH="${HOME}/.local/bin:${PATH}"
|
||||
#hadolint ignore=DL3013
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir poetry==2.3.4
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
|
||||
RUN poetry install --compile && \
|
||||
rm -rf ~/.cache/pip
|
||||
RUN uv sync --locked --compile-bytecode && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
# Install PowerShell modules
|
||||
RUN poetry run python prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
RUN .venv/bin/python prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
# Remove deprecated dash dependencies
|
||||
RUN pip uninstall dash-html-components -y && \
|
||||
pip uninstall dash-core-components -y
|
||||
|
||||
USER prowler
|
||||
ENTRYPOINT ["poetry", "run", "prowler"]
|
||||
ENTRYPOINT [".venv/bin/prowler"]
|
||||
|
||||
@@ -23,7 +23,7 @@ format: ## Format Code
|
||||
|
||||
lint: ## Lint Code
|
||||
@echo "Running flake8..."
|
||||
flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib
|
||||
flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude .venv,contrib
|
||||
@echo "Running black... "
|
||||
black --check .
|
||||
@echo "Running pylint..."
|
||||
@@ -35,7 +35,7 @@ pypi-clean: ## Delete the distribution files
|
||||
|
||||
pypi-build: ## Build package
|
||||
$(MAKE) pypi-clean && \
|
||||
poetry build
|
||||
uv build
|
||||
|
||||
pypi-upload: ## Upload package
|
||||
python3 -m twine upload --repository pypi dist/*
|
||||
@@ -56,4 +56,3 @@ run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, MCP,
|
||||
|
||||
##@ Development Environment
|
||||
build-and-run-api-dev: build-no-cache-dev run-api-dev
|
||||
|
||||
|
||||
@@ -104,22 +104,24 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 572 | 83 | 41 | 17 | Official | UI, API, CLI |
|
||||
| Azure | 165 | 20 | 18 | 13 | Official | UI, API, CLI |
|
||||
| GCP | 100 | 13 | 15 | 11 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 7 | 9 | Official | UI, API, CLI |
|
||||
| GitHub | 21 | 2 | 1 | 2 | Official | UI, API, CLI |
|
||||
| M365 | 89 | 9 | 4 | 5 | Official | UI, API, CLI |
|
||||
| OCI | 48 | 13 | 3 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 61 | 9 | 3 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 2 | 0 | 5 | Official | UI, API, CLI |
|
||||
| AWS | 600 | 84 | 44 | 18 | Official | UI, API, CLI |
|
||||
| Azure | 167 | 22 | 19 | 16 | Official | UI, API, CLI |
|
||||
| GCP | 102 | 18 | 17 | 12 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 7 | 11 | Official | UI, API, CLI |
|
||||
| GitHub | 24 | 3 | 1 | 5 | Official | UI, API, CLI |
|
||||
| M365 | 102 | 10 | 4 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 51 | 14 | 4 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 9 | 4 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 0 | 5 | Official | UI, API, CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 8 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| Image | N/A | N/A | N/A | N/A | Official | CLI, API |
|
||||
| Google Workspace | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| OpenStack | 27 | 4 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Vercel | 30 | 6 | 0 | 5 | Official | CLI |
|
||||
| Google Workspace | 39 | 5 | 2 | 5 | Official | UI, API, CLI |
|
||||
| OpenStack | 34 | 5 | 0 | 9 | Official | UI, API, CLI |
|
||||
| Vercel | 26 | 6 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
@@ -176,7 +178,7 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
**Requirements**
|
||||
|
||||
* `git` installed.
|
||||
* `poetry` v2 installed: [poetry installation](https://python-poetry.org/docs/#installation).
|
||||
* `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
* `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
@@ -185,8 +187,8 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/api
|
||||
poetry install
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
set -a
|
||||
source .env
|
||||
docker compose up postgres valkey -d
|
||||
@@ -194,11 +196,6 @@ cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
gunicorn -c config/guniconf.py config.wsgi:application
|
||||
```
|
||||
> [!IMPORTANT]
|
||||
> As of Poetry v2.0.0, the `poetry shell` command has been deprecated. Use `poetry env activate` instead for environment activation.
|
||||
>
|
||||
> If your Poetry version is below v2.0.0, continue using `poetry shell` to activate your environment.
|
||||
> For further guidance, refer to the Poetry Environment Activation Guide https://python-poetry.org/docs/managing-environments/#activating-the-environment.
|
||||
|
||||
> After completing the setup, access the API documentation at http://localhost:8080/api/v1/docs.
|
||||
|
||||
@@ -207,8 +204,8 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/api
|
||||
poetry install
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
set -a
|
||||
source .env
|
||||
cd src/backend
|
||||
@@ -220,8 +217,8 @@ python -m celery -A config.celery worker -l info -E
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/api
|
||||
poetry install
|
||||
eval $(poetry env activate)
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
set -a
|
||||
source .env
|
||||
cd src/backend
|
||||
@@ -282,23 +279,47 @@ The container images are available here:
|
||||
|
||||
### From GitHub
|
||||
|
||||
Python >=3.10, <3.13 is required with pip and Poetry:
|
||||
Python >=3.10, <3.13 is required with [uv](https://docs.astral.sh/uv/):
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler
|
||||
eval $(poetry env activate)
|
||||
poetry install
|
||||
uv sync
|
||||
source .venv/bin/activate
|
||||
python prowler-cli.py -v
|
||||
```
|
||||
> [!IMPORTANT]
|
||||
> To clone Prowler on Windows, configure Git to support long file paths by running the following command: `git config core.longpaths true`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> As of Poetry v2.0.0, the `poetry shell` command has been deprecated. Use `poetry env activate` instead for environment activation.
|
||||
>
|
||||
> If your Poetry version is below v2.0.0, continue using `poetry shell` to activate your environment.
|
||||
> For further guidance, refer to the Poetry Environment Activation Guide https://python-poetry.org/docs/managing-environments/#activating-the-environment.
|
||||
# 🛡️ GitHub Action
|
||||
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
|
||||
```yaml
|
||||
name: Prowler IaC Scan
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
prowler:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: prowler-cloud/prowler@5.25
|
||||
with:
|
||||
provider: iac
|
||||
output-formats: sarif json-ocsf
|
||||
upload-sarif: true
|
||||
flags: --severity critical high
|
||||
```
|
||||
|
||||
Full configuration, per-provider authentication, and SARIF examples: [Prowler GitHub Action tutorial](docs/user-guide/tutorials/prowler-app-github-action.mdx). Marketplace listing: [Prowler Security Scan](https://github.com/marketplace/actions/prowler-security-scan).
|
||||
|
||||
# ✏️ High level architecture
|
||||
|
||||
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
name: Prowler Security Scan
|
||||
description: Run Prowler cloud security scanner using the official Docker image
|
||||
branding:
|
||||
icon: cloud
|
||||
color: green
|
||||
|
||||
inputs:
|
||||
provider:
|
||||
description: Cloud provider to scan (e.g. aws, azure, gcp, github, kubernetes, iac). See https://docs.prowler.com for supported providers.
|
||||
required: true
|
||||
image-tag:
|
||||
description: >
|
||||
Docker image tag for prowlercloud/prowler.
|
||||
Default is "stable" (latest release). Available tags:
|
||||
"stable" (latest release), "latest" (master branch, not stable),
|
||||
"<x.y.z>" (pinned release version).
|
||||
See all tags at https://hub.docker.com/r/prowlercloud/prowler/tags
|
||||
required: false
|
||||
default: stable
|
||||
output-formats:
|
||||
description: Output format(s) for scan results (e.g. "json-ocsf", "sarif json-ocsf")
|
||||
required: false
|
||||
default: json-ocsf
|
||||
push-to-cloud:
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli
|
||||
required: false
|
||||
default: "false"
|
||||
flags:
|
||||
description: 'Additional CLI flags passed to the Prowler scan (e.g. "--severity critical high --compliance cis_aws"). Values containing spaces can be quoted, e.g. "--resource-tag ''Environment=My Server''".'
|
||||
required: false
|
||||
default: ""
|
||||
extra-env:
|
||||
description: >
|
||||
Space-, newline-, or comma-separated list of host environment variable NAMES to forward to the Prowler container
|
||||
(e.g. "AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN" for AWS,
|
||||
"GITHUB_PERSONAL_ACCESS_TOKEN" for GitHub, "CLOUDFLARE_API_TOKEN" for Cloudflare).
|
||||
List names only; set the values via `env:` at the workflow or job level (typically from `secrets.*`).
|
||||
See the README for per-provider examples.
|
||||
required: false
|
||||
default: ""
|
||||
upload-sarif:
|
||||
description: 'Upload SARIF results to GitHub Code Scanning (requires "sarif" in output-formats and both `security-events: write` and `actions: read` permissions)'
|
||||
required: false
|
||||
default: "false"
|
||||
sarif-file:
|
||||
description: Path to the SARIF file to upload (auto-detected from output/ if not set)
|
||||
required: false
|
||||
default: ""
|
||||
sarif-category:
|
||||
description: Category for the SARIF upload (used to distinguish multiple analyses)
|
||||
required: false
|
||||
default: prowler
|
||||
fail-on-findings:
|
||||
description: Fail the workflow step when Prowler detects findings (exit code 3). By default the action tolerates findings and succeeds.
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Validate inputs
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_IMAGE_TAG: ${{ inputs.image-tag }}
|
||||
INPUT_UPLOAD_SARIF: ${{ inputs.upload-sarif }}
|
||||
INPUT_OUTPUT_FORMATS: ${{ inputs.output-formats }}
|
||||
run: |
|
||||
# Validate image tag format (alphanumeric, dots, hyphens, underscores only)
|
||||
if [[ ! "$INPUT_IMAGE_TAG" =~ ^[a-zA-Z0-9._-]+$ ]]; then
|
||||
echo "::error::Invalid image-tag '${INPUT_IMAGE_TAG}'. Must contain only alphanumeric characters, dots, hyphens, and underscores."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Warn if upload-sarif is enabled but sarif not in output-formats
|
||||
if [ "$INPUT_UPLOAD_SARIF" = "true" ]; then
|
||||
if [[ ! "$INPUT_OUTPUT_FORMATS" =~ (^|[[:space:]])sarif($|[[:space:]]) ]]; then
|
||||
echo "::warning::upload-sarif is enabled but 'sarif' is not included in output-formats ('${INPUT_OUTPUT_FORMATS}'). SARIF upload will fail unless you add 'sarif' to output-formats."
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Run Prowler scan
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_PROVIDER: ${{ inputs.provider }}
|
||||
INPUT_IMAGE_TAG: ${{ inputs.image-tag }}
|
||||
INPUT_OUTPUT_FORMATS: ${{ inputs.output-formats }}
|
||||
INPUT_PUSH_TO_CLOUD: ${{ inputs.push-to-cloud }}
|
||||
INPUT_FLAGS: ${{ inputs.flags }}
|
||||
INPUT_EXTRA_ENV: ${{ inputs.extra-env }}
|
||||
INPUT_FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }}
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Parse space-separated inputs with shlex so values with spaces can be quoted
|
||||
# (e.g. `--resource-tag 'Environment=My Server'`).
|
||||
mapfile -t OUTPUT_FORMATS < <(python3 -c 'import shlex, os; [print(t) for t in shlex.split(os.environ.get("INPUT_OUTPUT_FORMATS", ""))]')
|
||||
mapfile -t EXTRA_FLAGS < <(python3 -c 'import shlex, os; [print(t) for t in shlex.split(os.environ.get("INPUT_FLAGS", ""))]')
|
||||
mapfile -t EXTRA_ENV_NAMES < <(python3 -c 'import shlex, os; [print(t) for t in shlex.split(os.environ.get("INPUT_EXTRA_ENV", "").replace(",", " "))]')
|
||||
|
||||
env_args=()
|
||||
for var in "${EXTRA_ENV_NAMES[@]}"; do
|
||||
[ -z "$var" ] && continue
|
||||
if [[ ! "$var" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
|
||||
echo "::error::Invalid env var name '${var}' in extra-env. Names must match ^[A-Za-z_][A-Za-z0-9_]*$."
|
||||
exit 1
|
||||
fi
|
||||
env_args+=("-e" "$var")
|
||||
done
|
||||
|
||||
push_args=()
|
||||
if [ "$INPUT_PUSH_TO_CLOUD" = "true" ]; then
|
||||
push_args=("--push-to-cloud")
|
||||
env_args+=("-e" "PROWLER_CLOUD_API_KEY")
|
||||
fi
|
||||
|
||||
mkdir -p "$GITHUB_WORKSPACE/output"
|
||||
chmod 777 "$GITHUB_WORKSPACE/output"
|
||||
|
||||
set +e
|
||||
docker run --rm \
|
||||
"${env_args[@]}" \
|
||||
-v "$GITHUB_WORKSPACE:/home/prowler/workspace" \
|
||||
-v "$GITHUB_WORKSPACE/output:/home/prowler/workspace/output" \
|
||||
-w /home/prowler/workspace \
|
||||
"prowlercloud/prowler:${INPUT_IMAGE_TAG}" \
|
||||
"$INPUT_PROVIDER" \
|
||||
--output-formats "${OUTPUT_FORMATS[@]}" \
|
||||
"${push_args[@]}" \
|
||||
"${EXTRA_FLAGS[@]}"
|
||||
exit_code=$?
|
||||
set -e
|
||||
|
||||
# Exit code 3 = findings detected
|
||||
if [ "$exit_code" -eq 3 ] && [ "$INPUT_FAIL_ON_FINDINGS" != "true" ]; then
|
||||
echo "::notice::Prowler detected findings (exit code 3). Set fail-on-findings to 'true' to fail the workflow on findings."
|
||||
exit 0
|
||||
fi
|
||||
exit $exit_code
|
||||
|
||||
- name: Upload scan results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
||||
with:
|
||||
name: prowler-${{ inputs.provider }}
|
||||
path: output/
|
||||
retention-days: 30
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Find SARIF file
|
||||
if: always() && inputs.upload-sarif == 'true'
|
||||
id: find-sarif
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_SARIF_FILE: ${{ inputs.sarif-file }}
|
||||
run: |
|
||||
if [ -n "$INPUT_SARIF_FILE" ]; then
|
||||
echo "sarif_path=$INPUT_SARIF_FILE" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
sarif_file=$(find output/ -name '*.sarif' -type f | head -1)
|
||||
if [ -z "$sarif_file" ]; then
|
||||
echo "::warning::No .sarif file found in output/. Ensure 'sarif' is included in output-formats."
|
||||
echo "sarif_path=" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "sarif_path=$sarif_file" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Upload SARIF to GitHub Code Scanning
|
||||
if: always() && inputs.upload-sarif == 'true' && steps.find-sarif.outputs.sarif_path != ''
|
||||
uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
sarif_file: ${{ steps.find-sarif.outputs.sarif_path }}
|
||||
category: ${{ inputs.sarif-category }}
|
||||
|
||||
- name: Write scan summary
|
||||
if: always()
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_PROVIDER: ${{ inputs.provider }}
|
||||
INPUT_UPLOAD_SARIF: ${{ inputs.upload-sarif }}
|
||||
INPUT_PUSH_TO_CLOUD: ${{ inputs.push-to-cloud }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPO_URL: ${{ github.server_url }}/${{ github.repository }}
|
||||
BRANCH: ${{ github.head_ref || github.ref_name }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set +e
|
||||
|
||||
# Build a link to the scan step in the workflow logs. Requires `actions: read`
|
||||
# on the caller's GITHUB_TOKEN; silently skips the link if unavailable.
|
||||
scan_step_url=""
|
||||
if [ -n "${GH_TOKEN:-}" ] && command -v gh >/dev/null 2>&1; then
|
||||
job_info=$(gh api \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT:-1}/jobs" \
|
||||
--jq ".jobs[] | select(.runner_name == \"${RUNNER_NAME:-}\")" 2>/dev/null)
|
||||
if [ -n "$job_info" ]; then
|
||||
job_id=$(jq -r '.id // empty' <<<"$job_info")
|
||||
step_number=$(jq -r '[.steps[]? | select((.name // "") | test("Run Prowler scan"; "i")) | .number] | first // empty' <<<"$job_info")
|
||||
if [ -z "$step_number" ]; then
|
||||
step_number=$(jq -r '[.steps[]? | select(.status == "in_progress") | .number] | first // empty' <<<"$job_info")
|
||||
fi
|
||||
if [ -n "$job_id" ] && [ -n "$step_number" ]; then
|
||||
scan_step_url="${REPO_URL}/actions/runs/${GITHUB_RUN_ID}/job/${job_id}#step:${step_number}:1"
|
||||
elif [ -n "$job_id" ]; then
|
||||
scan_step_url="${REPO_URL}/actions/runs/${GITHUB_RUN_ID}/job/${job_id}"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Map provider code to a properly-cased display name.
|
||||
case "$INPUT_PROVIDER" in
|
||||
alibabacloud) provider_name="Alibaba Cloud" ;;
|
||||
aws) provider_name="AWS" ;;
|
||||
azure) provider_name="Azure" ;;
|
||||
cloudflare) provider_name="Cloudflare" ;;
|
||||
gcp) provider_name="GCP" ;;
|
||||
github) provider_name="GitHub" ;;
|
||||
googleworkspace) provider_name="Google Workspace" ;;
|
||||
iac) provider_name="IaC" ;;
|
||||
image) provider_name="Container Image" ;;
|
||||
kubernetes) provider_name="Kubernetes" ;;
|
||||
llm) provider_name="LLM" ;;
|
||||
m365) provider_name="Microsoft 365" ;;
|
||||
mongodbatlas) provider_name="MongoDB Atlas" ;;
|
||||
nhn) provider_name="NHN" ;;
|
||||
openstack) provider_name="OpenStack" ;;
|
||||
oraclecloud) provider_name="Oracle Cloud" ;;
|
||||
vercel) provider_name="Vercel" ;;
|
||||
*) provider_name="${INPUT_PROVIDER^}" ;;
|
||||
esac
|
||||
|
||||
ocsf_file=$(find output/ -name '*.ocsf.json' -type f 2>/dev/null | head -1)
|
||||
|
||||
{
|
||||
echo "## Prowler ${provider_name} Scan Summary"
|
||||
echo ""
|
||||
|
||||
counts=""
|
||||
if [ -n "$ocsf_file" ] && [ -s "$ocsf_file" ]; then
|
||||
counts=$(jq -r '[
|
||||
length,
|
||||
([.[] | select(.status_code == "FAIL")] | length),
|
||||
([.[] | select(.status_code == "PASS")] | length),
|
||||
([.[] | select(.status_code == "MUTED")] | length),
|
||||
([.[] | select(.status_code == "FAIL" and .severity == "Critical")] | length),
|
||||
([.[] | select(.status_code == "FAIL" and .severity == "High")] | length),
|
||||
([.[] | select(.status_code == "FAIL" and .severity == "Medium")] | length),
|
||||
([.[] | select(.status_code == "FAIL" and .severity == "Low")] | length),
|
||||
([.[] | select(.status_code == "FAIL" and .severity == "Informational")] | length)
|
||||
] | @tsv' "$ocsf_file" 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [ -n "$counts" ]; then
|
||||
read -r total fail pass muted critical high medium low info <<<"$counts"
|
||||
|
||||
line="**${fail:-0} failing** · ${pass:-0} passing"
|
||||
[ "${muted:-0}" -gt 0 ] && line="${line} · ${muted} muted"
|
||||
echo "${line} — ${total:-0} checks total"
|
||||
echo ""
|
||||
echo "| Severity | Failing |"
|
||||
echo "|----------|---------|"
|
||||
echo "| ‼️ Critical | ${critical:-0} |"
|
||||
echo "| 🔴 High | ${high:-0} |"
|
||||
echo "| 🟠 Medium | ${medium:-0} |"
|
||||
echo "| 🔵 Low | ${low:-0} |"
|
||||
echo "| ⚪ Informational | ${info:-0} |"
|
||||
echo ""
|
||||
else
|
||||
echo "_No findings report was produced. Check the scan logs above._"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ -n "$scan_step_url" ]; then
|
||||
echo "**Scan logs:** [view in workflow run](${scan_step_url})"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo "**Get the full report:** [\`prowler-${INPUT_PROVIDER}\` artifact](${RUN_URL}#artifacts)"
|
||||
|
||||
if [ "$INPUT_UPLOAD_SARIF" = "true" ] && [ -n "$BRANCH" ]; then
|
||||
encoded_branch=$(jq -nr --arg b "$BRANCH" '$b|@uri')
|
||||
echo ""
|
||||
echo "**See results in GitHub Code Security:** [open alerts on \`${BRANCH}\`](${REPO_URL}/security/code-scanning?query=is%3Aopen+branch%3A${encoded_branch})"
|
||||
fi
|
||||
|
||||
if [ "$INPUT_PUSH_TO_CLOUD" != "true" ]; then
|
||||
echo ""
|
||||
echo "---"
|
||||
echo ""
|
||||
echo "### Scale ${provider_name} security with Prowler Cloud ☁️"
|
||||
echo ""
|
||||
echo "Send this scan's findings to **[Prowler Cloud](https://cloud.prowler.com)** and get:"
|
||||
echo ""
|
||||
echo "- **Unified findings** across every cloud, SaaS provider (M365, Google Workspace, GitHub, MongoDB Atlas), IaC repo, Kubernetes cluster, and container image"
|
||||
echo "- **Posture over time** with alerts, and notifications"
|
||||
echo "- **Prowler Lighthouse AI**: agentic assistant that triages findings, explains root cause and helps with remediation"
|
||||
echo "- **50+ Compliance frameworks** mapped automatically"
|
||||
echo "- **Enterprise-ready platform**: SOC 2 Type 2, SSO/SAML, AWS Security Hub, S3 and Jira integrations"
|
||||
echo ""
|
||||
echo "**Get started in 3 steps:**"
|
||||
echo "1. Create an account at [cloud.prowler.com](https://cloud.prowler.com)"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli))"
|
||||
echo "3. Add \`PROWLER_CLOUD_API_KEY\` to your GitHub secrets and set \`push-to-cloud: true\` on this action"
|
||||
echo ""
|
||||
echo "See [prowler.com/pricing](https://prowler.com/pricing) for plan details."
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
+8
-8
@@ -124,24 +124,24 @@ api/src/backend/
|
||||
|
||||
```bash
|
||||
# Development
|
||||
poetry run python src/backend/manage.py runserver
|
||||
poetry run celery -A config.celery worker -l INFO
|
||||
uv run python src/backend/manage.py runserver
|
||||
uv run celery -A config.celery worker -l INFO
|
||||
|
||||
# Database
|
||||
poetry run python src/backend/manage.py makemigrations
|
||||
poetry run python src/backend/manage.py migrate
|
||||
uv run python src/backend/manage.py makemigrations
|
||||
uv run python src/backend/manage.py migrate
|
||||
|
||||
# Testing & Linting
|
||||
poetry run pytest -x --tb=short
|
||||
poetry run make lint
|
||||
uv run pytest -x --tb=short
|
||||
uv run make lint
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## QA CHECKLIST
|
||||
|
||||
- [ ] `poetry run pytest` passes
|
||||
- [ ] `poetry run make lint` passes
|
||||
- [ ] `uv run pytest` passes
|
||||
- [ ] `uv run make lint` passes
|
||||
- [ ] Migrations created if models changed
|
||||
- [ ] New endpoints have `@extend_schema` decorators
|
||||
- [ ] RLS properly applied for tenant data
|
||||
|
||||
+79
-1
@@ -2,6 +2,80 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.28.0] (Prowler v5.27.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- GIN index on `findings(categories, resource_services, resource_regions, resource_types)` to speed up `/api/v1/finding-groups` array filters [(#11001)](https://github.com/prowler-cloud/prowler/pull/11001)
|
||||
- `GET /health/live` and `GET /health/ready` Kubernetes-style probe endpoints following the IETF Health Check Response Format (`application/health+json`). Readiness verifies PostgreSQL, Valkey and Neo4j connectivity and returns 503 with per-dependency detail when any is unreachable [(#11200)](https://github.com/prowler-cloud/prowler/pull/11200)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Replace `poetry` with `uv` as package manager [(#10775)](https://github.com/prowler-cloud/prowler/pull/10775)
|
||||
- Remove orphaned `gin_resources_search_idx` declaration from `Resource.Meta.indexes` (DB index dropped in `0072_drop_unused_indexes`) [(#11001)](https://github.com/prowler-cloud/prowler/pull/11001)
|
||||
- PDF compliance reports cap detail tables at 100 failed findings per check (configurable via `DJANGO_PDF_MAX_FINDINGS_PER_CHECK`) to bound worker memory on large scans [(#11160)](https://github.com/prowler-cloud/prowler/pull/11160)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `perform_scan_task` and `perform_scheduled_scan_task` now short-circuit with a warning and `return None` when the target provider no longer exists, instead of letting `handle_provider_deletion` raise `ProviderDeletedException`. `perform_scheduled_scan_task` also removes any orphan `PeriodicTask` it finds so beat stops re-firing scans for deleted providers. Prevents queued messages for deleted providers from being recorded as `FAILURE` [(#11185)](https://github.com/prowler-cloud/prowler/pull/11185)
|
||||
- Attack Paths: `BEDROCK-001` and `BEDROCK-002` now target roles trusting `bedrock-agentcore.amazonaws.com` instead of `bedrock.amazonaws.com`, eliminating false positives against regular Bedrock service roles (Agents, Knowledge Bases, model invocation) [(#11141)](https://github.com/prowler-cloud/prowler/pull/11141)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## [1.27.1] (Prowler v5.26.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `POST /api/v1/scans` was intermittently failing with `Scan matching query does not exist` in the `scan-perform` worker; the Celery task is now published via `transaction.on_commit` so the worker cannot read the Scan before the dispatch-wide transaction commits [(#11122)](https://github.com/prowler-cloud/prowler/pull/11122)
|
||||
|
||||
---
|
||||
|
||||
## [1.27.0] (Prowler v5.26.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `scan-reset-ephemeral-resources` post-scan task zeroes `failed_findings_count` for resources missing from the latest full-scope scan, keeping ephemeral resources from polluting the Resources page sort [(#10929)](https://github.com/prowler-cloud/prowler/pull/10929)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- ASD Essential Eight (AWS) compliance framework support [(#10982)](https://github.com/prowler-cloud/prowler/pull/10982)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `trivy` binary from 0.69.2 to 0.70.0 and `cryptography` from 46.0.6 to 46.0.7 (transitive via prowler SDK) in the API image for CVE-2026-33186 and CVE-2026-39892 [(#10978)](https://github.com/prowler-cloud/prowler/pull/10978)
|
||||
|
||||
---
|
||||
|
||||
## [1.26.1] (Prowler v5.25.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: AWS scans no longer fail when enabled regions cannot be retrieved, and scans stuck in `scheduled` state are now cleaned up after the stale threshold [(#10917)](https://github.com/prowler-cloud/prowler/pull/10917)
|
||||
- Scan report and compliance downloads now redirect to a presigned S3 URL instead of streaming through the API worker, preventing gunicorn timeouts on large files [(#10927)](https://github.com/prowler-cloud/prowler/pull/10927)
|
||||
|
||||
---
|
||||
|
||||
## [1.26.0] (Prowler v5.25.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- CIS Benchmark PDF report generation for scans, exposing the latest CIS version per provider via `GET /scans/{id}/cis/{name}/` [(#10650)](https://github.com/prowler-cloud/prowler/pull/10650)
|
||||
- `/overviews/resource-groups` (resource inventory), `/overviews/categories` and `/overviews/attack-surfaces` now reflect newly-muted findings without waiting for the next scan. The post-mute `reaggregate-all-finding-group-summaries` task now also dispatches `aggregate_scan_resource_group_summaries_task`, `aggregate_scan_category_summaries_task` and `aggregate_attack_surface_task` per latest scan of every `(provider, day)` pair, rebuilding `ScanGroupSummary`, `ScanCategorySummary` and `AttackSurfaceOverview` alongside the tables already covered in #10827 [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843)
|
||||
- Install zizmor v1.24.1 in API Docker image for GitHub Actions workflow scanning [(#10607)](https://github.com/prowler-cloud/prowler/pull/10607)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Allows tenant owners to expel users from their organizations [(#10787)](https://github.com/prowler-cloud/prowler/pull/10787)
|
||||
- `aggregate_findings`, `aggregate_attack_surface`, `aggregate_scan_resource_group_summaries` and `aggregate_scan_category_summaries` now upsert via `bulk_create(update_conflicts=True, ...)` instead of the prior `ignore_conflicts=True` / plain INSERT / `already backfilled` short-circuit. Re-runs triggered by the post-mute reaggregation pipeline no longer trip the `unique_*_per_scan` constraints nor silently drop updates, and are race-safe under concurrent writers (e.g. scan completion overlapping with a fresh mute rule) [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843)
|
||||
- Rename the scan-category and scan-resource-group summary aggregators from `backfill_*` to `aggregate_*` [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `generate_outputs_task` crashing with `KeyError` for compliance frameworks listed by `get_compliance_frameworks` but not loadable by `Compliance.get_bulk` [(#10903)](https://github.com/prowler-cloud/prowler/pull/10903)
|
||||
|
||||
---
|
||||
|
||||
## [1.25.4] (Prowler v5.24.4)
|
||||
|
||||
### 🚀 Added
|
||||
@@ -12,6 +86,10 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
- Attack Paths: Neo4j driver `connection_acquisition_timeout` is now configurable via `NEO4J_CONN_ACQUISITION_TIMEOUT` (default lowered from 120 s to 15 s) [(#10873)](https://github.com/prowler-cloud/prowler/pull/10873)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `/tmp/prowler_api_output` saturation in compliance report workers: the final `rmtree` in `generate_compliance_reports` now only waits on frameworks actually generated for the provider (so unsupported frameworks no longer leave a placeholder `results` entry that blocks cleanup), output directories are created lazily per enabled framework, and both `generate_compliance_reports` and `generate_outputs_task` run an opportunistic stale cleanup at task start with a 48h age threshold, a per-host `fcntl` throttle, a 50-deletions-per-run cap, and guards that protect EXECUTING scans and scans whose `output_location` still points to a local path (metadata lookups routed through the admin DB so RLS does not hide those rows) [(#10874)](https://github.com/prowler-cloud/prowler/pull/10874)
|
||||
|
||||
---
|
||||
|
||||
## [1.25.3] (Prowler v5.24.3)
|
||||
@@ -38,7 +116,6 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
- `/finding-groups/latest/<check_id>/resources` now selects the latest completed scan per provider by `-completed_at` (then `-inserted_at`) instead of `-inserted_at`, matching the `/finding-groups/latest` summary path and the daily-summary upsert so overlapping scans no longer produce diverging `delta`/`new_count` between the two endpoints [(#10802)](https://github.com/prowler-cloud/prowler/pull/10802)
|
||||
|
||||
---
|
||||
|
||||
## [1.25.1] (Prowler v5.24.1)
|
||||
|
||||
@@ -52,6 +129,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Attack Paths: Missing `tenant_id` filter while getting related findings after scan completes [(#10722)](https://github.com/prowler-cloud/prowler/pull/10722)
|
||||
- Finding group counters `pass_count`, `fail_count` and `manual_count` now exclude muted findings [(#10753)](https://github.com/prowler-cloud/prowler/pull/10753)
|
||||
- Silent data loss in `ResourceFindingMapping` bulk insert that left findings orphaned when `INSERT ... ON CONFLICT DO NOTHING` dropped rows without raising; added explicit `unique_fields` [(#10724)](https://github.com/prowler-cloud/prowler/pull/10724)
|
||||
- `DELETE /tenants/{tenant_pk}/memberships/{id}` now deletes the expelled user's account when the removed membership was their last one, and blacklists every outstanding refresh token for that user so their existing sessions can no longer mint new access tokens [(#10787)](https://github.com/prowler-cloud/prowler/pull/10787)
|
||||
|
||||
---
|
||||
|
||||
|
||||
+30
-9
@@ -5,12 +5,16 @@ LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.69.2
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
ENV ZIZMOR_VERSION=${ZIZMOR_VERSION}
|
||||
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
wget \
|
||||
git \
|
||||
libicu72 \
|
||||
gcc \
|
||||
g++ \
|
||||
@@ -22,6 +26,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libtool \
|
||||
libxslt1-dev \
|
||||
python3-dev \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install PowerShell
|
||||
@@ -57,6 +62,22 @@ RUN ARCH=$(uname -m) && \
|
||||
mkdir -p /tmp/.cache/trivy && \
|
||||
chmod 777 /tmp/.cache/trivy
|
||||
|
||||
# Install zizmor for GitHub Actions workflow scanning
|
||||
RUN ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "x86_64" ]; then \
|
||||
ZIZMOR_ARCH="x86_64-unknown-linux-gnu" ; \
|
||||
elif [ "$ARCH" = "aarch64" ]; then \
|
||||
ZIZMOR_ARCH="aarch64-unknown-linux-gnu" ; \
|
||||
else \
|
||||
echo "Unsupported architecture for zizmor: $ARCH" && exit 1 ; \
|
||||
fi && \
|
||||
wget --progress=dot:giga "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/zizmor-${ZIZMOR_ARCH}.tar.gz" -O /tmp/zizmor.tar.gz && \
|
||||
mkdir -p /tmp/zizmor-extract && \
|
||||
tar zxf /tmp/zizmor.tar.gz -C /tmp/zizmor-extract && \
|
||||
mv /tmp/zizmor-extract/zizmor /usr/local/bin/zizmor && \
|
||||
chmod +x /usr/local/bin/zizmor && \
|
||||
rm -rf /tmp/zizmor.tar.gz /tmp/zizmor-extract
|
||||
|
||||
# Add prowler user
|
||||
RUN addgroup --gid 1000 prowler && \
|
||||
adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler
|
||||
@@ -68,21 +89,21 @@ WORKDIR /home/prowler
|
||||
# Ensure output directory exists
|
||||
RUN mkdir -p /tmp/prowler_api_output
|
||||
|
||||
COPY pyproject.toml ./
|
||||
COPY --chown=prowler:prowler pyproject.toml uv.lock ./
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir poetry==2.3.4
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
|
||||
ENV PATH="/home/prowler/.local/bin:$PATH"
|
||||
|
||||
# Add `--no-root` to avoid installing the current project as a package
|
||||
RUN poetry install --no-root && \
|
||||
rm -rf ~/.cache/pip
|
||||
# Add `--no-install-project` to avoid installing the current project as a package
|
||||
RUN uv sync --locked --no-install-project && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
RUN poetry run python "$(poetry env info --path)/src/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py"
|
||||
RUN .venv/bin/python .venv/lib/python3.12/site-packages/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
COPY src/backend/ ./backend/
|
||||
COPY docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
COPY --chown=prowler:prowler src/backend/ ./backend/
|
||||
COPY --chown=prowler:prowler docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
|
||||
WORKDIR /home/prowler/backend
|
||||
|
||||
|
||||
+12
-19
@@ -25,12 +25,11 @@ If you don’t set `DJANGO_TOKEN_SIGNING_KEY` or `DJANGO_TOKEN_VERIFYING_KEY`, t
|
||||
**Important note**: Every Prowler version (or repository branches and tags) could have different variables set in its `.env` file. Please use the `.env` file that corresponds with each version.
|
||||
|
||||
## Local deployment
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the Poetry interpreter, not before. Otherwise, variables will not be loaded properly.
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the virtual environment, not before. Otherwise, variables will not be loaded properly.
|
||||
|
||||
To do this, you can run:
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
set -a
|
||||
source .env
|
||||
```
|
||||
@@ -78,7 +77,7 @@ docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
|
||||
## Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `poetry` and `docker compose` are installed.
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
|
||||
@@ -90,11 +89,10 @@ git clone https://github.com/prowler-cloud/api.git
|
||||
git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
### Install all dependencies with Poetry
|
||||
### Install all dependencies with uv
|
||||
|
||||
```console
|
||||
poetry install
|
||||
poetry shell
|
||||
uv sync
|
||||
```
|
||||
|
||||
## Start the PostgreSQL Database and Valkey
|
||||
@@ -139,7 +137,7 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
## Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `poetry` and `docker compose` are installed.
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
|
||||
@@ -165,11 +163,10 @@ docker compose up postgres valkey -d
|
||||
|
||||
### Install the Python dependencies
|
||||
|
||||
> You must have Poetry installed
|
||||
> You must have uv installed
|
||||
|
||||
```console
|
||||
poetry install
|
||||
poetry shell
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Apply migrations
|
||||
@@ -246,9 +243,8 @@ docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
uv run python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
## Apply fixtures
|
||||
@@ -256,9 +252,8 @@ python manage.py migrate --database admin
|
||||
Fixtures are used to populate the database with initial development data.
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
cd src/backend
|
||||
python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
uv run python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
```
|
||||
|
||||
> The default credentials are `dev@prowler.com:Thisisapassword123@` or `dev2@prowler.com:Thisisapassword123@`
|
||||
@@ -270,9 +265,8 @@ Note that the tests will fail if you use the same `.env` file as the development
|
||||
For best results, run in a new shell with no environment variables set.
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
cd src/backend
|
||||
pytest
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
# Custom commands
|
||||
@@ -284,8 +278,7 @@ Django provides a way to create custom commands that can be run from the command
|
||||
To run a custom command, you need to be in the `prowler/api/src/backend` directory and run:
|
||||
|
||||
```console
|
||||
poetry shell
|
||||
python manage.py <command_name>
|
||||
uv run python manage.py <command_name>
|
||||
```
|
||||
|
||||
## Generate dummy data
|
||||
@@ -308,7 +301,7 @@ This command creates, for a given tenant, a provider, scan and a set of findings
|
||||
### Example
|
||||
|
||||
```console
|
||||
~/backend $ poetry run python manage.py findings --tenant
|
||||
~/backend $ uv run python manage.py findings --tenant
|
||||
fffb1893-3fc7-4623-a5d9-fae47da1c528 --findings 25000 --re
|
||||
sources 1000 --batch 5000 --alias test-script
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ apply_migrations() {
|
||||
echo "Applying database migrations..."
|
||||
|
||||
# Fix Inconsistent migration history after adding sites app
|
||||
poetry run python manage.py check_and_fix_socialaccount_sites_migration --database admin
|
||||
uv run python manage.py check_and_fix_socialaccount_sites_migration --database admin
|
||||
|
||||
poetry run python manage.py migrate --database admin
|
||||
uv run python manage.py migrate --database admin
|
||||
}
|
||||
|
||||
apply_fixtures() {
|
||||
@@ -15,19 +15,19 @@ apply_fixtures() {
|
||||
for fixture in api/fixtures/dev/*.json; do
|
||||
if [ -f "$fixture" ]; then
|
||||
echo "Loading $fixture"
|
||||
poetry run python manage.py loaddata "$fixture" --database admin
|
||||
uv run python manage.py loaddata "$fixture" --database admin
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
start_dev_server() {
|
||||
echo "Starting the development server..."
|
||||
poetry run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
}
|
||||
|
||||
start_prod_server() {
|
||||
echo "Starting the Gunicorn server..."
|
||||
poetry run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
}
|
||||
|
||||
resolve_worker_hostname() {
|
||||
@@ -47,7 +47,7 @@ resolve_worker_hostname() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
poetry run python -m celery -A config.celery worker \
|
||||
uv run python -m celery -A config.celery worker \
|
||||
-n "$(resolve_worker_hostname)" \
|
||||
-l "${DJANGO_LOGGING_LEVEL:-info}" \
|
||||
-Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \
|
||||
@@ -56,7 +56,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
poetry run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
manage_db_partitions() {
|
||||
@@ -64,7 +64,7 @@ manage_db_partitions() {
|
||||
echo "Managing DB partitions..."
|
||||
# For now we skip the deletion of partitions until we define the data retention policy
|
||||
# --yes auto approves the operation without the need of an interactive terminal
|
||||
poetry run python manage.py pgpartition --using admin --skip-delete --yes
|
||||
uv run python manage.py pgpartition --using admin --skip-delete --yes
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
Generated
-9427
File diff suppressed because it is too large
Load Diff
+402
-30
@@ -1,6 +1,24 @@
|
||||
[build-system]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
requires = ["poetry-core"]
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"bandit==1.7.9",
|
||||
"coverage==7.5.4",
|
||||
"django-silk==5.3.2",
|
||||
"docker==7.1.0",
|
||||
"filelock==3.20.3",
|
||||
"freezegun==1.5.1",
|
||||
"mypy==1.10.1",
|
||||
"pylint==3.2.5",
|
||||
"pytest==9.0.3",
|
||||
"pytest-cov==5.0.0",
|
||||
"pytest-django==4.8.0",
|
||||
"pytest-env==1.1.3",
|
||||
"pytest-randomly==3.15.0",
|
||||
"pytest-xdist==3.6.1",
|
||||
"ruff==0.5.0",
|
||||
"tqdm==4.67.1",
|
||||
"vulture==2.14",
|
||||
"prek==0.3.9"
|
||||
]
|
||||
|
||||
[project]
|
||||
authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
|
||||
@@ -24,14 +42,14 @@ dependencies = [
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"defusedxml==0.7.1",
|
||||
"gunicorn==23.0.0",
|
||||
"lxml==5.3.2",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.24",
|
||||
"lxml==6.1.0",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.27",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"pytest-celery[redis] (==1.3.0)",
|
||||
"sentry-sdk[django] (==2.56.0)",
|
||||
"uuid6==2024.7.10",
|
||||
"openai (==1.109.1)",
|
||||
"xmlsec==1.3.14",
|
||||
"xmlsec==1.3.17",
|
||||
"h2 (==4.3.0)",
|
||||
"markdown (==3.10.2)",
|
||||
"drf-simple-apikey (==2.2.1)",
|
||||
@@ -50,28 +68,382 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.25.5"
|
||||
version = "1.28.1"
|
||||
|
||||
[project.scripts]
|
||||
celery = "src.backend.config.settings.celery"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
bandit = "1.7.9"
|
||||
coverage = "7.5.4"
|
||||
django-silk = "5.3.2"
|
||||
docker = "7.1.0"
|
||||
filelock = "3.20.3"
|
||||
freezegun = "1.5.1"
|
||||
mypy = "1.10.1"
|
||||
pylint = "3.2.5"
|
||||
pytest = "9.0.3"
|
||||
pytest-cov = "5.0.0"
|
||||
pytest-django = "4.8.0"
|
||||
pytest-env = "1.1.3"
|
||||
pytest-randomly = "3.15.0"
|
||||
pytest-xdist = "3.6.1"
|
||||
ruff = "0.5.0"
|
||||
safety = "3.7.0"
|
||||
tqdm = "4.67.1"
|
||||
vulture = "2.14"
|
||||
prek = "0.3.9"
|
||||
[tool.uv]
|
||||
# Transitive pins matching master to avoid silent drift; bump deliberately.
|
||||
constraint-dependencies = [
|
||||
"about-time==4.2.1",
|
||||
"adal==1.2.7",
|
||||
"aioboto3==15.5.0",
|
||||
"aiobotocore==2.25.1",
|
||||
"aiofiles==24.1.0",
|
||||
"aiohappyeyeballs==2.6.1",
|
||||
"aiohttp==3.13.5",
|
||||
"aioitertools==0.13.0",
|
||||
"aiosignal==1.4.0",
|
||||
"alibabacloud-actiontrail20200706==2.4.1",
|
||||
"alibabacloud-credentials==1.0.3",
|
||||
"alibabacloud-credentials-api==1.0.0",
|
||||
"alibabacloud-cs20151215==6.1.0",
|
||||
"alibabacloud-darabonba-array==0.1.0",
|
||||
"alibabacloud-darabonba-encode-util==0.0.2",
|
||||
"alibabacloud-darabonba-map==0.0.1",
|
||||
"alibabacloud-darabonba-signature-util==0.0.4",
|
||||
"alibabacloud-darabonba-string==0.0.4",
|
||||
"alibabacloud-darabonba-time==0.0.1",
|
||||
"alibabacloud-ecs20140526==7.2.5",
|
||||
"alibabacloud-endpoint-util==0.0.4",
|
||||
"alibabacloud-gateway-oss==0.0.17",
|
||||
"alibabacloud-gateway-oss-util==0.0.3",
|
||||
"alibabacloud-gateway-sls==0.4.0",
|
||||
"alibabacloud-gateway-sls-util==0.4.0",
|
||||
"alibabacloud-gateway-spi==0.0.3",
|
||||
"alibabacloud-openapi-util==0.2.4",
|
||||
"alibabacloud-oss-util==0.0.6",
|
||||
"alibabacloud-oss20190517==1.0.6",
|
||||
"alibabacloud-ram20150501==1.2.0",
|
||||
"alibabacloud-rds20140815==12.0.0",
|
||||
"alibabacloud-sas20181203==6.1.0",
|
||||
"alibabacloud-sls20201230==5.9.0",
|
||||
"alibabacloud-sts20150401==1.1.6",
|
||||
"alibabacloud-tea==0.4.3",
|
||||
"alibabacloud-tea-openapi==0.4.4",
|
||||
"alibabacloud-tea-util==0.3.14",
|
||||
"alibabacloud-tea-xml==0.0.3",
|
||||
"alibabacloud-vpc20160428==6.13.0",
|
||||
"alive-progress==3.3.0",
|
||||
"aliyun-log-fastpb==0.2.0",
|
||||
"amqp==5.3.1",
|
||||
"annotated-types==0.7.0",
|
||||
"anyio==4.12.1",
|
||||
"applicationinsights==0.11.10",
|
||||
"apscheduler==3.11.2",
|
||||
"argcomplete==3.5.3",
|
||||
"asgiref==3.11.0",
|
||||
"astroid==3.2.4",
|
||||
"async-timeout==5.0.1",
|
||||
"attrs==25.4.0",
|
||||
"authlib==1.6.9",
|
||||
"autopep8==2.3.2",
|
||||
"awsipranges==0.3.3",
|
||||
"azure-cli-core==2.83.0",
|
||||
"azure-cli-telemetry==1.1.0",
|
||||
"azure-common==1.1.28",
|
||||
"azure-core==1.38.1",
|
||||
"azure-identity==1.21.0",
|
||||
"azure-keyvault-certificates==4.10.0",
|
||||
"azure-keyvault-keys==4.10.0",
|
||||
"azure-keyvault-secrets==4.10.0",
|
||||
"azure-mgmt-apimanagement==5.0.0",
|
||||
"azure-mgmt-applicationinsights==4.1.0",
|
||||
"azure-mgmt-authorization==4.0.0",
|
||||
"azure-mgmt-compute==34.0.0",
|
||||
"azure-mgmt-containerinstance==10.1.0",
|
||||
"azure-mgmt-containerregistry==12.0.0",
|
||||
"azure-mgmt-containerservice==34.1.0",
|
||||
"azure-mgmt-core==1.6.0",
|
||||
"azure-mgmt-cosmosdb==9.7.0",
|
||||
"azure-mgmt-databricks==2.0.0",
|
||||
"azure-mgmt-datafactory==9.2.0",
|
||||
"azure-mgmt-eventgrid==10.4.0",
|
||||
"azure-mgmt-eventhub==11.2.0",
|
||||
"azure-mgmt-keyvault==10.3.1",
|
||||
"azure-mgmt-loganalytics==12.0.0",
|
||||
"azure-mgmt-logic==10.0.0",
|
||||
"azure-mgmt-monitor==6.0.2",
|
||||
"azure-mgmt-network==28.1.0",
|
||||
"azure-mgmt-postgresqlflexibleservers==1.1.0",
|
||||
"azure-mgmt-rdbms==10.1.0",
|
||||
"azure-mgmt-recoveryservices==3.1.0",
|
||||
"azure-mgmt-recoveryservicesbackup==9.2.0",
|
||||
"azure-mgmt-resource==24.0.0",
|
||||
"azure-mgmt-search==9.1.0",
|
||||
"azure-mgmt-security==7.0.0",
|
||||
"azure-mgmt-sql==3.0.1",
|
||||
"azure-mgmt-storage==22.1.1",
|
||||
"azure-mgmt-subscription==3.1.1",
|
||||
"azure-mgmt-synapse==2.0.0",
|
||||
"azure-mgmt-web==8.0.0",
|
||||
"azure-monitor-query==2.0.0",
|
||||
"azure-storage-blob==12.24.1",
|
||||
"azure-synapse-artifacts==0.21.0",
|
||||
"backoff==2.2.1",
|
||||
"bandit==1.7.9",
|
||||
"billiard==4.2.4",
|
||||
"blinker==1.9.0",
|
||||
"boto3==1.40.61",
|
||||
"botocore==1.40.61",
|
||||
"cartography==0.135.0",
|
||||
"celery==5.6.2",
|
||||
"certifi==2026.1.4",
|
||||
"cffi==2.0.0",
|
||||
"charset-normalizer==3.4.4",
|
||||
"circuitbreaker==2.1.3",
|
||||
"click==8.3.1",
|
||||
"click-didyoumean==0.3.1",
|
||||
"click-plugins==1.1.1.2",
|
||||
"click-repl==0.3.0",
|
||||
"cloudflare==4.3.1",
|
||||
"colorama==0.4.6",
|
||||
"contextlib2==21.6.0",
|
||||
"contourpy==1.3.3",
|
||||
"coverage==7.5.4",
|
||||
"cron-descriptor==1.4.5",
|
||||
"crowdstrike-falconpy==1.6.0",
|
||||
"cryptography==46.0.7",
|
||||
"cycler==0.12.1",
|
||||
"darabonba-core==1.0.5",
|
||||
"dash==3.1.1",
|
||||
"dash-bootstrap-components==2.0.3",
|
||||
"debugpy==1.8.20",
|
||||
"decorator==5.2.1",
|
||||
"defusedxml==0.7.1",
|
||||
"detect-secrets==1.5.0",
|
||||
"dill==0.4.1",
|
||||
"distro==1.9.0",
|
||||
"dj-rest-auth==7.0.1",
|
||||
"django==5.1.15",
|
||||
"django-allauth==65.15.0",
|
||||
"django-celery-beat==2.9.0",
|
||||
"django-celery-results==2.6.0",
|
||||
"django-cors-headers==4.4.0",
|
||||
"django-environ==0.11.2",
|
||||
"django-filter==24.3",
|
||||
"django-guid==3.5.0",
|
||||
"django-postgres-extra==2.0.9",
|
||||
"django-silk==5.3.2",
|
||||
"django-timezone-field==7.2.1",
|
||||
"djangorestframework==3.15.2",
|
||||
"djangorestframework-jsonapi==7.0.2",
|
||||
"djangorestframework-simplejwt==5.5.1",
|
||||
"dnspython==2.8.0",
|
||||
"docker==7.1.0",
|
||||
"dogpile-cache==1.5.0",
|
||||
"dparse==0.6.4",
|
||||
"drf-extensions==0.8.0",
|
||||
"drf-nested-routers==0.95.0",
|
||||
"drf-simple-apikey==2.2.1",
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"dulwich==0.23.0",
|
||||
"duo-client==5.5.0",
|
||||
"durationpy==0.10",
|
||||
"email-validator==2.2.0",
|
||||
"execnet==2.1.2",
|
||||
"filelock==3.20.3",
|
||||
"flask==3.1.3",
|
||||
"fonttools==4.62.1",
|
||||
"freezegun==1.5.1",
|
||||
"frozenlist==1.8.0",
|
||||
"gevent==25.9.1",
|
||||
"google-api-core==2.29.0",
|
||||
"google-api-python-client==2.163.0",
|
||||
"google-auth==2.48.0",
|
||||
"google-auth-httplib2==0.2.0",
|
||||
"google-cloud-access-context-manager==0.3.0",
|
||||
"google-cloud-asset==4.2.0",
|
||||
"google-cloud-org-policy==1.16.0",
|
||||
"google-cloud-os-config==1.23.0",
|
||||
"google-cloud-resource-manager==1.16.0",
|
||||
"googleapis-common-protos==1.72.0",
|
||||
"gprof2dot==2025.4.14",
|
||||
"graphemeu==0.7.2",
|
||||
"greenlet==3.3.1",
|
||||
"grpc-google-iam-v1==0.14.3",
|
||||
"grpcio==1.76.0",
|
||||
"grpcio-status==1.76.0",
|
||||
"gunicorn==23.0.0",
|
||||
"h11==0.16.0",
|
||||
"h2==4.3.0",
|
||||
"hpack==4.1.0",
|
||||
"httpcore==1.0.9",
|
||||
"httplib2==0.31.2",
|
||||
"httpx==0.28.1",
|
||||
"humanfriendly==10.0",
|
||||
"hyperframe==6.1.0",
|
||||
"iamdata==0.1.202602021",
|
||||
"idna==3.11",
|
||||
"importlib-metadata==8.7.1",
|
||||
"inflection==0.5.1",
|
||||
"iniconfig==2.3.0",
|
||||
"iso8601==2.1.0",
|
||||
"isodate==0.7.2",
|
||||
"isort==5.13.2",
|
||||
"itsdangerous==2.2.0",
|
||||
"jinja2==3.1.6",
|
||||
"jiter==0.13.0",
|
||||
"jmespath==1.1.0",
|
||||
"joblib==1.5.3",
|
||||
"jsonpatch==1.33",
|
||||
"jsonpickle==4.1.1",
|
||||
"jsonpointer==3.0.0",
|
||||
"jsonschema==4.23.0",
|
||||
"jsonschema-specifications==2025.9.1",
|
||||
"keystoneauth1==5.13.0",
|
||||
"kiwisolver==1.4.9",
|
||||
"knack==0.11.0",
|
||||
"kombu==5.6.2",
|
||||
"kubernetes==32.0.1",
|
||||
"lxml==6.1.0",
|
||||
"lz4==4.4.5",
|
||||
"markdown==3.10.2",
|
||||
"markdown-it-py==4.0.0",
|
||||
"markupsafe==3.0.3",
|
||||
"marshmallow==4.3.0",
|
||||
"matplotlib==3.10.8",
|
||||
"mccabe==0.7.0",
|
||||
"mdurl==0.1.2",
|
||||
"microsoft-kiota-abstractions==1.9.9",
|
||||
"microsoft-kiota-authentication-azure==1.9.9",
|
||||
"microsoft-kiota-http==1.9.9",
|
||||
"microsoft-kiota-serialization-form==1.9.9",
|
||||
"microsoft-kiota-serialization-json==1.9.9",
|
||||
"microsoft-kiota-serialization-multipart==1.9.9",
|
||||
"microsoft-kiota-serialization-text==1.9.9",
|
||||
"microsoft-security-utilities-secret-masker==1.0.0b4",
|
||||
"msal==1.35.0b1",
|
||||
"msal-extensions==1.2.0",
|
||||
"msgraph-core==1.3.8",
|
||||
"msgraph-sdk==1.55.0",
|
||||
"msrest==0.7.1",
|
||||
"msrestazure==0.6.4.post1",
|
||||
"multidict==6.7.1",
|
||||
"mypy==1.10.1",
|
||||
"mypy-extensions==1.1.0",
|
||||
"narwhals==2.16.0",
|
||||
"neo4j==6.1.0",
|
||||
"nest-asyncio==1.6.0",
|
||||
"nltk==3.9.4",
|
||||
"numpy==2.0.2",
|
||||
"oauthlib==3.3.1",
|
||||
"oci==2.169.0",
|
||||
"openai==1.109.1",
|
||||
"openstacksdk==4.2.0",
|
||||
"opentelemetry-api==1.39.1",
|
||||
"opentelemetry-sdk==1.39.1",
|
||||
"opentelemetry-semantic-conventions==0.60b1",
|
||||
"os-service-types==1.8.2",
|
||||
"packageurl-python==0.17.6",
|
||||
"packaging==26.0",
|
||||
"pagerduty==6.1.0",
|
||||
"pandas==2.2.3",
|
||||
"pbr==7.0.3",
|
||||
"pillow==12.2.0",
|
||||
"pkginfo==1.12.1.2",
|
||||
"platformdirs==4.5.1",
|
||||
"plotly==6.5.2",
|
||||
"pluggy==1.6.0",
|
||||
"policyuniverse==1.5.1.20231109",
|
||||
"portalocker==2.10.1",
|
||||
"prek==0.3.9",
|
||||
"prompt-toolkit==3.0.52",
|
||||
"propcache==0.4.1",
|
||||
"proto-plus==1.27.0",
|
||||
"protobuf==6.33.5",
|
||||
"psutil==7.2.2",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"py-deviceid==0.1.1",
|
||||
"py-iam-expand==0.1.0",
|
||||
"py-ocsf-models==0.8.1",
|
||||
"pyasn1==0.6.3",
|
||||
"pyasn1-modules==0.4.2",
|
||||
"pycodestyle==2.14.0",
|
||||
"pycparser==3.0",
|
||||
"pydantic==2.12.5",
|
||||
"pydantic-core==2.41.5",
|
||||
"pygithub==2.8.0",
|
||||
"pygments==2.20.0",
|
||||
"pyjwt==2.12.1",
|
||||
"pylint==3.2.5",
|
||||
"pymsalruntime==0.18.1",
|
||||
"pynacl==1.6.2",
|
||||
"pyopenssl==26.0.0",
|
||||
"pyparsing==3.3.2",
|
||||
"pyreadline3==3.5.4",
|
||||
"pysocks==1.7.1",
|
||||
"pytest==9.0.3",
|
||||
"pytest-celery==1.3.0",
|
||||
"pytest-cov==5.0.0",
|
||||
"pytest-django==4.8.0",
|
||||
"pytest-docker-tools==3.1.9",
|
||||
"pytest-env==1.1.3",
|
||||
"pytest-randomly==3.15.0",
|
||||
"pytest-xdist==3.6.1",
|
||||
"python-crontab==3.3.0",
|
||||
"python-dateutil==2.9.0.post0",
|
||||
"python-digitalocean==1.17.0",
|
||||
"python3-saml==1.16.0",
|
||||
"pytz==2025.1",
|
||||
"pywin32==311",
|
||||
"pyyaml==6.0.3",
|
||||
"redis==7.1.0",
|
||||
"referencing==0.37.0",
|
||||
"regex==2026.1.15",
|
||||
"reportlab==4.4.10",
|
||||
"requests==2.33.1",
|
||||
"requests-file==3.0.1",
|
||||
"requests-oauthlib==2.0.0",
|
||||
"requestsexceptions==1.4.0",
|
||||
"retrying==1.4.2",
|
||||
"rich==14.3.2",
|
||||
"rpds-py==0.30.0",
|
||||
"rsa==4.9.1",
|
||||
"ruamel-yaml==0.19.1",
|
||||
"ruff==0.5.0",
|
||||
"s3transfer==0.14.0",
|
||||
"scaleway==2.10.3",
|
||||
"scaleway-core==2.10.3",
|
||||
"schema==0.7.5",
|
||||
"sentry-sdk==2.56.0",
|
||||
"setuptools==80.10.2",
|
||||
"shellingham==1.5.4",
|
||||
"shodan==1.31.0",
|
||||
"six==1.17.0",
|
||||
"slack-sdk==3.39.0",
|
||||
"sniffio==1.3.1",
|
||||
"sqlparse==0.5.5",
|
||||
"statsd==4.0.1",
|
||||
"std-uritemplate==2.0.8",
|
||||
"stevedore==5.6.0",
|
||||
"tabulate==0.9.0",
|
||||
"tenacity==9.1.2",
|
||||
"tldextract==5.3.1",
|
||||
"tomlkit==0.14.0",
|
||||
"tqdm==4.67.1",
|
||||
"typer==0.21.1",
|
||||
"types-aiobotocore-ecr==3.1.1",
|
||||
"typing-extensions==4.15.0",
|
||||
"typing-inspection==0.4.2",
|
||||
"tzdata==2025.3",
|
||||
"tzlocal==5.3.1",
|
||||
"uritemplate==4.2.0",
|
||||
"urllib3==2.7.0",
|
||||
"uuid6==2024.7.10",
|
||||
"vine==5.1.0",
|
||||
"vulture==2.14",
|
||||
"wcwidth==0.5.3",
|
||||
"websocket-client==1.9.0",
|
||||
"werkzeug==3.1.7",
|
||||
"workos==6.0.4",
|
||||
"wrapt==1.17.3",
|
||||
"xlsxwriter==3.2.9",
|
||||
"xmlsec==1.3.17",
|
||||
"xmltodict==1.0.2",
|
||||
"yarl==1.22.0",
|
||||
"zipp==3.23.0",
|
||||
"zope-event==6.1",
|
||||
"zope-interface==8.2",
|
||||
"zstd==1.5.7.3"
|
||||
]
|
||||
# prowler@master needs okta==3.4.2; cartography 0.135.0 declares okta<1.0.0 for an
|
||||
# integration prowler does not import.
|
||||
#
|
||||
# prowler@master hard-pins microsoft-kiota-abstractions==1.9.2 in [project.dependencies].
|
||||
# The microsoft-kiota-http security bump to 1.9.9 (GHSA-7j59-v9qr-6fq9) requires
|
||||
# microsoft-kiota-abstractions>=1.9.9, which a constraint cannot satisfy against the
|
||||
# SDK's hard pin; override it to the patched, kiota-aligned version.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2",
|
||||
"microsoft-kiota-abstractions==1.9.9"
|
||||
]
|
||||
|
||||
@@ -52,7 +52,7 @@ class ApiConfig(AppConfig):
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip Neo4j initialization during tests, some Django commands, and Celery
|
||||
# Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook)
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
@@ -64,7 +64,7 @@ class ApiConfig(AppConfig):
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping Neo4j initialization because tests, some Django commands or Celery"
|
||||
"Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)"
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
@@ -484,8 +484,8 @@ AWS_BEDROCK_PRIVESC_PASSROLE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust Bedrock service (can be passed to Bedrock)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock.amazonaws.com'}})
|
||||
// Find roles that trust the Bedrock AgentCore service (can be passed to a code interpreter)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
|
||||
WHERE any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
@@ -536,8 +536,8 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust Bedrock service (already attached to existing code interpreters)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock.amazonaws.com'}})
|
||||
// Find roles that trust the Bedrock AgentCore service (already attached to existing code interpreters)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
|
||||
|
||||
WITH collect(path_principal) + collect(path_target) AS paths
|
||||
UNWIND paths AS p
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.config.config import get_available_compliance_frameworks
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
@@ -95,12 +94,12 @@ PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""
|
||||
Retrieve and cache the list of available compliance frameworks for a specific cloud provider.
|
||||
"""List compliance frameworks the API can load for `provider_type`.
|
||||
|
||||
This function lazily loads and caches the available compliance frameworks (e.g., CIS, MITRE, ISO)
|
||||
for each provider type (AWS, Azure, GCP, etc.) on first access. Subsequent calls for the same
|
||||
provider will return the cached result.
|
||||
The list is sourced from `Compliance.get_bulk` so that the names
|
||||
returned here are guaranteed to be loadable by the bulk loader. This
|
||||
prevents downstream key mismatches (e.g. CSV report generation iterating
|
||||
framework names and looking them up in the bulk dict).
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type for which to retrieve
|
||||
@@ -112,8 +111,8 @@ def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[s
|
||||
"""
|
||||
global AVAILABLE_COMPLIANCE_FRAMEWORKS
|
||||
if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS:
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = (
|
||||
get_available_compliance_frameworks(provider_type)
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = list(
|
||||
Compliance.get_bulk(provider_type).keys()
|
||||
)
|
||||
|
||||
return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type]
|
||||
|
||||
@@ -330,6 +330,7 @@ class MembershipFilter(FilterSet):
|
||||
model = Membership
|
||||
fields = {
|
||||
"tenant": ["exact"],
|
||||
"user": ["exact"],
|
||||
"role": ["exact"],
|
||||
"date_joined": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Liveness and readiness endpoints following the IETF Health Check Response
|
||||
Format (draft-inadarei-api-health-check-06).
|
||||
|
||||
Liveness reports only process status. Readiness verifies that PostgreSQL,
|
||||
Valkey and Neo4j are reachable and returns per-dependency detail when any
|
||||
of them is unreachable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import redis
|
||||
from config.version import API_VERSION, RELEASE_ID
|
||||
from django.conf import settings
|
||||
from django.db import connections
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.renderers import JSONRenderer
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.throttling import ScopedRateThrottle
|
||||
from rest_framework.views import APIView
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SERVICE_ID = "prowler-api"
|
||||
SERVICE_DESCRIPTION = "Prowler API"
|
||||
|
||||
# Status vocabulary from the IETF draft (section 3.1).
|
||||
STATUS_PASS = "pass"
|
||||
STATUS_FAIL = "fail"
|
||||
STATUS_WARN = "warn"
|
||||
|
||||
# Short socket timeout so a stuck Valkey cannot stall the probe.
|
||||
# Neo4j inherits its driver-level ``connection_acquisition_timeout``.
|
||||
VALKEY_PROBE_TIMEOUT_SECONDS = 2
|
||||
|
||||
# Brief cache window so high-frequency probes (ALB target groups, scrapers)
|
||||
# do not stampede the actual dependency checks.
|
||||
CACHE_CONTROL_HEADER = "max-age=3, must-revalidate"
|
||||
|
||||
# In-process readiness cache. Caps real dependency hits to roughly
|
||||
# (gunicorn workers / TTL) per second regardless of incoming RPS or the
|
||||
# source-IP distribution. Kept in sync with the Cache-Control max-age.
|
||||
# Access is guarded by a lock so concurrent readers do not race on the
|
||||
# read-decide-write cycle of the double-checked locking pattern below.
|
||||
READINESS_CACHE_TTL_SECONDS = 3.0
|
||||
_readiness_cache: tuple[float, dict[str, Any], int] | None = None
|
||||
_readiness_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
class HealthJSONRenderer(JSONRenderer):
|
||||
"""Emits responses with the ``application/health+json`` content type."""
|
||||
|
||||
media_type = "application/health+json"
|
||||
format = "health"
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return (
|
||||
datetime.now(timezone.utc)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def _measure(name: str, check_fn) -> tuple[dict[str, Any], float]:
|
||||
"""Time ``check_fn`` and return ``(result, elapsed_ms)``.
|
||||
|
||||
``check_fn`` returns ``None`` on success or raises on failure. The full
|
||||
exception is logged for operator diagnostics under ``name``; the
|
||||
response payload intentionally omits the error detail to avoid leaking
|
||||
infrastructure information (DNS names, ports, credentials, certificate
|
||||
chains) to anonymous clients.
|
||||
"""
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
check_fn()
|
||||
except Exception:
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
logger.warning("Health probe '%s' failed", name, exc_info=True)
|
||||
return ({"status": STATUS_FAIL}, elapsed_ms)
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
return ({"status": STATUS_PASS}, elapsed_ms)
|
||||
|
||||
|
||||
def _probe_postgres() -> None:
|
||||
with connections["default"].cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
cursor.fetchone()
|
||||
|
||||
|
||||
def _probe_valkey() -> None:
|
||||
client = redis.Redis.from_url(
|
||||
settings.CELERY_BROKER_URL,
|
||||
socket_connect_timeout=VALKEY_PROBE_TIMEOUT_SECONDS,
|
||||
socket_timeout=VALKEY_PROBE_TIMEOUT_SECONDS,
|
||||
)
|
||||
try:
|
||||
if not client.ping():
|
||||
raise RuntimeError("PING did not return PONG")
|
||||
finally:
|
||||
# Best-effort cleanup: a failure releasing the socket (e.g. broken
|
||||
# connection, half-closed by the server) must not mask the probe
|
||||
# result. Narrowed to the exception types redis-py and the stdlib
|
||||
# socket layer can raise on close.
|
||||
with suppress(redis.RedisError, OSError):
|
||||
client.close()
|
||||
|
||||
|
||||
def _probe_neo4j() -> None:
|
||||
# Lazy import: avoids pulling attack_paths into the boot import graph.
|
||||
from api.attack_paths.database import get_driver
|
||||
|
||||
get_driver().verify_connectivity()
|
||||
|
||||
|
||||
def _build_check_entry(
|
||||
component_id: str,
|
||||
component_type: str,
|
||||
result: dict[str, Any],
|
||||
elapsed_ms: float,
|
||||
) -> dict[str, Any]:
|
||||
entry: dict[str, Any] = {
|
||||
"componentId": component_id,
|
||||
"componentType": component_type,
|
||||
"observedValue": round(elapsed_ms, 2),
|
||||
"observedUnit": "ms",
|
||||
"status": result["status"],
|
||||
"time": _now_iso(),
|
||||
}
|
||||
if "output" in result:
|
||||
entry["output"] = result["output"]
|
||||
return entry
|
||||
|
||||
|
||||
def _aggregate_status(check_entries: list[dict[str, Any]]) -> str:
|
||||
statuses = {entry["status"] for entry in check_entries}
|
||||
if STATUS_FAIL in statuses:
|
||||
return STATUS_FAIL
|
||||
if STATUS_WARN in statuses:
|
||||
return STATUS_WARN
|
||||
return STATUS_PASS
|
||||
|
||||
|
||||
def _base_payload(overall_status: str) -> dict[str, Any]:
|
||||
return {
|
||||
"status": overall_status,
|
||||
"version": API_VERSION,
|
||||
"releaseId": RELEASE_ID,
|
||||
"serviceId": SERVICE_ID,
|
||||
"description": SERVICE_DESCRIPTION,
|
||||
}
|
||||
|
||||
|
||||
def _readiness_payload() -> tuple[dict[str, Any], int]:
|
||||
global _readiness_cache
|
||||
|
||||
# Lock-free fast path: a stale snapshot still satisfies the freshness
|
||||
# check correctly because we re-check after acquiring the lock below.
|
||||
snapshot = _readiness_cache
|
||||
if (
|
||||
snapshot is not None
|
||||
and time.monotonic() - snapshot[0] < READINESS_CACHE_TTL_SECONDS
|
||||
):
|
||||
return snapshot[1], snapshot[2]
|
||||
|
||||
with _readiness_cache_lock:
|
||||
# Double-checked locking: another thread may have refreshed while
|
||||
# we were waiting on the lock.
|
||||
snapshot = _readiness_cache
|
||||
if (
|
||||
snapshot is not None
|
||||
and time.monotonic() - snapshot[0] < READINESS_CACHE_TTL_SECONDS
|
||||
):
|
||||
return snapshot[1], snapshot[2]
|
||||
|
||||
postgres_result, postgres_ms = _measure("postgres", _probe_postgres)
|
||||
valkey_result, valkey_ms = _measure("valkey", _probe_valkey)
|
||||
neo4j_result, neo4j_ms = _measure("neo4j", _probe_neo4j)
|
||||
|
||||
entries = [
|
||||
_build_check_entry("postgres", "datastore", postgres_result, postgres_ms),
|
||||
_build_check_entry("valkey", "datastore", valkey_result, valkey_ms),
|
||||
_build_check_entry("neo4j", "datastore", neo4j_result, neo4j_ms),
|
||||
]
|
||||
overall = _aggregate_status(entries)
|
||||
|
||||
payload = _base_payload(overall)
|
||||
payload["checks"] = {
|
||||
"postgres:responseTime": [entries[0]],
|
||||
"valkey:responseTime": [entries[1]],
|
||||
"neo4j:responseTime": [entries[2]],
|
||||
}
|
||||
|
||||
http_status = (
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
if overall == STATUS_FAIL
|
||||
else status.HTTP_200_OK
|
||||
)
|
||||
_readiness_cache = (time.monotonic(), payload, http_status)
|
||||
return payload, http_status
|
||||
|
||||
|
||||
def _health_response(payload: dict[str, Any], http_status: int) -> Response:
|
||||
response = Response(payload, status=http_status)
|
||||
response["Cache-Control"] = CACHE_CONTROL_HEADER
|
||||
return response
|
||||
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
class LivenessView(APIView):
|
||||
"""Liveness probe. Always 200 when the process can serve requests.
|
||||
|
||||
Dependencies are intentionally not consulted: a failing liveness probe
|
||||
triggers a container restart, which must not happen for transient
|
||||
dependency outages. Throttled per-IP so the endpoint cannot be used as
|
||||
a cheap availability oracle for the process.
|
||||
"""
|
||||
|
||||
authentication_classes: list = []
|
||||
permission_classes: list = []
|
||||
renderer_classes = [HealthJSONRenderer]
|
||||
throttle_classes = [ScopedRateThrottle]
|
||||
throttle_scope = "health-live"
|
||||
|
||||
def get(self, _request, *_args, **_kwargs):
|
||||
return _health_response(_base_payload(STATUS_PASS), status.HTTP_200_OK)
|
||||
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
class ReadinessView(APIView):
|
||||
"""Readiness probe.
|
||||
|
||||
Returns 200 when PostgreSQL, Valkey and Neo4j all respond, or 503 with
|
||||
per-dependency detail when any of them is unreachable. Per-IP throttle
|
||||
plus the short in-process result cache cap the real dependency hits
|
||||
regardless of inbound traffic shape.
|
||||
"""
|
||||
|
||||
authentication_classes: list = []
|
||||
permission_classes: list = []
|
||||
renderer_classes = [HealthJSONRenderer]
|
||||
throttle_classes = [ScopedRateThrottle]
|
||||
throttle_scope = "health-ready"
|
||||
|
||||
def get(self, _request, *_args, **_kwargs):
|
||||
payload, http_status = _readiness_payload()
|
||||
return _health_response(payload, http_status)
|
||||
@@ -0,0 +1,31 @@
|
||||
from functools import partial
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
from api.db_utils import create_index_on_partitions, drop_index_on_partitions
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
("api", "0090_attack_paths_cleanup_priority"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(
|
||||
partial(
|
||||
create_index_on_partitions,
|
||||
parent_table="findings",
|
||||
index_name="gin_find_arrays_idx",
|
||||
columns="categories, resource_services, resource_regions, resource_types",
|
||||
method="GIN",
|
||||
all_partitions=True,
|
||||
),
|
||||
reverse_code=partial(
|
||||
drop_index_on_partitions,
|
||||
parent_table="findings",
|
||||
index_name="gin_find_arrays_idx",
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,73 @@
|
||||
import django.contrib.postgres.indexes
|
||||
from django.db import migrations
|
||||
|
||||
INDEX_NAME = "gin_find_arrays_idx"
|
||||
PARENT_TABLE = "findings"
|
||||
|
||||
|
||||
def create_parent_and_attach(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
# Idempotent: the parent index may already exist if it was created
|
||||
# manually on an environment before this migration ran.
|
||||
cursor.execute(
|
||||
f"CREATE INDEX IF NOT EXISTS {INDEX_NAME} ON ONLY {PARENT_TABLE} "
|
||||
f"USING gin (categories, resource_services, resource_regions, resource_types)"
|
||||
)
|
||||
cursor.execute(
|
||||
"SELECT inhrelid::regclass::text "
|
||||
"FROM pg_inherits "
|
||||
"WHERE inhparent = %s::regclass",
|
||||
[PARENT_TABLE],
|
||||
)
|
||||
for (partition,) in cursor.fetchall():
|
||||
child_idx = f"{partition.replace('.', '_')}_{INDEX_NAME}"
|
||||
# ALTER INDEX ... ATTACH PARTITION has no IF NOT ATTACHED clause,
|
||||
# so check pg_inherits first to keep the migration re-runnable.
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class p ON p.oid = i.inhparent
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE p.relname = %s AND c.relname = %s
|
||||
""",
|
||||
[INDEX_NAME, child_idx],
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
cursor.execute(f"ALTER INDEX {INDEX_NAME} ATTACH PARTITION {child_idx}")
|
||||
|
||||
|
||||
def drop_parent_index(apps, schema_editor):
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}")
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0091_findings_arrays_gin_index_partitions"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.SeparateDatabaseAndState(
|
||||
state_operations=[
|
||||
migrations.AddIndex(
|
||||
model_name="finding",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=[
|
||||
"categories",
|
||||
"resource_services",
|
||||
"resource_regions",
|
||||
"resource_types",
|
||||
],
|
||||
name=INDEX_NAME,
|
||||
),
|
||||
),
|
||||
],
|
||||
database_operations=[
|
||||
migrations.RunPython(
|
||||
create_parent_and_attach,
|
||||
reverse_code=drop_parent_index,
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -595,10 +595,40 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
|
||||
_SCOPING_SCANNER_ARG_KEYS_CACHE: tuple[str, ...] | None = None
|
||||
|
||||
@classmethod
|
||||
def get_scoping_scanner_arg_keys(cls) -> tuple[str, ...]:
|
||||
"""Return the scanner_args keys that mark a scan as scoped.
|
||||
|
||||
Derived from ``prowler.lib.scan.scan.Scan.__init__`` so the API stays
|
||||
in sync with whatever the SDK actually accepts as filters. Cached at
|
||||
class level — the signature is stable for the process lifetime.
|
||||
"""
|
||||
if cls._SCOPING_SCANNER_ARG_KEYS_CACHE is None:
|
||||
import inspect
|
||||
|
||||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
|
||||
params = inspect.signature(ProwlerScan.__init__).parameters
|
||||
cls._SCOPING_SCANNER_ARG_KEYS_CACHE = tuple(
|
||||
name for name in params if name not in ("self", "provider")
|
||||
)
|
||||
return cls._SCOPING_SCANNER_ARG_KEYS_CACHE
|
||||
|
||||
class TriggerChoices(models.TextChoices):
|
||||
SCHEDULED = "scheduled", _("Scheduled")
|
||||
MANUAL = "manual", _("Manual")
|
||||
|
||||
# Trigger values for scans that ran the SDK end-to-end. Imported scans (or
|
||||
# any future trigger) are intentionally NOT in this set — they may carry
|
||||
# only a partial slice of resources, so post-scan logic that depends on a
|
||||
# full-scope sweep (e.g. resetting ephemeral resource findings) must skip
|
||||
# them by default.
|
||||
LIVE_SCAN_TRIGGERS = frozenset(
|
||||
(TriggerChoices.SCHEDULED.value, TriggerChoices.MANUAL.value)
|
||||
)
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
|
||||
name = models.CharField(
|
||||
blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)]
|
||||
@@ -681,6 +711,24 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scans"
|
||||
|
||||
def is_full_scope(self) -> bool:
|
||||
"""Return True if this scan ran with no scoping filters at all.
|
||||
|
||||
Used to gate post-scan operations (such as resetting the
|
||||
failed_findings_count of resources missing from the scan) that are only
|
||||
safe when the scan covered every check, service, and category. Imported
|
||||
scans are NOT full-scope by definition — they may carry only a partial
|
||||
slice of resources, so they're rejected via ``trigger`` even before the
|
||||
scanner_args check.
|
||||
"""
|
||||
if self.trigger not in self.LIVE_SCAN_TRIGGERS:
|
||||
return False
|
||||
scanner_args = self.scanner_args or {}
|
||||
for key in self.get_scoping_scanner_arg_keys():
|
||||
if scanner_args.get(key):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
@@ -898,7 +946,6 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
OpClass(Upper("name"), name="gin_trgm_ops"),
|
||||
name="res_name_trgm_idx",
|
||||
),
|
||||
GinIndex(fields=["text_search"], name="gin_resources_search_idx"),
|
||||
models.Index(fields=["tenant_id", "id"], name="resources_tenant_id_idx"),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
@@ -1104,6 +1151,15 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
|
||||
fields=["tenant_id", "scan_id", "check_id"],
|
||||
name="find_tenant_scan_check_idx",
|
||||
),
|
||||
GinIndex(
|
||||
fields=[
|
||||
"categories",
|
||||
"resource_services",
|
||||
"resource_regions",
|
||||
"resource_types",
|
||||
],
|
||||
name="gin_find_arrays_idx",
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
|
||||
+1620
-58
File diff suppressed because it is too large
Load Diff
@@ -1,13 +1,18 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from api import compliance as compliance_module
|
||||
from api.compliance import (
|
||||
generate_compliance_overview_template,
|
||||
generate_scan_compliance,
|
||||
get_compliance_frameworks,
|
||||
get_prowler_provider_checks,
|
||||
get_prowler_provider_compliance,
|
||||
load_prowler_checks,
|
||||
)
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
|
||||
|
||||
class TestCompliance:
|
||||
@@ -250,3 +255,58 @@ class TestCompliance:
|
||||
}
|
||||
|
||||
assert template == expected_template
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_compliance_cache():
|
||||
"""Reset the module-level cache so each test starts cold."""
|
||||
previous = dict(compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS)
|
||||
compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.clear()
|
||||
compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.update(previous)
|
||||
|
||||
|
||||
class TestGetComplianceFrameworks:
|
||||
def test_returns_keys_from_compliance_get_bulk(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
"cis_1.4_aws": MagicMock(),
|
||||
"mitre_attack_aws": MagicMock(),
|
||||
}
|
||||
result = get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
assert sorted(result) == ["cis_1.4_aws", "mitre_attack_aws"]
|
||||
mock_compliance.get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
|
||||
def test_caches_result_per_provider(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
# Cached after first call.
|
||||
assert mock_compliance.get_bulk.call_count == 1
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_type",
|
||||
[choice.value for choice in Provider.ProviderChoices],
|
||||
)
|
||||
def test_listing_is_subset_of_bulk(self, reset_compliance_cache, provider_type):
|
||||
"""Regression for CLOUD-API-40S: every name returned by
|
||||
``get_compliance_frameworks`` must be loadable via ``Compliance.get_bulk``.
|
||||
|
||||
A divergence here is what produced ``KeyError: 'csa_ccm_4.0'`` in
|
||||
``generate_outputs_task`` after universal/multi-provider compliance
|
||||
JSONs were introduced at the top-level ``prowler/compliance/`` path.
|
||||
"""
|
||||
bulk_keys = set(Compliance.get_bulk(provider_type).keys())
|
||||
listed = set(get_compliance_frameworks(provider_type))
|
||||
|
||||
missing = listed - bulk_keys
|
||||
assert not missing, (
|
||||
f"get_compliance_frameworks({provider_type!r}) returned names not "
|
||||
f"loadable by Compliance.get_bulk: {sorted(missing)}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
"""Tests for the health endpoints.
|
||||
|
||||
Cover the IETF response envelope, status code mapping (200 / 503), the
|
||||
``application/health+json`` media type and per-probe failure modes.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from config import version as config_version
|
||||
from django.core.cache import cache
|
||||
from django.urls import reverse
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from api import health
|
||||
|
||||
|
||||
HEALTH_MEDIA_TYPE = "application/health+json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_health_state():
|
||||
"""Per-test isolation: clear throttle counters and the readiness cache.
|
||||
|
||||
DRF's ScopedRateThrottle persists state in Django's cache; without
|
||||
clearing it the throttle budget would be shared across tests and trip
|
||||
midway through the suite.
|
||||
"""
|
||||
cache.clear()
|
||||
health._readiness_cache = None
|
||||
yield
|
||||
cache.clear()
|
||||
health._readiness_cache = None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client():
|
||||
return APIClient()
|
||||
|
||||
|
||||
def _assert_health_envelope(body):
|
||||
"""Every health response must carry the RFC top-level descriptors."""
|
||||
assert body["version"] == config_version.API_VERSION
|
||||
assert body["releaseId"] == config_version.RELEASE_ID
|
||||
assert body["serviceId"] == health.SERVICE_ID
|
||||
assert body["description"] == health.SERVICE_DESCRIPTION
|
||||
|
||||
|
||||
class TestLivenessEndpoint:
|
||||
def test_returns_200_with_pass_status(self, api_client):
|
||||
response = api_client.get(reverse("health-live"))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"].startswith(HEALTH_MEDIA_TYPE)
|
||||
assert response["Cache-Control"] == health.CACHE_CONTROL_HEADER
|
||||
body = response.json()
|
||||
assert body["status"] == "pass"
|
||||
_assert_health_envelope(body)
|
||||
|
||||
def test_does_not_require_authentication(self, api_client):
|
||||
api_client.credentials()
|
||||
|
||||
response = api_client.get(reverse("health-live"))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
def test_does_not_run_dependency_checks(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres") as mock_pg,
|
||||
patch("api.health._probe_valkey") as mock_vk,
|
||||
patch("api.health._probe_neo4j") as mock_neo,
|
||||
):
|
||||
response = api_client.get(reverse("health-live"))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
mock_pg.assert_not_called()
|
||||
mock_vk.assert_not_called()
|
||||
mock_neo.assert_not_called()
|
||||
|
||||
|
||||
class TestReadinessEndpoint:
|
||||
@staticmethod
|
||||
def _patch_probes():
|
||||
return (
|
||||
patch("api.health._probe_postgres", return_value=None),
|
||||
patch("api.health._probe_valkey", return_value=None),
|
||||
patch("api.health._probe_neo4j", return_value=None),
|
||||
)
|
||||
|
||||
def test_returns_200_and_pass_when_all_dependencies_healthy(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres"),
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"].startswith(HEALTH_MEDIA_TYPE)
|
||||
assert response["Cache-Control"] == health.CACHE_CONTROL_HEADER
|
||||
|
||||
body = response.json()
|
||||
_assert_health_envelope(body)
|
||||
assert body["status"] == "pass"
|
||||
|
||||
# Per RFC, `checks` values are arrays of one or more measurement
|
||||
# objects. We use a single measurement per dependency.
|
||||
assert set(body["checks"].keys()) == {
|
||||
"postgres:responseTime",
|
||||
"valkey:responseTime",
|
||||
"neo4j:responseTime",
|
||||
}
|
||||
for key in body["checks"]:
|
||||
entries = body["checks"][key]
|
||||
assert isinstance(entries, list) and len(entries) == 1
|
||||
entry = entries[0]
|
||||
assert entry["status"] == "pass"
|
||||
assert entry["componentType"] == "datastore"
|
||||
assert entry["observedUnit"] == "ms"
|
||||
assert isinstance(entry["observedValue"], (int, float))
|
||||
assert entry["observedValue"] >= 0
|
||||
assert "time" in entry
|
||||
# `output` must not leak when the check passed.
|
||||
assert "output" not in entry
|
||||
|
||||
def test_returns_503_and_fail_when_postgres_is_down(self, api_client):
|
||||
with (
|
||||
patch(
|
||||
"api.health._probe_postgres",
|
||||
side_effect=RuntimeError("connection refused"),
|
||||
),
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
body = response.json()
|
||||
assert body["status"] == "fail"
|
||||
pg_entry = body["checks"]["postgres:responseTime"][0]
|
||||
assert pg_entry["status"] == "fail"
|
||||
# Exception detail is never echoed in the response, only logged.
|
||||
assert "output" not in pg_entry
|
||||
assert body["checks"]["valkey:responseTime"][0]["status"] == "pass"
|
||||
assert body["checks"]["neo4j:responseTime"][0]["status"] == "pass"
|
||||
|
||||
def test_returns_503_and_fail_when_valkey_is_down(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres"),
|
||||
patch("api.health._probe_valkey", side_effect=ConnectionError("timeout")),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
body = response.json()
|
||||
assert body["status"] == "fail"
|
||||
vk_entry = body["checks"]["valkey:responseTime"][0]
|
||||
assert vk_entry["status"] == "fail"
|
||||
assert "output" not in vk_entry
|
||||
|
||||
def test_returns_503_and_fail_when_neo4j_is_down(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres"),
|
||||
patch("api.health._probe_valkey"),
|
||||
patch(
|
||||
"api.health._probe_neo4j",
|
||||
side_effect=RuntimeError("ServiceUnavailable"),
|
||||
),
|
||||
):
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
body = response.json()
|
||||
assert body["status"] == "fail"
|
||||
neo_entry = body["checks"]["neo4j:responseTime"][0]
|
||||
assert neo_entry["status"] == "fail"
|
||||
assert "output" not in neo_entry
|
||||
|
||||
def test_reports_all_failures_simultaneously(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres", side_effect=RuntimeError("pg down")),
|
||||
patch("api.health._probe_valkey", side_effect=RuntimeError("vk down")),
|
||||
patch("api.health._probe_neo4j", side_effect=RuntimeError("neo down")),
|
||||
):
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
body = response.json()
|
||||
assert body["status"] == "fail"
|
||||
for key in (
|
||||
"postgres:responseTime",
|
||||
"valkey:responseTime",
|
||||
"neo4j:responseTime",
|
||||
):
|
||||
entry = body["checks"][key][0]
|
||||
assert entry["status"] == "fail"
|
||||
# No dependency-specific error string leaks into the payload.
|
||||
assert "output" not in entry
|
||||
|
||||
def test_does_not_leak_exception_detail_on_failure(self, api_client):
|
||||
# Sanity check: an exception message resembling infra detail
|
||||
# (host, port, credentials) must not surface in the response under
|
||||
# any field.
|
||||
sensitive = (
|
||||
"connection to server at "
|
||||
'"postgres-rw.prod.svc.cluster.local" (10.0.0.5), port 5432 '
|
||||
'failed: FATAL: password authentication failed for user "prowler_user"'
|
||||
)
|
||||
with (
|
||||
patch("api.health._probe_postgres", side_effect=RuntimeError(sensitive)),
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
body = response.json()
|
||||
assert "output" not in body["checks"]["postgres:responseTime"][0]
|
||||
payload_text = response.content.decode()
|
||||
for token in (
|
||||
"postgres-rw",
|
||||
"10.0.0.5",
|
||||
"5432",
|
||||
"prowler_user",
|
||||
"password authentication failed",
|
||||
):
|
||||
assert token not in payload_text
|
||||
|
||||
def test_does_not_require_authentication(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres"),
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
api_client.credentials()
|
||||
response = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
|
||||
class TestReadinessCache:
|
||||
"""In-process cache caps the rate at which real probes hit the deps."""
|
||||
|
||||
def test_result_is_cached_for_ttl_seconds(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres") as pg,
|
||||
patch("api.health._probe_valkey") as vk,
|
||||
patch("api.health._probe_neo4j") as neo,
|
||||
):
|
||||
r1 = api_client.get(reverse("health-ready"))
|
||||
r2 = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert r1.status_code == status.HTTP_200_OK
|
||||
assert r2.status_code == status.HTTP_200_OK
|
||||
# Second request must not trigger fresh dep checks within the TTL.
|
||||
assert pg.call_count == 1
|
||||
assert vk.call_count == 1
|
||||
assert neo.call_count == 1
|
||||
# The cached payload is returned verbatim (same timestamps too).
|
||||
assert r1.json() == r2.json()
|
||||
|
||||
def test_re_probes_after_cache_ttl_expires(self, api_client):
|
||||
with (
|
||||
patch("api.health._probe_postgres") as pg,
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
api_client.get(reverse("health-ready"))
|
||||
assert pg.call_count == 1
|
||||
|
||||
# Rewind the cached timestamp past the TTL so the next request
|
||||
# is forced to recompute.
|
||||
cached_ts, payload, http_status_code = health._readiness_cache
|
||||
health._readiness_cache = (
|
||||
cached_ts - health.READINESS_CACHE_TTL_SECONDS - 0.1,
|
||||
payload,
|
||||
http_status_code,
|
||||
)
|
||||
api_client.get(reverse("health-ready"))
|
||||
|
||||
assert pg.call_count == 2
|
||||
|
||||
def test_cache_persists_a_failing_result(self, api_client):
|
||||
# A failing readiness result is cached too; this is intentional so
|
||||
# an attacker spamming the endpoint during an outage cannot amplify
|
||||
# the dependency load.
|
||||
with (
|
||||
patch("api.health._probe_postgres", side_effect=RuntimeError("down")) as pg,
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
):
|
||||
r1 = api_client.get(reverse("health-ready"))
|
||||
r2 = api_client.get(reverse("health-ready"))
|
||||
|
||||
assert r1.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
assert r2.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
assert pg.call_count == 1
|
||||
|
||||
|
||||
class TestRateLimiting:
|
||||
"""The endpoints are unauthenticated and exposed; per-IP throttle caps
|
||||
naive single-source floods."""
|
||||
|
||||
def test_live_blocks_after_budget_exhausted(self, api_client):
|
||||
# Shrink the budget to 3 req per window so the test stays fast and
|
||||
# deterministic. parse_rate runs once per throttle instance and
|
||||
# each request gets a fresh instance, so this patch propagates.
|
||||
from rest_framework.throttling import ScopedRateThrottle
|
||||
|
||||
with patch.object(ScopedRateThrottle, "parse_rate", return_value=(3, 60)):
|
||||
statuses = [
|
||||
api_client.get(reverse("health-live")).status_code for _ in range(4)
|
||||
]
|
||||
|
||||
assert statuses[:3] == [status.HTTP_200_OK] * 3
|
||||
assert statuses[3] == status.HTTP_429_TOO_MANY_REQUESTS
|
||||
|
||||
def test_ready_blocks_after_budget_exhausted(self, api_client):
|
||||
from rest_framework.throttling import ScopedRateThrottle
|
||||
|
||||
with (
|
||||
patch("api.health._probe_postgres"),
|
||||
patch("api.health._probe_valkey"),
|
||||
patch("api.health._probe_neo4j"),
|
||||
patch.object(ScopedRateThrottle, "parse_rate", return_value=(2, 60)),
|
||||
):
|
||||
statuses = [
|
||||
api_client.get(reverse("health-ready")).status_code for _ in range(3)
|
||||
]
|
||||
|
||||
assert statuses[:2] == [status.HTTP_200_OK] * 2
|
||||
assert statuses[2] == status.HTTP_429_TOO_MANY_REQUESTS
|
||||
|
||||
|
||||
class TestProbeImplementations:
|
||||
"""Smoke tests for each probe primitive."""
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_postgres_probe_succeeds_against_real_db(self):
|
||||
assert health._probe_postgres() is None
|
||||
|
||||
def test_postgres_probe_propagates_db_errors(self):
|
||||
class _BoomCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
return False
|
||||
|
||||
def execute(self, *_args, **_kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
def fetchone(self): # pragma: no cover - never reached
|
||||
return None
|
||||
|
||||
with patch("api.health.connections") as mock_connections:
|
||||
mock_connections.__getitem__.return_value.cursor.return_value = (
|
||||
_BoomCursor()
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
health._probe_postgres()
|
||||
|
||||
def test_valkey_probe_succeeds_when_ping_returns_true(self):
|
||||
with patch("api.health.redis.Redis.from_url") as mock_from_url:
|
||||
mock_from_url.return_value.ping.return_value = True
|
||||
assert health._probe_valkey() is None
|
||||
|
||||
def test_valkey_probe_raises_when_ping_returns_false(self):
|
||||
with patch("api.health.redis.Redis.from_url") as mock_from_url:
|
||||
mock_from_url.return_value.ping.return_value = False
|
||||
with pytest.raises(RuntimeError, match="PING"):
|
||||
health._probe_valkey()
|
||||
|
||||
def test_valkey_probe_propagates_connection_errors(self):
|
||||
with patch("api.health.redis.Redis.from_url") as mock_from_url:
|
||||
mock_from_url.return_value.ping.side_effect = ConnectionError("nope")
|
||||
with pytest.raises(ConnectionError, match="nope"):
|
||||
health._probe_valkey()
|
||||
|
||||
def test_valkey_probe_suppresses_redis_error_on_close(self):
|
||||
# A redis-py-level failure releasing the socket must not mask a
|
||||
# successful PING (best-effort cleanup contract).
|
||||
import redis as redis_pkg
|
||||
|
||||
with patch("api.health.redis.Redis.from_url") as mock_from_url:
|
||||
client = mock_from_url.return_value
|
||||
client.ping.return_value = True
|
||||
client.close.side_effect = redis_pkg.RedisError("connection reset")
|
||||
|
||||
assert health._probe_valkey() is None
|
||||
|
||||
client.close.assert_called_once_with()
|
||||
|
||||
def test_valkey_probe_suppresses_oserror_on_close(self):
|
||||
# Socket-layer failures (OSError family) on close are also part of
|
||||
# the swallowed scope.
|
||||
with patch("api.health.redis.Redis.from_url") as mock_from_url:
|
||||
client = mock_from_url.return_value
|
||||
client.ping.return_value = True
|
||||
client.close.side_effect = OSError("EBADF")
|
||||
|
||||
assert health._probe_valkey() is None
|
||||
|
||||
client.close.assert_called_once_with()
|
||||
|
||||
def test_valkey_probe_lets_unexpected_close_errors_propagate(self):
|
||||
# The suppress() is deliberately narrow: anything outside
|
||||
# (redis.RedisError, OSError) must surface so it is not silently
|
||||
# hidden.
|
||||
with patch("api.health.redis.Redis.from_url") as mock_from_url:
|
||||
client = mock_from_url.return_value
|
||||
client.ping.return_value = True
|
||||
client.close.side_effect = RuntimeError("bug")
|
||||
|
||||
with pytest.raises(RuntimeError, match="bug"):
|
||||
health._probe_valkey()
|
||||
|
||||
def test_neo4j_probe_calls_verify_connectivity(self):
|
||||
with patch("api.attack_paths.database.get_driver") as mock_get_driver:
|
||||
mock_get_driver.return_value.verify_connectivity.return_value = None
|
||||
assert health._probe_neo4j() is None
|
||||
mock_get_driver.return_value.verify_connectivity.assert_called_once_with()
|
||||
|
||||
def test_neo4j_probe_propagates_driver_errors(self):
|
||||
with patch("api.attack_paths.database.get_driver") as mock_get_driver:
|
||||
mock_get_driver.return_value.verify_connectivity.side_effect = RuntimeError(
|
||||
"unreachable"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="unreachable"):
|
||||
health._probe_neo4j()
|
||||
|
||||
|
||||
class TestStatusAggregation:
|
||||
def test_pass_when_all_checks_pass(self):
|
||||
entries = [{"status": "pass"}, {"status": "pass"}]
|
||||
assert health._aggregate_status(entries) == "pass"
|
||||
|
||||
def test_warn_when_any_check_warns_and_none_fail(self):
|
||||
entries = [{"status": "pass"}, {"status": "warn"}]
|
||||
assert health._aggregate_status(entries) == "warn"
|
||||
|
||||
def test_fail_when_any_check_fails(self):
|
||||
entries = [{"status": "pass"}, {"status": "warn"}, {"status": "fail"}]
|
||||
assert health._aggregate_status(entries) == "fail"
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Drift checks for the API version constants.
|
||||
|
||||
Guarantee that ``config.version`` always reflects the canonical
|
||||
``[project].version`` declared in ``api/pyproject.toml``.
|
||||
"""
|
||||
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from config import version as config_version
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pyproject_data():
|
||||
here = Path(__file__).resolve()
|
||||
for directory in here.parents:
|
||||
candidate = directory / "pyproject.toml"
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
with candidate.open("rb") as f:
|
||||
data = tomllib.load(f)
|
||||
if data.get("project", {}).get("name") == "prowler-api":
|
||||
return data
|
||||
raise AssertionError("api/pyproject.toml not reachable from the test runner")
|
||||
|
||||
|
||||
def test_release_id_matches_pyproject(pyproject_data):
|
||||
assert config_version.RELEASE_ID == pyproject_data["project"]["version"]
|
||||
|
||||
|
||||
def test_api_version_is_major_of_release_id():
|
||||
assert config_version.API_VERSION == config_version.RELEASE_ID.split(".", 1)[0]
|
||||
assert config_version.API_VERSION.isdigit()
|
||||
|
||||
|
||||
def test_api_version_matches_v1_url_prefix():
|
||||
# The public contract version surfaced in the health payload must match
|
||||
# the URL namespace the API is published under.
|
||||
assert config_version.API_VERSION == "1"
|
||||
@@ -32,6 +32,11 @@ from django_celery_results.models import TaskResult
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import PermissionDenied
|
||||
from rest_framework.response import Response
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
|
||||
from api.attack_paths import (
|
||||
AttackPathsQueryDefinition,
|
||||
@@ -47,6 +52,7 @@ from api.models import (
|
||||
Finding,
|
||||
Integration,
|
||||
Invitation,
|
||||
InvitationRoleRelationship,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
LighthouseTenantConfiguration,
|
||||
@@ -746,6 +752,39 @@ class TestTenantViewSet:
|
||||
# Test user + 2 extra users for tenant 2
|
||||
assert len(response.json()["data"]) == 3
|
||||
|
||||
def test_tenants_list_memberships_filter_by_user(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
_, tenant2, _ = tenants_fixture
|
||||
_, user3_membership = extra_users
|
||||
user3, membership3 = user3_membership
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("tenant-membership-list", kwargs={"tenant_pk": tenant2.id}),
|
||||
{"filter[user]": str(user3.id)},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert len(data) == 1
|
||||
assert data[0]["id"] == str(membership3.id)
|
||||
|
||||
def test_tenants_list_memberships_filter_by_user_no_match(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
_, tenant2, _ = tenants_fixture
|
||||
unrelated_user = User.objects.create_user(
|
||||
name="unrelated",
|
||||
password=TEST_PASSWORD,
|
||||
email="unrelated@gmail.com",
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("tenant-membership-list", kwargs={"tenant_pk": tenant2.id}),
|
||||
{"filter[user]": str(unrelated_user.id)},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["data"] == []
|
||||
|
||||
def test_tenants_list_memberships_as_member(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
@@ -803,6 +842,7 @@ class TestTenantViewSet:
|
||||
):
|
||||
_, tenant2, _ = tenants_fixture
|
||||
user_membership = Membership.objects.get(tenant=tenant2, user__email=TEST_USER)
|
||||
user_id = user_membership.user_id
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
@@ -811,6 +851,127 @@ class TestTenantViewSet:
|
||||
)
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
assert Membership.objects.filter(id=user_membership.id).exists()
|
||||
assert User.objects.filter(id=user_id).exists()
|
||||
|
||||
def test_expel_user_deletes_account_if_last_membership(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
# TEST_USER is OWNER of tenant2; user3 is MEMBER only in tenant2
|
||||
_, tenant2, _ = tenants_fixture
|
||||
_, user3_membership = extra_users
|
||||
user3, membership3 = user3_membership
|
||||
|
||||
assert Membership.objects.filter(user=user3).count() == 1
|
||||
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
kwargs={"tenant_pk": tenant2.id, "pk": membership3.id},
|
||||
)
|
||||
)
|
||||
assert response.status_code == status.HTTP_204_NO_CONTENT
|
||||
assert not Membership.objects.filter(id=membership3.id).exists()
|
||||
assert not User.objects.filter(id=user3.id).exists()
|
||||
|
||||
def test_expel_user_blacklists_refresh_tokens(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
_, tenant2, _ = tenants_fixture
|
||||
_, user3_membership = extra_users
|
||||
user3, membership3 = user3_membership
|
||||
|
||||
# Issue two refresh tokens to simulate active sessions
|
||||
RefreshToken.for_user(user3)
|
||||
RefreshToken.for_user(user3)
|
||||
outstanding_ids = list(
|
||||
OutstandingToken.objects.filter(user=user3).values_list("id", flat=True)
|
||||
)
|
||||
assert len(outstanding_ids) == 2
|
||||
assert not BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_ids
|
||||
).exists()
|
||||
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
kwargs={"tenant_pk": tenant2.id, "pk": membership3.id},
|
||||
)
|
||||
)
|
||||
assert response.status_code == status.HTTP_204_NO_CONTENT
|
||||
assert (
|
||||
BlacklistedToken.objects.filter(token_id__in=outstanding_ids).count() == 2
|
||||
)
|
||||
|
||||
def test_expel_user_blacklists_refresh_tokens_is_idempotent(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
# Regression test for the bulk blacklisting path: if one of the
|
||||
# user's refresh tokens is already blacklisted when the expel
|
||||
# endpoint runs, the remaining tokens must still be blacklisted
|
||||
# and the already-blacklisted one must not be duplicated.
|
||||
tenant1, tenant2, _ = tenants_fixture
|
||||
_, user3_membership = extra_users
|
||||
user3, membership3 = user3_membership
|
||||
|
||||
# Keep the user alive after the expel so the assertions below can
|
||||
# still query OutstandingToken by user_id.
|
||||
Membership.objects.create(
|
||||
user=user3,
|
||||
tenant=tenant1,
|
||||
role=Membership.RoleChoices.MEMBER,
|
||||
)
|
||||
|
||||
RefreshToken.for_user(user3)
|
||||
RefreshToken.for_user(user3)
|
||||
outstanding_ids = list(
|
||||
OutstandingToken.objects.filter(user=user3).values_list("id", flat=True)
|
||||
)
|
||||
assert len(outstanding_ids) == 2
|
||||
|
||||
# Pre-blacklist one of the two tokens to simulate a prior revocation.
|
||||
BlacklistedToken.objects.create(token_id=outstanding_ids[0])
|
||||
assert (
|
||||
BlacklistedToken.objects.filter(token_id__in=outstanding_ids).count() == 1
|
||||
)
|
||||
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
kwargs={"tenant_pk": tenant2.id, "pk": membership3.id},
|
||||
)
|
||||
)
|
||||
assert response.status_code == status.HTTP_204_NO_CONTENT
|
||||
|
||||
blacklisted = BlacklistedToken.objects.filter(token_id__in=outstanding_ids)
|
||||
assert blacklisted.count() == 2
|
||||
assert set(blacklisted.values_list("token_id", flat=True)) == set(
|
||||
outstanding_ids
|
||||
)
|
||||
|
||||
def test_expel_user_keeps_account_if_has_other_memberships(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
):
|
||||
tenant1, tenant2, _ = tenants_fixture
|
||||
_, user3_membership = extra_users
|
||||
user3, membership3 = user3_membership
|
||||
|
||||
# Give user3 an additional membership in tenant1 so they are not orphaned
|
||||
other_membership = Membership.objects.create(
|
||||
user=user3,
|
||||
tenant=tenant1,
|
||||
role=Membership.RoleChoices.MEMBER,
|
||||
)
|
||||
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
kwargs={"tenant_pk": tenant2.id, "pk": membership3.id},
|
||||
)
|
||||
)
|
||||
assert response.status_code == status.HTTP_204_NO_CONTENT
|
||||
assert not Membership.objects.filter(id=membership3.id).exists()
|
||||
assert User.objects.filter(id=user3.id).exists()
|
||||
assert Membership.objects.filter(id=other_membership.id).exists()
|
||||
|
||||
def test_tenants_delete_another_membership_as_owner(
|
||||
self, authenticated_client, tenants_fixture, extra_users
|
||||
@@ -882,6 +1043,128 @@ class TestTenantViewSet:
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
assert Membership.objects.filter(id=other_membership.id).exists()
|
||||
|
||||
def test_delete_membership_cleans_up_orphaned_role_grants(
|
||||
self, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Test that deleting a membership removes UserRoleRelationship records
|
||||
for that tenant while preserving grants in other tenants."""
|
||||
tenant1, tenant2, _ = tenants_fixture
|
||||
|
||||
# Create a user with memberships in both tenants
|
||||
user = User.objects.create_user(
|
||||
name="Multi-tenant User",
|
||||
password=TEST_PASSWORD,
|
||||
email="multitenant@test.com",
|
||||
)
|
||||
|
||||
# Create memberships in both tenants
|
||||
Membership.objects.create(
|
||||
user=user, tenant=tenant1, role=Membership.RoleChoices.MEMBER
|
||||
)
|
||||
membership2 = Membership.objects.create(
|
||||
user=user, tenant=tenant2, role=Membership.RoleChoices.MEMBER
|
||||
)
|
||||
|
||||
# Create roles in both tenants
|
||||
role1 = Role.objects.create(
|
||||
name="Test Role 1", tenant=tenant1, manage_providers=True
|
||||
)
|
||||
role2 = Role.objects.create(
|
||||
name="Test Role 2", tenant=tenant2, manage_scans=True
|
||||
)
|
||||
|
||||
# Create user role relationships for both tenants
|
||||
UserRoleRelationship.objects.create(user=user, role=role1, tenant=tenant1)
|
||||
UserRoleRelationship.objects.create(user=user, role=role2, tenant=tenant2)
|
||||
|
||||
# Verify initial state
|
||||
assert UserRoleRelationship.objects.filter(user=user, tenant=tenant1).exists()
|
||||
assert UserRoleRelationship.objects.filter(user=user, tenant=tenant2).exists()
|
||||
assert Role.objects.filter(id=role1.id).exists()
|
||||
assert Role.objects.filter(id=role2.id).exists()
|
||||
|
||||
# Delete membership from tenant2 (authenticated user is owner of tenant2)
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
kwargs={"tenant_pk": tenant2.id, "pk": membership2.id},
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_204_NO_CONTENT
|
||||
|
||||
# Verify the membership was deleted
|
||||
assert not Membership.objects.filter(id=membership2.id).exists()
|
||||
|
||||
# Verify UserRoleRelationship for tenant2 was deleted
|
||||
assert not UserRoleRelationship.objects.filter(
|
||||
user=user, tenant=tenant2
|
||||
).exists()
|
||||
|
||||
# Verify UserRoleRelationship for tenant1 is preserved
|
||||
assert UserRoleRelationship.objects.filter(user=user, tenant=tenant1).exists()
|
||||
|
||||
# Verify orphaned role2 was deleted (no more user or invitation relationships)
|
||||
assert not Role.objects.filter(id=role2.id).exists()
|
||||
|
||||
# Verify role1 is preserved (still has user relationship)
|
||||
assert Role.objects.filter(id=role1.id).exists()
|
||||
|
||||
# Verify the user still exists (has other memberships)
|
||||
assert User.objects.filter(id=user.id).exists()
|
||||
|
||||
def test_delete_membership_preserves_role_with_invitation_relationship(
|
||||
self, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Test that roles are not deleted if they have invitation relationships."""
|
||||
_, tenant2, _ = tenants_fixture
|
||||
|
||||
# Create a user with membership
|
||||
user = User.objects.create_user(
|
||||
name="Test User", password=TEST_PASSWORD, email="testuser@test.com"
|
||||
)
|
||||
membership = Membership.objects.create(
|
||||
user=user, tenant=tenant2, role=Membership.RoleChoices.MEMBER
|
||||
)
|
||||
|
||||
# Create a role and user relationship
|
||||
role = Role.objects.create(
|
||||
name="Shared Role", tenant=tenant2, manage_providers=True
|
||||
)
|
||||
UserRoleRelationship.objects.create(user=user, role=role, tenant=tenant2)
|
||||
|
||||
# Create an invitation with the same role
|
||||
invitation = Invitation.objects.create(email="pending@test.com", tenant=tenant2)
|
||||
InvitationRoleRelationship.objects.create(
|
||||
invitation=invitation, role=role, tenant=tenant2
|
||||
)
|
||||
|
||||
# Verify initial state
|
||||
assert UserRoleRelationship.objects.filter(user=user, role=role).exists()
|
||||
assert InvitationRoleRelationship.objects.filter(
|
||||
invitation=invitation, role=role
|
||||
).exists()
|
||||
assert Role.objects.filter(id=role.id).exists()
|
||||
|
||||
# Delete the membership
|
||||
response = authenticated_client.delete(
|
||||
reverse(
|
||||
"tenant-membership-detail",
|
||||
kwargs={"tenant_pk": tenant2.id, "pk": membership.id},
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_204_NO_CONTENT
|
||||
|
||||
# Verify UserRoleRelationship was deleted
|
||||
assert not UserRoleRelationship.objects.filter(user=user, role=role).exists()
|
||||
|
||||
# Verify role is preserved because invitation relationship exists
|
||||
assert Role.objects.filter(id=role.id).exists()
|
||||
assert InvitationRoleRelationship.objects.filter(
|
||||
invitation=invitation, role=role
|
||||
).exists()
|
||||
|
||||
def test_tenants_list_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
@@ -3558,9 +3841,14 @@ class TestScanViewSet:
|
||||
"prowler-output-123_threatscore_report.pdf",
|
||||
)
|
||||
|
||||
presigned_url = (
|
||||
"https://test-bucket.s3.amazonaws.com/"
|
||||
"tenant-id/scan-id/threatscore/prowler-output-123_threatscore_report.pdf"
|
||||
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300"
|
||||
)
|
||||
mock_s3_client = Mock()
|
||||
mock_s3_client.list_objects_v2.return_value = {"Contents": [{"Key": pdf_key}]}
|
||||
mock_s3_client.get_object.return_value = {"Body": io.BytesIO(b"pdf-bytes")}
|
||||
mock_s3_client.generate_presigned_url.return_value = presigned_url
|
||||
|
||||
mock_env_str.return_value = bucket
|
||||
mock_get_s3_client.return_value = mock_s3_client
|
||||
@@ -3569,19 +3857,26 @@ class TestScanViewSet:
|
||||
url = reverse("scan-threatscore", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"] == "application/pdf"
|
||||
assert response["Content-Disposition"].endswith(
|
||||
'"prowler-output-123_threatscore_report.pdf"'
|
||||
)
|
||||
assert response.content == b"pdf-bytes"
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
assert response["Location"] == presigned_url
|
||||
mock_s3_client.list_objects_v2.assert_called_once()
|
||||
mock_s3_client.get_object.assert_called_once_with(Bucket=bucket, Key=pdf_key)
|
||||
mock_s3_client.generate_presigned_url.assert_called_once_with(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": bucket,
|
||||
"Key": pdf_key,
|
||||
"ResponseContentDisposition": (
|
||||
'attachment; filename="prowler-output-123_threatscore_report.pdf"'
|
||||
),
|
||||
"ResponseContentType": "application/pdf",
|
||||
},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
|
||||
def test_report_s3_success(self, authenticated_client, scans_fixture, monkeypatch):
|
||||
"""
|
||||
When output_location is an S3 URL and the S3 client returns the file successfully,
|
||||
the view should return the ZIP file with HTTP 200 and proper headers.
|
||||
When output_location is an S3 URL and the object exists,
|
||||
the view should return a 302 redirect to a presigned S3 URL.
|
||||
"""
|
||||
scan = scans_fixture[0]
|
||||
bucket = "test-bucket"
|
||||
@@ -3595,22 +3890,33 @@ class TestScanViewSet:
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
presigned_url = (
|
||||
"https://test-bucket.s3.amazonaws.com/report.zip"
|
||||
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300"
|
||||
)
|
||||
|
||||
class FakeS3Client:
|
||||
def get_object(self, Bucket, Key):
|
||||
def head_object(self, Bucket, Key):
|
||||
assert Bucket == bucket
|
||||
assert Key == key
|
||||
return {"Body": io.BytesIO(b"s3 zip content")}
|
||||
return {}
|
||||
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
assert ClientMethod == "get_object"
|
||||
assert Params["Bucket"] == bucket
|
||||
assert Params["Key"] == key
|
||||
assert Params["ResponseContentDisposition"] == (
|
||||
'attachment; filename="report.zip"'
|
||||
)
|
||||
assert ExpiresIn == 300
|
||||
return presigned_url
|
||||
|
||||
monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client())
|
||||
|
||||
url = reverse("scan-report", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
assert response.status_code == 200
|
||||
expected_filename = os.path.basename("report.zip")
|
||||
content_disposition = response.get("Content-Disposition")
|
||||
assert content_disposition.startswith('attachment; filename="')
|
||||
assert f'filename="{expected_filename}"' in content_disposition
|
||||
assert response.content == b"s3 zip content"
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
assert response["Location"] == presigned_url
|
||||
|
||||
def test_report_s3_success_no_local_files(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
@@ -3749,23 +4055,31 @@ class TestScanViewSet:
|
||||
)
|
||||
|
||||
match_key = "path/compliance/mitre_attack_aws.csv"
|
||||
presigned_url = (
|
||||
"https://test-bucket.s3.amazonaws.com/path/compliance/mitre_attack_aws.csv"
|
||||
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300"
|
||||
)
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
return {"Contents": [{"Key": match_key}]}
|
||||
|
||||
def get_object(self, Bucket, Key):
|
||||
return {"Body": io.BytesIO(b"ignored")}
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
assert ClientMethod == "get_object"
|
||||
assert Params["Key"] == match_key
|
||||
assert Params["ResponseContentDisposition"] == (
|
||||
'attachment; filename="mitre_attack_aws.csv"'
|
||||
)
|
||||
assert ExpiresIn == 300
|
||||
return presigned_url
|
||||
|
||||
monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client())
|
||||
|
||||
framework = match_key.split("/")[-1].split(".")[0]
|
||||
url = reverse("scan-compliance", kwargs={"pk": scan.id, "name": framework})
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
cd = resp["Content-Disposition"]
|
||||
assert cd.startswith('attachment; filename="')
|
||||
assert cd.endswith('filename="mitre_attack_aws.csv"')
|
||||
assert resp.status_code == status.HTTP_302_FOUND
|
||||
assert resp["Location"] == presigned_url
|
||||
|
||||
def test_compliance_s3_not_found(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
@@ -3830,6 +4144,51 @@ class TestScanViewSet:
|
||||
assert cd.startswith('attachment; filename="')
|
||||
assert cd.endswith(f'filename="{fname.name}"')
|
||||
|
||||
def test_cis_no_output(self, authenticated_client, scans_fixture):
|
||||
"""CIS PDF endpoint must 404 when the scan has no output_location."""
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.output_location = ""
|
||||
scan.save()
|
||||
|
||||
url = reverse("scan-cis", kwargs={"pk": scan.id})
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_404_NOT_FOUND
|
||||
assert (
|
||||
resp.json()["errors"]["detail"]
|
||||
== "The scan has no reports, or the CIS report generation task has not started yet."
|
||||
)
|
||||
|
||||
def test_cis_local_file(self, authenticated_client, scans_fixture, monkeypatch):
|
||||
"""CIS PDF endpoint must serve the latest generated PDF."""
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
base = tmp_path / "reports"
|
||||
cis_dir = base / "cis"
|
||||
cis_dir.mkdir(parents=True, exist_ok=True)
|
||||
fname = cis_dir / "prowler-output-aws-20260101000000_cis_report.pdf"
|
||||
fname.write_bytes(b"%PDF-1.4 fake pdf")
|
||||
|
||||
scan.output_location = str(base / "scan.zip")
|
||||
scan.save()
|
||||
|
||||
monkeypatch.setattr(
|
||||
glob,
|
||||
"glob",
|
||||
lambda p: [str(fname)] if p.endswith("*_cis_report.pdf") else [],
|
||||
)
|
||||
|
||||
url = reverse("scan-cis", kwargs={"pk": scan.id})
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
assert resp["Content-Type"] == "application/pdf"
|
||||
cd = resp["Content-Disposition"]
|
||||
assert cd.startswith('attachment; filename="')
|
||||
assert cd.endswith(f'filename="{fname.name}"')
|
||||
|
||||
@patch("api.v1.views.Task.objects.get")
|
||||
@patch("api.v1.views.TaskSerializer")
|
||||
def test__get_task_status_returns_none_if_task_not_executing(
|
||||
@@ -3923,8 +4282,8 @@ class TestScanViewSet:
|
||||
scan.save()
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.get_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "NoSuchKey"}}, "GetObject"
|
||||
fake_client.head_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "NoSuchKey"}}, "HeadObject"
|
||||
)
|
||||
mock_get_s3_client.return_value = fake_client
|
||||
|
||||
@@ -3947,8 +4306,8 @@ class TestScanViewSet:
|
||||
scan.save()
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.get_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "AccessDenied"}}, "GetObject"
|
||||
fake_client.head_object.side_effect = ClientError(
|
||||
{"Error": {"Code": "AccessDenied"}}, "HeadObject"
|
||||
)
|
||||
mock_get_s3_client.return_value = fake_client
|
||||
|
||||
|
||||
+300
-58
@@ -4,6 +4,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -16,10 +17,11 @@ from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
|
||||
from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter
|
||||
from allauth.socialaccount.providers.saml.views import FinishACSView, LoginView
|
||||
from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
|
||||
from celery import chain
|
||||
from celery import chain, states
|
||||
from celery.result import AsyncResult
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from config.version import RELEASE_ID
|
||||
from config.settings.social_login import (
|
||||
GITHUB_OAUTH_CALLBACK_URL,
|
||||
GOOGLE_OAUTH_CALLBACK_URL,
|
||||
@@ -53,13 +55,14 @@ from django.db.models import (
|
||||
)
|
||||
from django.db.models.fields.json import KeyTextTransform
|
||||
from django.db.models.functions import Cast, Coalesce, RowNumber
|
||||
from django.http import HttpResponse, QueryDict
|
||||
from django.http import HttpResponse, HttpResponseBase, HttpResponseRedirect, QueryDict
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import reverse
|
||||
from django.utils.dateparse import parse_date
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.views.decorators.cache import cache_control
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from drf_spectacular.settings import spectacular_settings
|
||||
from drf_spectacular.types import OpenApiTypes
|
||||
from drf_spectacular.utils import (
|
||||
@@ -83,6 +86,10 @@ from rest_framework.permissions import SAFE_METHODS
|
||||
from rest_framework_json_api import filters as jsonapi_filters
|
||||
from rest_framework_json_api.views import RelationshipView, Response
|
||||
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
from tasks.beat import schedule_provider_scan
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.export import get_s3_client
|
||||
@@ -169,6 +176,7 @@ from api.models import (
|
||||
FindingGroupDailySummary,
|
||||
Integration,
|
||||
Invitation,
|
||||
InvitationRoleRelationship,
|
||||
LighthouseConfiguration,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
@@ -417,7 +425,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.25.5"
|
||||
spectacular_settings.VERSION = RELEASE_ID
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -1330,9 +1338,11 @@ class MembershipViewSet(BaseTenantViewset):
|
||||
),
|
||||
destroy=extend_schema(
|
||||
summary="Delete tenant memberships",
|
||||
description="Delete the membership details of users in a tenant. You need to be one of the owners to delete a "
|
||||
"membership that is not yours. If you are the last owner of a tenant, you cannot delete your own "
|
||||
"membership.",
|
||||
description="Delete a user's membership from a tenant. This action: (1) removes the membership, "
|
||||
"(2) revokes all refresh tokens for the expelled user, (3) removes their role grants for this tenant, "
|
||||
"(4) cleans up orphaned roles, and (5) deletes the user account if this was their last membership. "
|
||||
"You must be a tenant owner to delete another user's membership. The last owner of a tenant cannot "
|
||||
"delete their own membership.",
|
||||
tags=["Tenant"],
|
||||
),
|
||||
)
|
||||
@@ -1341,6 +1351,7 @@ class TenantMembersViewSet(BaseTenantViewset):
|
||||
http_method_names = ["get", "delete"]
|
||||
serializer_class = MembershipSerializer
|
||||
queryset = Membership.objects.none()
|
||||
filterset_class = MembershipFilter
|
||||
# Authorization is handled by get_requesting_membership (owner/member checks),
|
||||
# not by RBAC, since the target tenant differs from the JWT tenant.
|
||||
required_permissions = []
|
||||
@@ -1398,7 +1409,84 @@ class TenantMembersViewSet(BaseTenantViewset):
|
||||
"You do not have permission to delete this membership."
|
||||
)
|
||||
|
||||
membership_to_delete.delete()
|
||||
user_to_check_id = membership_to_delete.user_id
|
||||
tenant_id = membership_to_delete.tenant_id
|
||||
# All writes run on the admin connection so that the uncommitted
|
||||
# membership delete is visible to the subsequent "other memberships"
|
||||
# check. Splitting the delete and the check across the default
|
||||
# (prowler_user, RLS) and admin connections caused the admin side to
|
||||
# miss the just-deleted row and leave the User row orphaned.
|
||||
with transaction.atomic(using=MainRouter.admin_db):
|
||||
Membership.objects.using(MainRouter.admin_db).filter(
|
||||
id=membership_to_delete.id
|
||||
).delete()
|
||||
|
||||
# Remove role grants for this user in this tenant to prevent
|
||||
# orphaned permissions that could allow access after expulsion
|
||||
deleted_role_relationships = UserRoleRelationship.objects.using(
|
||||
MainRouter.admin_db
|
||||
).filter(user_id=user_to_check_id, tenant_id=tenant_id)
|
||||
|
||||
# Collect role IDs that might become orphaned after deletion
|
||||
role_ids_to_check = list(
|
||||
deleted_role_relationships.values_list("role_id", flat=True)
|
||||
)
|
||||
|
||||
# Delete the user role relationships for this tenant
|
||||
deleted_role_relationships.delete()
|
||||
|
||||
# Clean up orphaned roles that have no remaining user or invitation relationships
|
||||
if role_ids_to_check:
|
||||
for role_id in role_ids_to_check:
|
||||
has_user_relationships = (
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db)
|
||||
.filter(role_id=role_id)
|
||||
.exists()
|
||||
)
|
||||
|
||||
has_invitation_relationships = (
|
||||
InvitationRoleRelationship.objects.using(MainRouter.admin_db)
|
||||
.filter(role_id=role_id)
|
||||
.exists()
|
||||
)
|
||||
|
||||
if not has_user_relationships and not has_invitation_relationships:
|
||||
Role.objects.using(MainRouter.admin_db).filter(
|
||||
id=role_id
|
||||
).delete()
|
||||
|
||||
# Revoke any refresh tokens the expelled user still holds so they
|
||||
# cannot mint fresh access tokens. This must happen before the
|
||||
# User row is deleted, because OutstandingToken.user is
|
||||
# on_delete=SET_NULL in djangorestframework-simplejwt 5.5.1
|
||||
# (see rest_framework_simplejwt/token_blacklist/models.py): once
|
||||
# the user row is gone, user_id becomes NULL and we can no longer
|
||||
# look up that user's outstanding tokens. Access tokens already
|
||||
# issued remain valid until SIMPLE_JWT["ACCESS_TOKEN_LIFETIME"]
|
||||
# expires.
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.using(MainRouter.admin_db)
|
||||
.filter(user_id=user_to_check_id)
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
if outstanding_token_ids:
|
||||
BlacklistedToken.objects.using(MainRouter.admin_db).bulk_create(
|
||||
[
|
||||
BlacklistedToken(token_id=token_id)
|
||||
for token_id in outstanding_token_ids
|
||||
],
|
||||
ignore_conflicts=True,
|
||||
)
|
||||
|
||||
has_other_memberships = (
|
||||
Membership.objects.using(MainRouter.admin_db)
|
||||
.filter(user_id=user_to_check_id)
|
||||
.exists()
|
||||
)
|
||||
if not has_other_memberships:
|
||||
User.objects.using(MainRouter.admin_db).filter(
|
||||
id=user_to_check_id
|
||||
).delete()
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@@ -1841,6 +1929,27 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet):
|
||||
),
|
||||
},
|
||||
),
|
||||
cis=extend_schema(
|
||||
tags=["Scan"],
|
||||
summary="Retrieve CIS Benchmark compliance report",
|
||||
description="Download the CIS Benchmark compliance report as a PDF file. "
|
||||
"When a provider ships multiple CIS versions, the report is generated "
|
||||
"for the highest available version.",
|
||||
request=None,
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
description="PDF file containing the CIS compliance report"
|
||||
),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
401: OpenApiResponse(
|
||||
description="API key missing or user not Authenticated"
|
||||
),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="The scan has no CIS reports, or the CIS report generation task has not started yet"
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
@method_decorator(CACHE_DECORATOR, name="list")
|
||||
@method_decorator(CACHE_DECORATOR, name="retrieve")
|
||||
@@ -1909,6 +2018,9 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
elif self.action == "csa":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "cis":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return super().get_serializer_class()
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
@@ -1971,24 +2083,38 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
},
|
||||
)
|
||||
|
||||
def _load_file(self, path_pattern, s3=False, bucket=None, list_objects=False):
|
||||
def _load_file(
|
||||
self,
|
||||
path_pattern,
|
||||
s3=False,
|
||||
bucket=None,
|
||||
list_objects=False,
|
||||
content_type=None,
|
||||
):
|
||||
"""
|
||||
Loads a binary file (e.g., ZIP or CSV) and returns its content and filename.
|
||||
Resolve a report file location and return the bytes (filesystem) or a redirect (S3).
|
||||
|
||||
Depending on the input parameters, this method supports loading:
|
||||
- From S3 using a direct key.
|
||||
- From S3 by listing objects under a prefix and matching suffix.
|
||||
- From the local filesystem using glob pattern matching.
|
||||
- From S3 using a direct key, returns a 302 to a short-lived presigned URL.
|
||||
- From S3 by listing objects under a prefix and matching suffix, returns a 302 to a short-lived presigned URL.
|
||||
- From the local filesystem using glob pattern matching, returns the file bytes.
|
||||
|
||||
The S3 branch never streams bytes through the worker; this prevents gunicorn
|
||||
worker timeouts on large reports.
|
||||
|
||||
Args:
|
||||
path_pattern (str): The key or glob pattern representing the file location.
|
||||
s3 (bool, optional): Whether the file is stored in S3. Defaults to False.
|
||||
bucket (str, optional): The name of the S3 bucket, required if `s3=True`. Defaults to None.
|
||||
list_objects (bool, optional): If True and `s3=True`, list objects by prefix to find the file. Defaults to False.
|
||||
content_type (str, optional): On the S3 branch, forwarded as `ResponseContentType`
|
||||
so the presigned download advertises the same Content-Type the API used to send.
|
||||
Ignored on the filesystem branch.
|
||||
|
||||
Returns:
|
||||
tuple[bytes, str]: A tuple containing the file content as bytes and the filename if successful.
|
||||
Response: A DRF `Response` object with an appropriate status and error detail if an error occurs.
|
||||
tuple[bytes, str]: For the filesystem branch, the file content and filename.
|
||||
HttpResponseRedirect: For the S3 branch on success, a 302 redirect to a presigned `GetObject` URL.
|
||||
Response: For any error path, a DRF `Response` with an appropriate status and detail.
|
||||
"""
|
||||
if s3:
|
||||
try:
|
||||
@@ -2035,25 +2161,45 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
# path_pattern here is prefix, but in compliance we build correct suffix check before
|
||||
key = keys[0]
|
||||
else:
|
||||
# path_pattern is exact key
|
||||
# path_pattern is exact key; HEAD before presigning to preserve the 404 contract.
|
||||
key = path_pattern
|
||||
try:
|
||||
s3_obj = client.get_object(Bucket=bucket, Key=key)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code")
|
||||
if code == "NoSuchKey":
|
||||
try:
|
||||
client.head_object(Bucket=bucket, Key=key)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code")
|
||||
if code in ("NoSuchKey", "404"):
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
{"detail": "There is a problem with credentials."},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
return Response(
|
||||
{"detail": "There is a problem with credentials."},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
content = s3_obj["Body"].read()
|
||||
|
||||
filename = os.path.basename(key)
|
||||
# escape quotes and strip CR/LF so a malformed key cannot break out of the header
|
||||
safe_filename = (
|
||||
filename.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\r", "")
|
||||
.replace("\n", "")
|
||||
)
|
||||
params = {
|
||||
"Bucket": bucket,
|
||||
"Key": key,
|
||||
"ResponseContentDisposition": f'attachment; filename="{safe_filename}"',
|
||||
}
|
||||
if content_type:
|
||||
params["ResponseContentType"] = content_type
|
||||
url = client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params=params,
|
||||
ExpiresIn=300,
|
||||
)
|
||||
return HttpResponseRedirect(url)
|
||||
else:
|
||||
files = glob.glob(path_pattern)
|
||||
if not files:
|
||||
@@ -2096,12 +2242,16 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
loader = self._load_file(
|
||||
key_prefix, s3=True, bucket=bucket, list_objects=False
|
||||
key_prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=False,
|
||||
content_type="application/x-zip-compressed",
|
||||
)
|
||||
else:
|
||||
loader = self._load_file(scan.output_location, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2139,18 +2289,69 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix), "compliance", f"{name}.csv"
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="text/csv",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "compliance", f"*_{name}.csv")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "text/csv")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_name="cis",
|
||||
)
|
||||
def cis(self, request, pk=None):
|
||||
scan = self.get_object()
|
||||
running_resp = self._get_task_status(scan)
|
||||
if running_resp:
|
||||
return running_resp
|
||||
|
||||
if not scan.output_location:
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the CIS report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
if scan.output_location.startswith("s3://"):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix),
|
||||
"cis",
|
||||
"*_cis_report.pdf",
|
||||
)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "cis", "*_cis_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "application/pdf")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
@@ -2179,13 +2380,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"threatscore",
|
||||
"*_threatscore_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "threatscore", "*_threatscore_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2219,13 +2426,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"ens",
|
||||
"*_ens_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "ens", "*_ens_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2258,13 +2471,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"nis2",
|
||||
"*_nis2_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "nis2", "*_nis2_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2297,13 +2516,19 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"csa",
|
||||
"*_csa_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="application/pdf",
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "csa", "*_csa_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
@@ -2312,28 +2537,45 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
def create(self, request, *args, **kwargs):
|
||||
input_serializer = self.get_serializer(data=request.data)
|
||||
input_serializer.is_valid(raise_exception=True)
|
||||
|
||||
# Broker publish is deferred to on_commit so the worker cannot read
|
||||
# Scan before BaseRLSViewSet's dispatch-wide atomic commits.
|
||||
pre_task_id = str(uuid.uuid4())
|
||||
|
||||
with transaction.atomic():
|
||||
scan = input_serializer.save()
|
||||
with transaction.atomic():
|
||||
task = perform_scan_task.apply_async(
|
||||
kwargs={
|
||||
"tenant_id": self.request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
},
|
||||
scan.task_id = pre_task_id
|
||||
scan.save(update_fields=["task_id"])
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
task_result, _ = TaskResult.objects.get_or_create(
|
||||
task_id=pre_task_id,
|
||||
defaults={"status": states.PENDING, "task_name": "scan-perform"},
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=pre_task_id,
|
||||
tenant_id=self.request.tenant_id,
|
||||
defaults={"task_runner_task": task_result},
|
||||
)
|
||||
|
||||
prowler_task = Task.objects.get(id=task.id)
|
||||
scan.task_id = task.id
|
||||
scan.save(update_fields=["task_id"])
|
||||
scan_kwargs = {
|
||||
"tenant_id": self.request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
}
|
||||
|
||||
transaction.on_commit(
|
||||
lambda: perform_scan_task.apply_async(
|
||||
kwargs=scan_kwargs, task_id=pre_task_id
|
||||
)
|
||||
)
|
||||
|
||||
self.response_serializer_class = TaskSerializer
|
||||
output_serializer = self.get_serializer(prowler_task)
|
||||
|
||||
@@ -118,6 +118,8 @@ REST_FRAMEWORK = {
|
||||
"attack-paths-custom-query": env(
|
||||
"DJANGO_THROTTLE_ATTACK_PATHS_CUSTOM_QUERY", default="10/min"
|
||||
),
|
||||
"health-live": env("DJANGO_THROTTLE_HEALTH_LIVE", default="120/min"),
|
||||
"health-ready": env("DJANGO_THROTTLE_HEALTH_READY", default="60/min"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from django.urls import include, path
|
||||
|
||||
from api.health import LivenessView, ReadinessView
|
||||
|
||||
urlpatterns = [
|
||||
path("api/v1/", include("api.v1.urls")),
|
||||
path("health/live", LivenessView.as_view(), name="health-live"),
|
||||
path("health/ready", ReadinessView.as_view(), name="health-ready"),
|
||||
]
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Single source of truth for the API version.
|
||||
|
||||
The semantic version is read once from ``api/pyproject.toml`` at module
|
||||
import; consumers (health payload, OpenAPI schema) read the resulting
|
||||
constants. Fails fast at boot if the file cannot be located, so a
|
||||
packaging mistake surfaces immediately rather than serving stale data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
_PROJECT_NAME = "prowler-api"
|
||||
|
||||
|
||||
def _discover_release_id() -> str:
|
||||
here = Path(__file__).resolve()
|
||||
for directory in here.parents:
|
||||
candidate = directory / "pyproject.toml"
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
with candidate.open("rb") as f:
|
||||
data = tomllib.load(f)
|
||||
project = data.get("project") or {}
|
||||
if project.get("name") != _PROJECT_NAME:
|
||||
continue
|
||||
version = project.get("version")
|
||||
if not isinstance(version, str) or not version:
|
||||
raise RuntimeError(
|
||||
f"{candidate} declares an empty or invalid [project].version"
|
||||
)
|
||||
return version
|
||||
raise RuntimeError(
|
||||
f"Could not locate the {_PROJECT_NAME} pyproject.toml from {here}"
|
||||
)
|
||||
|
||||
|
||||
RELEASE_ID: str = _discover_release_id()
|
||||
# Public contract major (e.g. "1"); matches the /api/v1/ namespace.
|
||||
API_VERSION: str = RELEASE_ID.split(".", 1)[0]
|
||||
@@ -14,8 +14,8 @@ from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
aggregate_scan_category_summaries,
|
||||
aggregate_scan_resource_group_summaries,
|
||||
)
|
||||
|
||||
from api.attack_paths import (
|
||||
@@ -1445,8 +1445,8 @@ def latest_scan_finding_with_categories(
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_category_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_resource_group_summaries(tenant_id, str(scan.id))
|
||||
aggregate_scan_category_summaries(tenant_id, str(scan.id))
|
||||
aggregate_scan_resource_group_summaries(tenant_id, str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 131 KiB |
@@ -49,7 +49,7 @@ def start_aws_ingestion(
|
||||
}
|
||||
|
||||
boto3_session = get_boto3_session(prowler_api_provider, prowler_sdk_provider)
|
||||
regions: list[str] = list(prowler_sdk_provider._enabled_regions)
|
||||
regions: list[str] = resolve_aws_regions(prowler_api_provider, prowler_sdk_provider)
|
||||
requested_syncs = list(cartography_aws.RESOURCE_FUNCTIONS.keys())
|
||||
|
||||
sync_args = cartography_aws._build_aws_sync_kwargs(
|
||||
@@ -226,6 +226,48 @@ def get_boto3_session(
|
||||
return boto3_session
|
||||
|
||||
|
||||
def resolve_aws_regions(
|
||||
prowler_api_provider: ProwlerAPIProvider,
|
||||
prowler_sdk_provider: ProwlerSDKProvider,
|
||||
) -> list[str]:
|
||||
"""Resolve the regions to scan, falling back when `_enabled_regions` is `None`.
|
||||
|
||||
The SDK silently sets `_enabled_regions` to `None` when `ec2:DescribeRegions`
|
||||
fails (missing IAM permission, transient error). Without a fallback the
|
||||
Cartography ingestion crashes with a non-actionable `TypeError`. Try the
|
||||
user's `audited_regions` next, then the partition's static region list.
|
||||
Excluded regions are honored on every branch.
|
||||
"""
|
||||
if prowler_sdk_provider._enabled_regions is not None:
|
||||
regions = set(prowler_sdk_provider._enabled_regions)
|
||||
|
||||
elif prowler_sdk_provider.identity.audited_regions:
|
||||
regions = set(prowler_sdk_provider.identity.audited_regions)
|
||||
|
||||
else:
|
||||
partition = prowler_sdk_provider.identity.partition
|
||||
try:
|
||||
regions = prowler_sdk_provider.get_available_aws_service_regions(
|
||||
"ec2", partition
|
||||
)
|
||||
|
||||
except KeyError:
|
||||
raise RuntimeError(
|
||||
f"No region data available for partition {partition!r}; "
|
||||
f"cannot determine regions to scan for "
|
||||
f"{prowler_api_provider.uid}"
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
f"Could not enumerate enabled regions for AWS account "
|
||||
f"{prowler_api_provider.uid}; falling back to all regions in "
|
||||
f"partition {partition!r}"
|
||||
)
|
||||
|
||||
excluded = set(getattr(prowler_sdk_provider, "_excluded_regions", None) or ())
|
||||
return sorted(regions - excluded)
|
||||
|
||||
|
||||
def get_aioboto3_session(boto3_session: boto3.Session) -> aioboto3.Session:
|
||||
return aioboto3.Session(botocore_session=boto3_session._session)
|
||||
|
||||
|
||||
@@ -18,28 +18,45 @@ logger = get_task_logger(__name__)
|
||||
|
||||
def cleanup_stale_attack_paths_scans() -> dict:
|
||||
"""
|
||||
Find `EXECUTING` `AttackPathsScan` scans whose workers are dead or that have
|
||||
exceeded the stale threshold, and mark them as `FAILED`.
|
||||
Mark stale `AttackPathsScan` rows as `FAILED`.
|
||||
|
||||
Two-pass detection:
|
||||
Covers two stuck-state scenarios:
|
||||
1. `EXECUTING` scans whose workers are dead, or that have exceeded the
|
||||
stale threshold while alive.
|
||||
2. `SCHEDULED` scans that never made it to a worker — parent scan
|
||||
crashed before dispatch, broker lost the message, etc. Detected by
|
||||
age plus the parent `Scan` no longer being in flight.
|
||||
"""
|
||||
threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
cutoff = now - threshold
|
||||
|
||||
cleaned_up: list[str] = []
|
||||
cleaned_up.extend(_cleanup_stale_executing_scans(cutoff))
|
||||
cleaned_up.extend(_cleanup_stale_scheduled_scans(cutoff))
|
||||
|
||||
logger.info(
|
||||
f"Stale `AttackPathsScan` cleanup: {len(cleaned_up)} scan(s) cleaned up"
|
||||
)
|
||||
return {"cleaned_up_count": len(cleaned_up), "scan_ids": cleaned_up}
|
||||
|
||||
|
||||
def _cleanup_stale_executing_scans(cutoff: datetime) -> list[str]:
|
||||
"""
|
||||
Two-pass detection for `EXECUTING` scans:
|
||||
1. If `TaskResult.worker` exists, ping the worker.
|
||||
- Dead worker: cleanup immediately (any age).
|
||||
- Alive + past threshold: revoke the task, then cleanup.
|
||||
- Alive + within threshold: skip.
|
||||
2. If no worker field: fall back to time-based heuristic only.
|
||||
"""
|
||||
threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES)
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
cutoff = now - threshold
|
||||
|
||||
executing_scans = (
|
||||
executing_scans = list(
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
.filter(state=StateChoices.EXECUTING)
|
||||
.select_related("task__task_runner_task")
|
||||
)
|
||||
|
||||
# Cache worker liveness so each worker is pinged at most once
|
||||
executing_scans = list(executing_scans)
|
||||
workers = {
|
||||
tr.worker
|
||||
for scan in executing_scans
|
||||
@@ -48,7 +65,7 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
}
|
||||
worker_alive = {w: _is_worker_alive(w) for w in workers}
|
||||
|
||||
cleaned_up = []
|
||||
cleaned_up: list[str] = []
|
||||
|
||||
for scan in executing_scans:
|
||||
task_result = (
|
||||
@@ -65,9 +82,7 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
|
||||
# Alive but stale — revoke before cleanup
|
||||
_revoke_task(task_result)
|
||||
reason = (
|
||||
"Scan exceeded stale threshold — " "cleaned up by periodic task"
|
||||
)
|
||||
reason = "Scan exceeded stale threshold — cleaned up by periodic task"
|
||||
else:
|
||||
reason = "Worker dead — cleaned up by periodic task"
|
||||
else:
|
||||
@@ -82,10 +97,57 @@ def cleanup_stale_attack_paths_scans() -> dict:
|
||||
if _cleanup_scan(scan, task_result, reason):
|
||||
cleaned_up.append(str(scan.id))
|
||||
|
||||
logger.info(
|
||||
f"Stale `AttackPathsScan` cleanup: {len(cleaned_up)} scan(s) cleaned up"
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
"""
|
||||
Cleanup `SCHEDULED` scans that never reached a worker.
|
||||
|
||||
Detection:
|
||||
- `state == SCHEDULED`
|
||||
- `started_at < cutoff`
|
||||
- parent `Scan` is no longer in flight (terminal state or missing). This
|
||||
avoids cleaning up rows whose parent Prowler scan is legitimately still
|
||||
running.
|
||||
|
||||
For each match: revoke the queued task (best-effort; harmless if already
|
||||
consumed), atomically flip to `FAILED`, and mark the `TaskResult`. The
|
||||
temp Neo4j database is never created while `SCHEDULED`, so no drop is
|
||||
needed.
|
||||
"""
|
||||
scheduled_scans = list(
|
||||
AttackPathsScan.all_objects.using(MainRouter.admin_db)
|
||||
.filter(
|
||||
state=StateChoices.SCHEDULED,
|
||||
started_at__lt=cutoff,
|
||||
)
|
||||
.select_related("task__task_runner_task", "scan")
|
||||
)
|
||||
return {"cleaned_up_count": len(cleaned_up), "scan_ids": cleaned_up}
|
||||
|
||||
cleaned_up: list[str] = []
|
||||
parent_terminal = (
|
||||
StateChoices.COMPLETED,
|
||||
StateChoices.FAILED,
|
||||
StateChoices.CANCELLED,
|
||||
)
|
||||
|
||||
for scan in scheduled_scans:
|
||||
parent_scan = scan.scan
|
||||
if parent_scan is not None and parent_scan.state not in parent_terminal:
|
||||
continue
|
||||
|
||||
task_result = (
|
||||
getattr(scan.task, "task_runner_task", None) if scan.task else None
|
||||
)
|
||||
if task_result:
|
||||
_revoke_task(task_result, terminate=False)
|
||||
|
||||
reason = "Scan never started — cleaned up by periodic task"
|
||||
if _cleanup_scheduled_scan(scan, task_result, reason):
|
||||
cleaned_up.append(str(scan.id))
|
||||
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _is_worker_alive(worker: str) -> bool:
|
||||
@@ -98,12 +160,17 @@ def _is_worker_alive(worker: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _revoke_task(task_result) -> None:
|
||||
"""Send `SIGTERM` to a hung Celery task. Non-fatal on failure."""
|
||||
def _revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task. Non-fatal on failure.
|
||||
|
||||
`terminate=True` SIGTERMs the worker if the task is mid-execution; use
|
||||
for EXECUTING cleanup. `terminate=False` only marks the task id revoked
|
||||
across workers, so any worker pulling the queued message discards it;
|
||||
use for SCHEDULED cleanup where the task hasn't run yet.
|
||||
"""
|
||||
try:
|
||||
current_app.control.revoke(
|
||||
task_result.task_id, terminate=True, signal="SIGTERM"
|
||||
)
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
@@ -125,28 +192,64 @@ def _cleanup_scan(scan, task_result, reason: str) -> bool:
|
||||
except Exception:
|
||||
logger.exception(f"Failed to drop temp database {tmp_db_name}")
|
||||
|
||||
# 2. Lock row, verify still EXECUTING, mark FAILED — all atomic
|
||||
with rls_transaction(str(scan.tenant_id)):
|
||||
try:
|
||||
fresh_scan = AttackPathsScan.objects.select_for_update().get(id=scan.id)
|
||||
except AttackPathsScan.DoesNotExist:
|
||||
logger.warning(f"Scan {scan_id_str} no longer exists, skipping")
|
||||
return False
|
||||
fresh_scan = _finalize_failed_scan(scan, StateChoices.EXECUTING, reason)
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
if fresh_scan.state != StateChoices.EXECUTING:
|
||||
logger.info(f"Scan {scan_id_str} is now {fresh_scan.state}, skipping")
|
||||
return False
|
||||
|
||||
_mark_scan_finished(fresh_scan, StateChoices.FAILED, {"global_error": reason})
|
||||
|
||||
# 3. Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock)
|
||||
# Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock)
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=timezone.utc)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
# 4. Recover graph_data_ready if provider data still exists
|
||||
recover_graph_data_ready(fresh_scan)
|
||||
|
||||
logger.info(f"Cleaned up stale scan {scan_id_str}: {reason}")
|
||||
return True
|
||||
|
||||
|
||||
def _cleanup_scheduled_scan(scan, task_result, reason: str) -> bool:
|
||||
"""
|
||||
Clean up a `SCHEDULED` scan that never reached a worker.
|
||||
|
||||
Skips the temp Neo4j drop — the database is only created once the worker
|
||||
enters `EXECUTING`, so dropping it here just produces noisy log output.
|
||||
|
||||
Returns `True` if the scan was actually cleaned up, `False` if skipped.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
|
||||
fresh_scan = _finalize_failed_scan(scan, StateChoices.SCHEDULED, reason)
|
||||
if fresh_scan is None:
|
||||
return False
|
||||
|
||||
if task_result:
|
||||
task_result.status = states.FAILURE
|
||||
task_result.date_done = datetime.now(tz=timezone.utc)
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
logger.info(f"Cleaned up scheduled scan {scan_id_str}: {reason}")
|
||||
return True
|
||||
|
||||
|
||||
def _finalize_failed_scan(scan, expected_state: str, reason: str):
|
||||
"""
|
||||
Atomically lock the row, verify it's still in `expected_state`, and
|
||||
mark it `FAILED`. Returns the locked row on success, `None` if the
|
||||
row is gone or has already moved on.
|
||||
"""
|
||||
scan_id_str = str(scan.id)
|
||||
with rls_transaction(str(scan.tenant_id)):
|
||||
try:
|
||||
fresh_scan = AttackPathsScan.objects.select_for_update().get(id=scan.id)
|
||||
except AttackPathsScan.DoesNotExist:
|
||||
logger.warning(f"Scan {scan_id_str} no longer exists, skipping")
|
||||
return None
|
||||
|
||||
if fresh_scan.state != expected_state:
|
||||
logger.info(f"Scan {scan_id_str} is now {fresh_scan.state}, skipping")
|
||||
return None
|
||||
|
||||
_mark_scan_finished(fresh_scan, StateChoices.FAILED, {"global_error": reason})
|
||||
|
||||
return fresh_scan
|
||||
|
||||
@@ -67,25 +67,52 @@ def retrieve_attack_paths_scan(
|
||||
return None
|
||||
|
||||
|
||||
def set_attack_paths_scan_task_id(
|
||||
tenant_id: str,
|
||||
scan_pk: str,
|
||||
task_id: str,
|
||||
) -> None:
|
||||
"""Persist the Celery `task_id` on the `AttackPathsScan` row.
|
||||
|
||||
Called at dispatch time (when `apply_async` returns) so the row carries
|
||||
the task id even while still `SCHEDULED`. This lets the periodic
|
||||
cleanup revoke queued messages for scans that never reached a worker.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
ProwlerAPIAttackPathsScan.objects.filter(id=scan_pk).update(task_id=task_id)
|
||||
|
||||
|
||||
def starting_attack_paths_scan(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
task_id: str,
|
||||
cartography_config: CartographyConfig,
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.task_id = task_id
|
||||
attack_paths_scan.state = StateChoices.EXECUTING
|
||||
attack_paths_scan.started_at = datetime.now(tz=timezone.utc)
|
||||
attack_paths_scan.update_tag = cartography_config.update_tag
|
||||
) -> bool:
|
||||
"""Flip the row from `SCHEDULED` to `EXECUTING` atomically.
|
||||
|
||||
attack_paths_scan.save(
|
||||
update_fields=[
|
||||
"task_id",
|
||||
"state",
|
||||
"started_at",
|
||||
"update_tag",
|
||||
]
|
||||
)
|
||||
Returns `False` if the row is gone or has already moved past
|
||||
`SCHEDULED` (e.g., periodic cleanup raced ahead and marked it
|
||||
`FAILED` while the worker message was still in flight).
|
||||
"""
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
try:
|
||||
locked = ProwlerAPIAttackPathsScan.objects.select_for_update().get(
|
||||
id=attack_paths_scan.id
|
||||
)
|
||||
except ProwlerAPIAttackPathsScan.DoesNotExist:
|
||||
return False
|
||||
|
||||
if locked.state != StateChoices.SCHEDULED:
|
||||
return False
|
||||
|
||||
locked.state = StateChoices.EXECUTING
|
||||
locked.started_at = datetime.now(tz=timezone.utc)
|
||||
locked.update_tag = cartography_config.update_tag
|
||||
locked.save(update_fields=["state", "started_at", "update_tag"])
|
||||
|
||||
# Keep the in-memory object the caller is holding in sync.
|
||||
attack_paths_scan.state = locked.state
|
||||
attack_paths_scan.started_at = locked.started_at
|
||||
attack_paths_scan.update_tag = locked.update_tag
|
||||
return True
|
||||
|
||||
|
||||
def _mark_scan_finished(
|
||||
|
||||
@@ -97,6 +97,19 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
)
|
||||
attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id)
|
||||
|
||||
# Idempotency guard: cleanup may have flipped this row to a terminal state
|
||||
# while the message was still in flight. Bail out before touching state.
|
||||
if attack_paths_scan and attack_paths_scan.state in (
|
||||
StateChoices.FAILED,
|
||||
StateChoices.COMPLETED,
|
||||
StateChoices.CANCELLED,
|
||||
):
|
||||
logger.warning(
|
||||
f"Attack Paths scan {attack_paths_scan.id} already in terminal "
|
||||
f"state {attack_paths_scan.state}; skipping execution"
|
||||
)
|
||||
return {}
|
||||
|
||||
# Checks before starting the scan
|
||||
if not cartography_ingestion_function:
|
||||
ingestion_exceptions = {
|
||||
@@ -114,12 +127,17 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
|
||||
else:
|
||||
if not attack_paths_scan:
|
||||
# Safety net for in-flight messages or direct task invocations; dispatcher normally pre-creates the row.
|
||||
logger.warning(
|
||||
f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then"
|
||||
)
|
||||
attack_paths_scan = db_utils.create_attack_paths_scan(
|
||||
tenant_id, scan_id, prowler_api_provider.id
|
||||
)
|
||||
if attack_paths_scan and task_id:
|
||||
db_utils.set_attack_paths_scan_task_id(
|
||||
tenant_id, attack_paths_scan.id, task_id
|
||||
)
|
||||
|
||||
tmp_database_name = graph_database.get_database_name(
|
||||
attack_paths_scan.id, temporary=True
|
||||
@@ -141,9 +159,13 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
# Starting the Attack Paths scan
|
||||
db_utils.starting_attack_paths_scan(
|
||||
attack_paths_scan, task_id, tenant_cartography_config
|
||||
)
|
||||
if not db_utils.starting_attack_paths_scan(
|
||||
attack_paths_scan, tenant_cartography_config
|
||||
):
|
||||
logger.warning(
|
||||
f"Attack Paths scan {attack_paths_scan.id} no longer in SCHEDULED state; cleanup likely raced ahead"
|
||||
)
|
||||
return {}
|
||||
|
||||
scan_t0 = time.perf_counter()
|
||||
logger.info(
|
||||
|
||||
@@ -297,12 +297,15 @@ def backfill_daily_severity_summaries(tenant_id: str, days: int = None):
|
||||
}
|
||||
|
||||
|
||||
def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
def aggregate_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanCategorySummary for a completed scan.
|
||||
|
||||
Aggregates category counts from all findings in the scan and creates
|
||||
one ScanCategorySummary row per (category, severity) combination.
|
||||
Idempotent: re-runs replace the scan's existing rows so counts stay in
|
||||
sync with `Finding.muted` updates triggered outside scan completion
|
||||
(e.g. mute rules).
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
@@ -312,11 +315,6 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
dict: Status indicating whether backfill was performed
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists():
|
||||
return {"status": "already backfilled"}
|
||||
|
||||
if not Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
@@ -337,9 +335,6 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
cache=category_counts,
|
||||
)
|
||||
|
||||
if not category_counts:
|
||||
return {"status": "no categories to backfill"}
|
||||
|
||||
category_summaries = [
|
||||
ScanCategorySummary(
|
||||
tenant_id=tenant_id,
|
||||
@@ -353,20 +348,38 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
for (category, severity), counts in category_counts.items()
|
||||
]
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
ScanCategorySummary.objects.bulk_create(
|
||||
category_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
if category_summaries:
|
||||
with rls_transaction(tenant_id):
|
||||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||||
# `unique_category_severity_per_scan`; race-safe under concurrent writers.
|
||||
ScanCategorySummary.objects.bulk_create(
|
||||
category_summaries,
|
||||
batch_size=500,
|
||||
update_conflicts=True,
|
||||
unique_fields=["tenant_id", "scan_id", "category", "severity"],
|
||||
update_fields=[
|
||||
"total_findings",
|
||||
"failed_findings",
|
||||
"new_failed_findings",
|
||||
],
|
||||
)
|
||||
|
||||
if not category_counts:
|
||||
return {"status": "no categories to backfill"}
|
||||
|
||||
return {"status": "backfilled", "categories_count": len(category_counts)}
|
||||
|
||||
|
||||
def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
def aggregate_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanGroupSummary for a completed scan.
|
||||
|
||||
Aggregates resource group counts from all findings in the scan and creates
|
||||
one ScanGroupSummary row per (resource_group, severity) combination.
|
||||
Idempotent: re-runs replace the scan's existing rows so counts stay in
|
||||
sync with `Finding.muted` updates triggered outside scan completion
|
||||
(e.g. mute rules) and with resource-inventory views reading from this
|
||||
table.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
@@ -376,11 +389,6 @@ def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
dict: Status indicating whether backfill was performed
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists():
|
||||
return {"status": "already backfilled"}
|
||||
|
||||
if not Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
@@ -418,9 +426,6 @@ def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
if not resource_group_counts:
|
||||
return {"status": "no resource groups to backfill"}
|
||||
|
||||
# Compute group-level resource counts (same value for all severity rows in a group)
|
||||
group_resource_counts = {
|
||||
grp: len(uids) for grp, uids in group_resources_cache.items()
|
||||
@@ -439,10 +444,25 @@ def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
for (grp, severity), counts in resource_group_counts.items()
|
||||
]
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
ScanGroupSummary.objects.bulk_create(
|
||||
resource_group_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
if resource_group_summaries:
|
||||
with rls_transaction(tenant_id):
|
||||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||||
# `unique_resource_group_severity_per_scan`; race-safe under concurrent writers.
|
||||
ScanGroupSummary.objects.bulk_create(
|
||||
resource_group_summaries,
|
||||
batch_size=500,
|
||||
update_conflicts=True,
|
||||
unique_fields=["tenant_id", "scan_id", "resource_group", "severity"],
|
||||
update_fields=[
|
||||
"total_findings",
|
||||
"failed_findings",
|
||||
"new_failed_findings",
|
||||
"resources_count",
|
||||
],
|
||||
)
|
||||
|
||||
if not resource_group_counts:
|
||||
return {"status": "no resource groups to backfill"}
|
||||
|
||||
return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)}
|
||||
|
||||
|
||||
@@ -47,6 +47,9 @@ from prowler.lib.outputs.compliance.csa.csa_oraclecloud import OracleCloudCSA
|
||||
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
|
||||
from prowler.lib.outputs.compliance.asd_essential_eight.asd_essential_eight_aws import (
|
||||
ASDEssentialEightAWS,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.iso27001.iso27001_aws import AWSISO27001
|
||||
from prowler.lib.outputs.compliance.iso27001.iso27001_azure import AzureISO27001
|
||||
from prowler.lib.outputs.compliance.iso27001.iso27001_gcp import GCPISO27001
|
||||
@@ -100,6 +103,7 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("ccc_"), CCC_AWS),
|
||||
(lambda name: name.startswith("c5_"), AWSC5),
|
||||
(lambda name: name.startswith("csa_"), AWSCSA),
|
||||
(lambda name: name == "asd_essential_eight_aws", ASDEssentialEightAWS),
|
||||
],
|
||||
"azure": [
|
||||
(lambda name: name.startswith("cis_"), AzureCIS),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,9 @@ from .charts import (
|
||||
get_chart_color_for_percentage,
|
||||
)
|
||||
|
||||
# Framework-specific generators
|
||||
from .cis import CISReportGenerator
|
||||
|
||||
# Reusable components
|
||||
# Reusable components: Color helpers, Badge components, Risk component,
|
||||
# Table components, Section components
|
||||
@@ -31,10 +34,12 @@ from .components import (
|
||||
create_section_header,
|
||||
create_status_badge,
|
||||
create_summary_table,
|
||||
escape_html,
|
||||
get_color_for_compliance,
|
||||
get_color_for_risk_level,
|
||||
get_color_for_weight,
|
||||
get_status_color,
|
||||
truncate_text,
|
||||
)
|
||||
|
||||
# Framework configuration: Main configuration, Color constants, ENS colors,
|
||||
@@ -90,8 +95,6 @@ from .config import (
|
||||
FrameworkConfig,
|
||||
get_framework_config,
|
||||
)
|
||||
|
||||
# Framework-specific generators
|
||||
from .csa import CSAReportGenerator
|
||||
from .ens import ENSReportGenerator
|
||||
from .nis2 import NIS2ReportGenerator
|
||||
@@ -109,6 +112,7 @@ __all__ = [
|
||||
"ENSReportGenerator",
|
||||
"NIS2ReportGenerator",
|
||||
"CSAReportGenerator",
|
||||
"CISReportGenerator",
|
||||
# Configuration
|
||||
"FrameworkConfig",
|
||||
"FRAMEWORK_REGISTRY",
|
||||
@@ -182,6 +186,9 @@ __all__ = [
|
||||
# Section components
|
||||
"create_section_header",
|
||||
"create_summary_table",
|
||||
# Text helpers
|
||||
"truncate_text",
|
||||
"escape_html",
|
||||
# Chart functions
|
||||
"get_chart_color_for_percentage",
|
||||
"create_vertical_bar_chart",
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import gc
|
||||
import os
|
||||
import resource as _resource_module
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
@@ -41,6 +44,7 @@ from .config import (
|
||||
COLOR_LIGHT_BLUE,
|
||||
COLOR_LIGHTER_BLUE,
|
||||
COLOR_PROWLER_DARK_GREEN,
|
||||
FINDINGS_TABLE_CHUNK_SIZE,
|
||||
PADDING_LARGE,
|
||||
PADDING_SMALL,
|
||||
FrameworkConfig,
|
||||
@@ -48,6 +52,46 @@ from .config import (
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _log_phase(phase: str, **tags: Any):
|
||||
"""Log start/end timing and RSS deltas around a long-running task section.
|
||||
|
||||
Generic helper: callers pass arbitrary ``key=value`` tags
|
||||
(e.g. ``scan_id``, ``framework``, ``provider_id``) and they are
|
||||
emitted as part of the structured log line, so Grafana/Datadog/
|
||||
CloudWatch queries can pivot by whichever dimension is relevant to
|
||||
the task. ``getrusage`` returns KB on Linux and bytes on macOS;
|
||||
the values are still useful in relative terms even though units
|
||||
differ across platforms.
|
||||
"""
|
||||
tag_str = " ".join(f"{key}={value}" for key, value in tags.items())
|
||||
suffix = f" {tag_str}" if tag_str else ""
|
||||
|
||||
start = time.perf_counter()
|
||||
rss_before = _resource_module.getrusage(_resource_module.RUSAGE_SELF).ru_maxrss
|
||||
logger.info("phase_start phase=%s%s rss_kb=%d", phase, suffix, rss_before)
|
||||
try:
|
||||
yield
|
||||
except Exception:
|
||||
elapsed = time.perf_counter() - start
|
||||
logger.exception(
|
||||
"phase_failed phase=%s%s elapsed_s=%.2f", phase, suffix, elapsed
|
||||
)
|
||||
raise
|
||||
else:
|
||||
elapsed = time.perf_counter() - start
|
||||
rss_after = _resource_module.getrusage(_resource_module.RUSAGE_SELF).ru_maxrss
|
||||
logger.info(
|
||||
"phase_end phase=%s%s elapsed_s=%.2f rss_kb=%d delta_rss_kb=%d",
|
||||
phase,
|
||||
suffix,
|
||||
elapsed,
|
||||
rss_after,
|
||||
rss_after - rss_before,
|
||||
)
|
||||
|
||||
|
||||
# Register fonts (done once at module load)
|
||||
_fonts_registered: bool = False
|
||||
|
||||
@@ -335,6 +379,7 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Provider | None = None,
|
||||
requirement_statistics: dict[str, dict[str, int]] | None = None,
|
||||
findings_cache: dict[str, list[FindingOutput]] | None = None,
|
||||
prowler_provider: Any | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Generate the PDF compliance report.
|
||||
@@ -351,23 +396,35 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Optional pre-fetched Provider object
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings cache
|
||||
prowler_provider: Optional pre-initialized Prowler provider. When
|
||||
generating multiple reports for the same scan the master
|
||||
function initializes this once and passes it in to avoid
|
||||
re-running boto3/Azure-SDK setup per framework.
|
||||
**kwargs: Additional framework-specific arguments
|
||||
"""
|
||||
framework = self.config.display_name
|
||||
logger.info(
|
||||
"Generating %s report for scan %s", self.config.display_name, scan_id
|
||||
"report_generation_start framework=%s scan_id=%s compliance_id=%s",
|
||||
framework,
|
||||
scan_id,
|
||||
compliance_id,
|
||||
)
|
||||
|
||||
try:
|
||||
# 1. Load compliance data
|
||||
data = self._load_compliance_data(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
compliance_id=compliance_id,
|
||||
provider_id=provider_id,
|
||||
provider_obj=provider_obj,
|
||||
requirement_statistics=requirement_statistics,
|
||||
findings_cache=findings_cache,
|
||||
)
|
||||
with _log_phase(
|
||||
"load_compliance_data", scan_id=scan_id, framework=framework
|
||||
):
|
||||
data = self._load_compliance_data(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
compliance_id=compliance_id,
|
||||
provider_id=provider_id,
|
||||
provider_obj=provider_obj,
|
||||
requirement_statistics=requirement_statistics,
|
||||
findings_cache=findings_cache,
|
||||
prowler_provider=prowler_provider,
|
||||
)
|
||||
|
||||
# 2. Create PDF document
|
||||
doc = self._create_document(output_path, data)
|
||||
@@ -377,37 +434,54 @@ class BaseComplianceReportGenerator(ABC):
|
||||
elements = []
|
||||
|
||||
# Cover page (lightweight)
|
||||
elements.extend(self.create_cover_page(data))
|
||||
elements.append(PageBreak())
|
||||
with _log_phase("cover_page", scan_id=scan_id, framework=framework):
|
||||
elements.extend(self.create_cover_page(data))
|
||||
elements.append(PageBreak())
|
||||
|
||||
# Executive summary (framework-specific)
|
||||
elements.extend(self.create_executive_summary(data))
|
||||
with _log_phase("executive_summary", scan_id=scan_id, framework=framework):
|
||||
elements.extend(self.create_executive_summary(data))
|
||||
|
||||
# Body sections (charts + requirements index)
|
||||
# Override _build_body_sections() in subclasses to change section order
|
||||
elements.extend(self._build_body_sections(data))
|
||||
with _log_phase("body_sections", scan_id=scan_id, framework=framework):
|
||||
elements.extend(self._build_body_sections(data))
|
||||
|
||||
# Detailed findings - heaviest section, loads findings on-demand
|
||||
logger.info("Building detailed findings section...")
|
||||
elements.extend(self.create_detailed_findings(data, **kwargs))
|
||||
gc.collect() # Free findings data after processing
|
||||
with _log_phase("detailed_findings", scan_id=scan_id, framework=framework):
|
||||
elements.extend(self.create_detailed_findings(data, **kwargs))
|
||||
gc.collect() # Free findings data after processing
|
||||
|
||||
# 4. Build the PDF
|
||||
logger.info("Building PDF document with %d elements...", len(elements))
|
||||
self._build_pdf(doc, elements, data)
|
||||
logger.info(
|
||||
"doc_build_about_to_run framework=%s scan_id=%s elements=%d",
|
||||
framework,
|
||||
scan_id,
|
||||
len(elements),
|
||||
)
|
||||
with _log_phase("doc_build", scan_id=scan_id, framework=framework):
|
||||
self._build_pdf(doc, elements, data)
|
||||
|
||||
# Final cleanup
|
||||
del elements
|
||||
gc.collect()
|
||||
|
||||
logger.info("Successfully generated report at %s", output_path)
|
||||
logger.info(
|
||||
"report_generation_end framework=%s scan_id=%s output_path=%s",
|
||||
framework,
|
||||
scan_id,
|
||||
output_path,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
tb_lineno = e.__traceback__.tb_lineno if e.__traceback__ else "unknown"
|
||||
logger.error("Error generating report, line %s -- %s", tb_lineno, e)
|
||||
logger.error("Full traceback:\n%s", traceback.format_exc())
|
||||
except Exception:
|
||||
# logger.exception captures the full traceback; the contextual
|
||||
# keys keep production search-by-scan-id viable.
|
||||
logger.exception(
|
||||
"report_generation_failed framework=%s scan_id=%s compliance_id=%s",
|
||||
framework,
|
||||
scan_id,
|
||||
compliance_id,
|
||||
)
|
||||
raise
|
||||
|
||||
def _build_body_sections(self, data: ComplianceData) -> list:
|
||||
@@ -638,15 +712,25 @@ class BaseComplianceReportGenerator(ABC):
|
||||
for req in requirements:
|
||||
check_ids_to_load.extend(req.checks)
|
||||
|
||||
# Load findings on-demand only for the checks that will be displayed
|
||||
# Uses the shared findings cache to avoid duplicate queries across reports
|
||||
# Load findings on-demand only for the checks that will be displayed.
|
||||
# When ``only_failed`` is active at requirement level, also push the
|
||||
# FAIL filter down to the finding level: a requirement marked FAIL
|
||||
# because 1/1000 findings failed must not render a table dominated by
|
||||
# 999 PASS rows. That hides the actual failure under noise and
|
||||
# makes the per-check cap truncate the wrong rows.
|
||||
# ``total_counts`` is populated with the pre-cap total per check_id
|
||||
# (FAIL-only when only_failed is active) so the "Showing first N of
|
||||
# M" banner uses the same denominator the reader cares about.
|
||||
logger.info("Loading findings on-demand for %d requirements", len(requirements))
|
||||
total_counts: dict[str, int] = {}
|
||||
findings_by_check_id = _load_findings_for_requirement_checks(
|
||||
data.tenant_id,
|
||||
data.scan_id,
|
||||
check_ids_to_load,
|
||||
data.prowler_provider,
|
||||
data.findings_by_check_id, # Pass the cache to update it
|
||||
total_counts_out=total_counts,
|
||||
only_failed_findings=only_failed,
|
||||
)
|
||||
|
||||
for req in requirements:
|
||||
@@ -678,9 +762,31 @@ class BaseComplianceReportGenerator(ABC):
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Create findings table
|
||||
findings_table = self._create_findings_table(findings)
|
||||
elements.append(findings_table)
|
||||
# Surface truncation BEFORE the tables so readers see it
|
||||
# at the same scroll position as the data itself, not
|
||||
# after thousands of rendered rows.
|
||||
loaded = len(findings)
|
||||
total = total_counts.get(check_id, loaded)
|
||||
if total > loaded:
|
||||
kind = "failed findings" if only_failed else "findings"
|
||||
elements.append(
|
||||
Paragraph(
|
||||
f"<b>⚠ Showing first {loaded:,} of "
|
||||
f"{total:,} {kind} for this check.</b> "
|
||||
f"Use the CSV or JSON-OCSF export for the full "
|
||||
f"list. The PDF caps detail rows to keep "
|
||||
f"the report readable and bounded in size.",
|
||||
self.styles["normal"],
|
||||
)
|
||||
)
|
||||
elements.append(Spacer(1, 0.05 * inch))
|
||||
|
||||
# Create chunked findings tables to prevent OOM when a
|
||||
# single check has thousands of findings (ReportLab
|
||||
# resolves layout per Flowable, so many small tables
|
||||
# render contiguously with a bounded memory peak).
|
||||
findings_tables = self._create_findings_tables(findings)
|
||||
elements.extend(findings_tables)
|
||||
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
@@ -735,6 +841,7 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Provider | None,
|
||||
requirement_statistics: dict | None,
|
||||
findings_cache: dict | None,
|
||||
prowler_provider: Any | None = None,
|
||||
) -> ComplianceData:
|
||||
"""Load and aggregate compliance data from the database.
|
||||
|
||||
@@ -746,6 +853,9 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Optional pre-fetched Provider
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings
|
||||
prowler_provider: Optional pre-initialized Prowler provider. When
|
||||
the master function initializes it once and passes it in,
|
||||
we skip the per-report ``initialize_prowler_provider`` call.
|
||||
|
||||
Returns:
|
||||
Aggregated ComplianceData object
|
||||
@@ -755,7 +865,8 @@ class BaseComplianceReportGenerator(ABC):
|
||||
if provider_obj is None:
|
||||
provider_obj = Provider.objects.get(id=provider_id)
|
||||
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
if prowler_provider is None:
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework
|
||||
@@ -823,13 +934,32 @@ class BaseComplianceReportGenerator(ABC):
|
||||
) -> SimpleDocTemplate:
|
||||
"""Create the PDF document template.
|
||||
|
||||
Validates that ``output_path`` is a filesystem path string with an
|
||||
existing parent directory. SimpleDocTemplate technically accepts a
|
||||
BytesIO too, but we want every report to land on disk so the
|
||||
Celery worker doesn't hold the full PDF in memory while uploading
|
||||
to S3.
|
||||
|
||||
Args:
|
||||
output_path: Path for the output PDF
|
||||
data: Compliance data for metadata
|
||||
|
||||
Returns:
|
||||
Configured SimpleDocTemplate
|
||||
|
||||
Raises:
|
||||
TypeError: ``output_path`` is not a string.
|
||||
FileNotFoundError: The parent directory does not exist.
|
||||
"""
|
||||
if not isinstance(output_path, str):
|
||||
raise TypeError(
|
||||
"output_path must be a filesystem path string; "
|
||||
f"got {type(output_path).__name__}"
|
||||
)
|
||||
parent_dir = os.path.dirname(output_path)
|
||||
if parent_dir and not os.path.isdir(parent_dir):
|
||||
raise FileNotFoundError(f"Output directory does not exist: {parent_dir}")
|
||||
|
||||
return SimpleDocTemplate(
|
||||
output_path,
|
||||
pagesize=letter,
|
||||
@@ -876,47 +1006,10 @@ class BaseComplianceReportGenerator(ABC):
|
||||
onLaterPages=add_footer,
|
||||
)
|
||||
|
||||
def _create_findings_table(self, findings: list[FindingOutput]) -> Any:
|
||||
"""Create a findings table.
|
||||
|
||||
Args:
|
||||
findings: List of finding objects
|
||||
|
||||
Returns:
|
||||
ReportLab Table element
|
||||
"""
|
||||
|
||||
def get_finding_title(f):
|
||||
metadata = getattr(f, "metadata", None)
|
||||
if metadata:
|
||||
return getattr(metadata, "CheckTitle", getattr(f, "check_id", ""))
|
||||
return getattr(f, "check_id", "")
|
||||
|
||||
def get_resource_name(f):
|
||||
name = getattr(f, "resource_name", "")
|
||||
if not name:
|
||||
name = getattr(f, "resource_uid", "")
|
||||
return name
|
||||
|
||||
def get_severity(f):
|
||||
metadata = getattr(f, "metadata", None)
|
||||
if metadata:
|
||||
return getattr(metadata, "Severity", "").capitalize()
|
||||
return ""
|
||||
|
||||
# Convert findings to dicts for the table
|
||||
data = []
|
||||
for f in findings:
|
||||
item = {
|
||||
"title": get_finding_title(f),
|
||||
"resource_name": get_resource_name(f),
|
||||
"severity": get_severity(f),
|
||||
"status": getattr(f, "status", "").upper(),
|
||||
"region": getattr(f, "region", "global"),
|
||||
}
|
||||
data.append(item)
|
||||
|
||||
columns = [
|
||||
# Column layout shared by all findings sub-tables. Defined as a method so
|
||||
# subclasses can override it without re-implementing the chunking logic.
|
||||
def _findings_table_columns(self) -> list[ColumnConfig]:
|
||||
return [
|
||||
ColumnConfig("Finding", 2.5 * inch, "title"),
|
||||
ColumnConfig("Resource", 3 * inch, "resource_name"),
|
||||
ColumnConfig("Severity", 0.9 * inch, "severity"),
|
||||
@@ -924,9 +1017,122 @@ class BaseComplianceReportGenerator(ABC):
|
||||
ColumnConfig("Region", 0.9 * inch, "region"),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _finding_to_row(f: FindingOutput) -> dict[str, str]:
|
||||
"""Project a FindingOutput onto the row dict the table expects.
|
||||
|
||||
Kept defensive: missing metadata or attributes return empty strings
|
||||
rather than raising, so a single malformed finding never breaks the
|
||||
whole report.
|
||||
"""
|
||||
metadata = getattr(f, "metadata", None)
|
||||
title = (
|
||||
getattr(metadata, "CheckTitle", getattr(f, "check_id", ""))
|
||||
if metadata
|
||||
else getattr(f, "check_id", "")
|
||||
)
|
||||
resource_name = getattr(f, "resource_name", "") or getattr(
|
||||
f, "resource_uid", ""
|
||||
)
|
||||
severity = getattr(metadata, "Severity", "").capitalize() if metadata else ""
|
||||
return {
|
||||
"title": title,
|
||||
"resource_name": resource_name,
|
||||
"severity": severity,
|
||||
"status": getattr(f, "status", "").upper(),
|
||||
"region": getattr(f, "region", "global"),
|
||||
}
|
||||
|
||||
def _create_findings_tables(
|
||||
self,
|
||||
findings: list[FindingOutput],
|
||||
chunk_size: int | None = None,
|
||||
) -> list[Any]:
|
||||
"""Build a list of small findings tables to keep ``doc.build()`` memory bounded.
|
||||
|
||||
ReportLab resolves layout (column widths, row heights, page-breaks)
|
||||
per Flowable. A single ``LongTable`` of 15k rows forces all of that
|
||||
to be computed at once and reliably OOMs the worker on large scans.
|
||||
Splitting into chunks of ``chunk_size`` rows produces an equivalent-
|
||||
looking PDF (LongTable repeats headers; chunks render contiguously)
|
||||
with a bounded memory peak per chunk.
|
||||
|
||||
Args:
|
||||
findings: List of finding objects for a single check.
|
||||
chunk_size: Rows per sub-table. ``None`` uses
|
||||
``FINDINGS_TABLE_CHUNK_SIZE`` from config.
|
||||
|
||||
Returns:
|
||||
List of ReportLab flowables (interleaved ``Table``/``LongTable``
|
||||
and small ``Spacer`` between chunks). Empty list when there are
|
||||
no findings.
|
||||
"""
|
||||
if not findings:
|
||||
return []
|
||||
|
||||
chunk_size = chunk_size or FINDINGS_TABLE_CHUNK_SIZE
|
||||
|
||||
# Build all rows first so we can chunk without re-walking the
|
||||
# FindingOutput list. Malformed findings are skipped with a logged
|
||||
# exception, never enough to abort the entire report.
|
||||
rows: list[dict[str, str]] = []
|
||||
for f in findings:
|
||||
try:
|
||||
rows.append(self._finding_to_row(f))
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Skipping malformed finding while building table for check %s",
|
||||
getattr(f, "check_id", "unknown"),
|
||||
)
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
columns = self._findings_table_columns()
|
||||
|
||||
flowables: list = []
|
||||
total = len(rows)
|
||||
for start in range(0, total, chunk_size):
|
||||
chunk = rows[start : start + chunk_size]
|
||||
flowables.append(
|
||||
create_data_table(
|
||||
data=chunk,
|
||||
columns=columns,
|
||||
header_color=self.config.primary_color,
|
||||
normal_style=self.styles["normal_center"],
|
||||
)
|
||||
)
|
||||
# A tiny spacer between chunks keeps them visually contiguous
|
||||
# without forcing a page-break (KeepTogether would negate the
|
||||
# memory benefit of chunking).
|
||||
if start + chunk_size < total:
|
||||
flowables.append(Spacer(1, 0.05 * inch))
|
||||
|
||||
if total > chunk_size:
|
||||
logger.debug(
|
||||
"Built %d findings sub-tables (chunk_size=%d, total_findings=%d)",
|
||||
(total + chunk_size - 1) // chunk_size,
|
||||
chunk_size,
|
||||
total,
|
||||
)
|
||||
|
||||
return flowables
|
||||
|
||||
def _create_findings_table(self, findings: list[FindingOutput]) -> Any:
|
||||
"""Deprecated alias kept for backwards compatibility.
|
||||
|
||||
Returns the first chunk produced by ``_create_findings_tables``.
|
||||
New callers MUST use ``_create_findings_tables``, which returns a
|
||||
list of flowables and is what ``create_detailed_findings`` invokes.
|
||||
"""
|
||||
flowables = self._create_findings_tables(findings)
|
||||
if flowables:
|
||||
return flowables[0]
|
||||
# Empty input → return an empty (header-only) table so callers that
|
||||
# used to receive a Table never get None.
|
||||
return create_data_table(
|
||||
data=data,
|
||||
columns=columns,
|
||||
data=[],
|
||||
columns=self._findings_table_columns(),
|
||||
header_color=self.config.primary_color,
|
||||
normal_style=self.styles["normal_center"],
|
||||
)
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import gc
|
||||
import io
|
||||
import math
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
import matplotlib
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
# Use non-interactive Agg backend for memory efficiency in server environments
|
||||
# This MUST be set before importing pyplot
|
||||
@@ -20,6 +22,26 @@ from .config import ( # noqa: E402
|
||||
CHART_DPI_DEFAULT,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def _log_chart_built(name: str, dpi: int, buffer: io.BytesIO, started: float) -> None:
|
||||
"""Emit a structured DEBUG line summarising a chart render.
|
||||
|
||||
Centralised so the formatting stays consistent across all chart helpers
|
||||
and so we never accidentally pay for buffer.getbuffer().nbytes when
|
||||
debug logging is disabled.
|
||||
"""
|
||||
if logger.isEnabledFor(10): # logging.DEBUG
|
||||
logger.debug(
|
||||
"chart_built name=%s dpi=%d bytes=%d elapsed_s=%.2f",
|
||||
name,
|
||||
dpi,
|
||||
buffer.getbuffer().nbytes,
|
||||
time.perf_counter() - started,
|
||||
)
|
||||
|
||||
|
||||
# Use centralized DPI setting from config
|
||||
DEFAULT_CHART_DPI = CHART_DPI_DEFAULT
|
||||
|
||||
@@ -77,6 +99,7 @@ def create_vertical_bar_chart(
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
_started = time.perf_counter()
|
||||
if color_func is None:
|
||||
color_func = get_chart_color_for_percentage
|
||||
|
||||
@@ -122,6 +145,7 @@ def create_vertical_bar_chart(
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
_log_chart_built("vertical_bar", dpi, buffer, _started)
|
||||
return buffer
|
||||
|
||||
|
||||
@@ -156,6 +180,7 @@ def create_horizontal_bar_chart(
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
_started = time.perf_counter()
|
||||
if color_func is None:
|
||||
color_func = get_chart_color_for_percentage
|
||||
|
||||
@@ -207,6 +232,7 @@ def create_horizontal_bar_chart(
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
_log_chart_built("horizontal_bar", dpi, buffer, _started)
|
||||
return buffer
|
||||
|
||||
|
||||
@@ -239,6 +265,7 @@ def create_radar_chart(
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
_started = time.perf_counter()
|
||||
num_vars = len(labels)
|
||||
angles = [n / float(num_vars) * 2 * math.pi for n in range(num_vars)]
|
||||
|
||||
@@ -275,6 +302,7 @@ def create_radar_chart(
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
_log_chart_built("radar", dpi, buffer, _started)
|
||||
return buffer
|
||||
|
||||
|
||||
@@ -303,6 +331,7 @@ def create_pie_chart(
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
_started = time.perf_counter()
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
_, _, autotexts = ax.pie(
|
||||
@@ -330,6 +359,7 @@ def create_pie_chart(
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
_log_chart_built("pie", dpi, buffer, _started)
|
||||
return buffer
|
||||
|
||||
|
||||
@@ -362,6 +392,7 @@ def create_stacked_bar_chart(
|
||||
Returns:
|
||||
BytesIO buffer containing the PNG image
|
||||
"""
|
||||
_started = time.perf_counter()
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
# Default colors if not provided
|
||||
@@ -401,4 +432,5 @@ def create_stacked_bar_chart(
|
||||
plt.close(fig)
|
||||
gc.collect() # Force garbage collection after heavy matplotlib operation
|
||||
|
||||
_log_chart_built("stacked_bar", dpi, buffer, _started)
|
||||
return buffer
|
||||
|
||||
@@ -0,0 +1,755 @@
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle
|
||||
|
||||
from api.models import StatusChoices
|
||||
|
||||
from .base import (
|
||||
BaseComplianceReportGenerator,
|
||||
ComplianceData,
|
||||
RequirementData,
|
||||
get_requirement_metadata,
|
||||
)
|
||||
from .charts import (
|
||||
create_horizontal_bar_chart,
|
||||
create_pie_chart,
|
||||
create_stacked_bar_chart,
|
||||
get_chart_color_for_percentage,
|
||||
)
|
||||
from .components import ColumnConfig, create_data_table, escape_html, truncate_text
|
||||
from .config import (
|
||||
CHART_COLOR_GREEN_1,
|
||||
CHART_COLOR_RED,
|
||||
CHART_COLOR_YELLOW,
|
||||
COLOR_BG_BLUE,
|
||||
COLOR_BLUE,
|
||||
COLOR_BORDER_GRAY,
|
||||
COLOR_DARK_GRAY,
|
||||
COLOR_GRAY,
|
||||
COLOR_GRID_GRAY,
|
||||
COLOR_HIGH_RISK,
|
||||
COLOR_LIGHT_BLUE,
|
||||
COLOR_SAFE,
|
||||
COLOR_WHITE,
|
||||
)
|
||||
|
||||
# Ordered buckets used both in the executive summary tables and the charts
|
||||
# section. Exposed as module constants so the two call sites never drift.
|
||||
_PROFILE_BUCKET_ORDER: tuple[str, ...] = ("L1", "L2", "Other")
|
||||
_ASSESSMENT_BUCKET_ORDER: tuple[str, ...] = ("Automated", "Manual")
|
||||
|
||||
# Anchored matchers for profile normalization — substring checks on "L1"/"L2"
|
||||
# would happily match unrelated tokens like "CL2 Worker" or "HL2" coming from
|
||||
# future CIS profile enum values.
|
||||
_LEVEL_2_RE = re.compile(r"(?:\bLevel\s*2\b|\bL2\b|Level_2)")
|
||||
_LEVEL_1_RE = re.compile(r"(?:\bLevel\s*1\b|\bL1\b|Level_1)")
|
||||
|
||||
|
||||
def _normalize_profile(profile: Any) -> str:
|
||||
"""Bucket a CIS Profile enum/string into one of: ``L1``, ``L2``, ``Other``.
|
||||
|
||||
The ``CIS_Requirement_Attribute_Profile`` enum has values like
|
||||
``"Level 1"``, ``"Level 2"``, ``"E3 Level 1"``, ``"E5 Level 2"``. We
|
||||
collapse them into three buckets to keep charts and badges readable
|
||||
across CIS variants, using anchored regex matches so that future enum
|
||||
values cannot accidentally promote e.g. ``"CL2 Worker"`` into ``L2``.
|
||||
|
||||
Args:
|
||||
profile: The profile value (enum member, string, or ``None``).
|
||||
|
||||
Returns:
|
||||
One of ``"L1"``, ``"L2"``, ``"Other"``.
|
||||
"""
|
||||
if profile is None:
|
||||
return "Other"
|
||||
value = getattr(profile, "value", None) or str(profile)
|
||||
if _LEVEL_2_RE.search(value):
|
||||
return "L2"
|
||||
if _LEVEL_1_RE.search(value):
|
||||
return "L1"
|
||||
return "Other"
|
||||
|
||||
|
||||
def _profile_badge_text(bucket: str) -> str:
|
||||
"""Map a normalized profile bucket (L1/L2/Other) to a short badge label."""
|
||||
return {"L1": "Level 1", "L2": "Level 2"}.get(bucket, "Other")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CIS Report Generator
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class CISReportGenerator(BaseComplianceReportGenerator):
|
||||
"""
|
||||
PDF report generator for CIS (Center for Internet Security) Benchmarks.
|
||||
|
||||
CIS differs from single-version frameworks (ENS, NIS2, CSA) in that:
|
||||
- Each provider has multiple CIS versions (e.g. AWS: 1.4, 1.5, ..., 6.0).
|
||||
- Section names differ across versions and providers and MUST be derived
|
||||
at runtime from the loaded compliance data.
|
||||
- Requirements carry Profile (Level 1/Level 2) and AssessmentStatus
|
||||
(Automated/Manual) attributes that drive the executive summary and
|
||||
charts.
|
||||
|
||||
This generator produces:
|
||||
- Cover page with Prowler logo and dynamic CIS version/provider metadata
|
||||
- Executive summary with overall compliance score, counts, and breakdowns
|
||||
by Profile and AssessmentStatus
|
||||
- Charts: overall status pie, pass rate by section (horizontal bar),
|
||||
Level 1 vs Level 2 pass/fail distribution (stacked bar)
|
||||
- Requirements index grouped by dynamic section
|
||||
- Detailed findings for FAIL requirements with CIS-specific audit /
|
||||
remediation / rationale details
|
||||
"""
|
||||
|
||||
# Per-run memoization cache for ``_compute_statistics``. ``generate()``
|
||||
# is the public entry point and is called once per PDF, so scoping the
|
||||
# cache to the last seen ComplianceData instance is enough to avoid the
|
||||
# double computation between executive summary and charts section.
|
||||
_stats_cache_key: int | None = None
|
||||
_stats_cache_value: dict | None = None
|
||||
|
||||
# Body section ordering — ensure every top-level section starts on its
|
||||
# own clean page. The base class only puts a PageBreak AFTER Charts and
|
||||
# Requirements Index, so Executive Summary and Charts end up sharing a
|
||||
# page. This override prepends a PageBreak so Compliance Analysis always
|
||||
# begins on a fresh page.
|
||||
def _build_body_sections(self, data: ComplianceData) -> list:
|
||||
return [PageBreak(), *super()._build_body_sections(data)]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Cover page override — shows dynamic CIS version + provider in the title
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def create_cover_page(self, data: ComplianceData) -> list:
|
||||
"""Create the CIS report cover page with Prowler + CIS logos side by side."""
|
||||
elements = []
|
||||
|
||||
# Create logos side by side (same pattern as NIS2 / ENS)
|
||||
prowler_logo_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../assets/img/prowler_logo.png"
|
||||
)
|
||||
cis_logo_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../assets/img/cis_logo.png"
|
||||
)
|
||||
|
||||
if os.path.exists(cis_logo_path):
|
||||
prowler_logo = Image(prowler_logo_path, width=3.5 * inch, height=0.7 * inch)
|
||||
cis_logo = Image(cis_logo_path, width=2.3 * inch, height=1.1 * inch)
|
||||
logos_table = Table(
|
||||
[[prowler_logo, cis_logo]], colWidths=[4 * inch, 2.5 * inch]
|
||||
)
|
||||
logos_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("ALIGN", (0, 0), (0, 0), "LEFT"),
|
||||
("ALIGN", (1, 0), (1, 0), "RIGHT"),
|
||||
("VALIGN", (0, 0), (0, 0), "MIDDLE"),
|
||||
("VALIGN", (1, 0), (1, 0), "MIDDLE"),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(logos_table)
|
||||
elif os.path.exists(prowler_logo_path):
|
||||
# Fallback: only the Prowler logo if the CIS asset is missing
|
||||
elements.append(Image(prowler_logo_path, width=5 * inch, height=1 * inch))
|
||||
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
# Dynamic title: "CIS Benchmark v5.0 — AWS Compliance Report"
|
||||
provider_label = ""
|
||||
if data.provider_obj:
|
||||
provider_label = f" — {data.provider_obj.provider.upper()}"
|
||||
title_text = (
|
||||
f"CIS Benchmark v{data.version}{provider_label}<br/>Compliance Report"
|
||||
)
|
||||
elements.append(Paragraph(title_text, self.styles["title"]))
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
# Metadata table via base class helper
|
||||
info_rows = self._build_info_rows(data, language=self.config.language)
|
||||
metadata_data = []
|
||||
for label, value in info_rows:
|
||||
if label in ("Name:", "Description:") and value:
|
||||
metadata_data.append(
|
||||
[label, Paragraph(str(value), self.styles["normal_center"])]
|
||||
)
|
||||
else:
|
||||
metadata_data.append([label, value])
|
||||
|
||||
metadata_table = Table(metadata_data, colWidths=[2 * inch, 4 * inch])
|
||||
metadata_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, -1), COLOR_BLUE),
|
||||
("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
|
||||
("BACKGROUND", (1, 0), (1, -1), COLOR_BG_BLUE),
|
||||
("TEXTCOLOR", (1, 0), (1, -1), COLOR_GRAY),
|
||||
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
|
||||
("ALIGN", (0, 0), (-1, -1), "LEFT"),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 11),
|
||||
("GRID", (0, 0), (-1, -1), 1, COLOR_BORDER_GRAY),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 10),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 10),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 8),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 8),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(metadata_table)
|
||||
|
||||
return elements
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Executive Summary
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def create_executive_summary(self, data: ComplianceData) -> list:
|
||||
"""Create the CIS executive summary section."""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Executive Summary", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
stats = self._compute_statistics(data)
|
||||
|
||||
# --- Summary metrics table ---
|
||||
summary_data = [
|
||||
["Metric", "Value"],
|
||||
["Total Requirements", str(stats["total"])],
|
||||
["Passed", str(stats["passed"])],
|
||||
["Failed", str(stats["failed"])],
|
||||
["Manual", str(stats["manual"])],
|
||||
["Overall Compliance", f"{stats['overall_compliance']:.1f}%"],
|
||||
]
|
||||
summary_table = Table(summary_data, colWidths=[3 * inch, 2 * inch])
|
||||
summary_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("BACKGROUND", (0, 2), (0, 2), COLOR_SAFE),
|
||||
("TEXTCOLOR", (0, 2), (0, 2), COLOR_WHITE),
|
||||
("BACKGROUND", (0, 3), (0, 3), COLOR_HIGH_RISK),
|
||||
("TEXTCOLOR", (0, 3), (0, 3), COLOR_WHITE),
|
||||
("BACKGROUND", (0, 4), (0, 4), COLOR_DARK_GRAY),
|
||||
("TEXTCOLOR", (0, 4), (0, 4), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "PlusJakartaSans"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 12),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 10),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, COLOR_BORDER_GRAY),
|
||||
("BOTTOMPADDING", (0, 0), (-1, 0), 10),
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(1, 1),
|
||||
(1, -1),
|
||||
[COLOR_WHITE, COLOR_BG_BLUE],
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(summary_table)
|
||||
elements.append(Spacer(1, 0.25 * inch))
|
||||
|
||||
# --- Profile breakdown table ---
|
||||
elements.append(Paragraph("Breakdown by Profile", self.styles["h2"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
profile_counts = stats["profile_counts"]
|
||||
profile_table_data = [["Profile", "Passed", "Failed", "Manual", "Total"]]
|
||||
for bucket in _PROFILE_BUCKET_ORDER:
|
||||
counts = profile_counts.get(bucket, {"passed": 0, "failed": 0, "manual": 0})
|
||||
total = counts["passed"] + counts["failed"] + counts["manual"]
|
||||
if total == 0:
|
||||
continue
|
||||
profile_table_data.append(
|
||||
[
|
||||
_profile_badge_text(bucket),
|
||||
str(counts["passed"]),
|
||||
str(counts["failed"]),
|
||||
str(counts["manual"]),
|
||||
str(total),
|
||||
]
|
||||
)
|
||||
profile_table = Table(
|
||||
profile_table_data,
|
||||
colWidths=[1.5 * inch, 1 * inch, 1 * inch, 1 * inch, 1 * inch],
|
||||
)
|
||||
profile_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 9),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY),
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(0, 1),
|
||||
(-1, -1),
|
||||
[COLOR_WHITE, COLOR_BG_BLUE],
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(profile_table)
|
||||
elements.append(Spacer(1, 0.25 * inch))
|
||||
|
||||
# --- Assessment status breakdown ---
|
||||
elements.append(Paragraph("Breakdown by Assessment Status", self.styles["h2"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
assessment_counts = stats["assessment_counts"]
|
||||
assessment_table_data = [["Assessment", "Passed", "Failed", "Manual", "Total"]]
|
||||
for bucket in _ASSESSMENT_BUCKET_ORDER:
|
||||
counts = assessment_counts.get(
|
||||
bucket, {"passed": 0, "failed": 0, "manual": 0}
|
||||
)
|
||||
total = counts["passed"] + counts["failed"] + counts["manual"]
|
||||
if total == 0:
|
||||
continue
|
||||
assessment_table_data.append(
|
||||
[
|
||||
bucket,
|
||||
str(counts["passed"]),
|
||||
str(counts["failed"]),
|
||||
str(counts["manual"]),
|
||||
str(total),
|
||||
]
|
||||
)
|
||||
assessment_table = Table(
|
||||
assessment_table_data,
|
||||
colWidths=[1.5 * inch, 1 * inch, 1 * inch, 1 * inch, 1 * inch],
|
||||
)
|
||||
assessment_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_LIGHT_BLUE),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 9),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY),
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(0, 1),
|
||||
(-1, -1),
|
||||
[COLOR_WHITE, COLOR_BG_BLUE],
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(assessment_table)
|
||||
elements.append(Spacer(1, 0.25 * inch))
|
||||
|
||||
# --- Top 5 failing sections ---
|
||||
top_failing = stats["top_failing_sections"]
|
||||
if top_failing:
|
||||
elements.append(
|
||||
Paragraph("Top Sections with Lowest Compliance", self.styles["h2"])
|
||||
)
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
top_table_data = [["Section", "Passed", "Failed", "Compliance"]]
|
||||
for section_label, section_stats in top_failing:
|
||||
passed = section_stats["passed"]
|
||||
failed = section_stats["failed"]
|
||||
total = passed + failed
|
||||
pct = (passed / total * 100) if total > 0 else 100
|
||||
top_table_data.append(
|
||||
[
|
||||
truncate_text(section_label, 55),
|
||||
str(passed),
|
||||
str(failed),
|
||||
f"{pct:.1f}%",
|
||||
]
|
||||
)
|
||||
top_table = Table(
|
||||
top_table_data,
|
||||
colWidths=[3.5 * inch, 0.9 * inch, 0.9 * inch, 1.2 * inch],
|
||||
)
|
||||
top_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_HIGH_RISK),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 1), (-1, -1), 9),
|
||||
("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY),
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(0, 1),
|
||||
(-1, -1),
|
||||
[COLOR_WHITE, COLOR_BG_BLUE],
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
elements.append(top_table)
|
||||
|
||||
return elements
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Charts section
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def create_charts_section(self, data: ComplianceData) -> list:
|
||||
"""Create the CIS charts section."""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Compliance Analysis", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
# --- Pie chart: overall Pass / Fail / Manual ---
|
||||
stats = self._compute_statistics(data)
|
||||
pie_labels = []
|
||||
pie_values = []
|
||||
pie_colors = []
|
||||
if stats["passed"] > 0:
|
||||
pie_labels.append(f"Pass ({stats['passed']})")
|
||||
pie_values.append(stats["passed"])
|
||||
pie_colors.append(CHART_COLOR_GREEN_1)
|
||||
if stats["failed"] > 0:
|
||||
pie_labels.append(f"Fail ({stats['failed']})")
|
||||
pie_values.append(stats["failed"])
|
||||
pie_colors.append(CHART_COLOR_RED)
|
||||
if stats["manual"] > 0:
|
||||
pie_labels.append(f"Manual ({stats['manual']})")
|
||||
pie_values.append(stats["manual"])
|
||||
pie_colors.append(CHART_COLOR_YELLOW)
|
||||
|
||||
if pie_values:
|
||||
elements.append(Paragraph("Overall Status Distribution", self.styles["h2"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
pie_buffer = create_pie_chart(
|
||||
labels=pie_labels,
|
||||
values=pie_values,
|
||||
colors=pie_colors,
|
||||
)
|
||||
pie_buffer.seek(0)
|
||||
elements.append(Image(pie_buffer, width=4.5 * inch, height=4.5 * inch))
|
||||
elements.append(Spacer(1, 0.2 * inch))
|
||||
|
||||
# --- Horizontal bar: pass rate by section ---
|
||||
section_stats = stats["section_stats"]
|
||||
if section_stats:
|
||||
elements.append(PageBreak())
|
||||
elements.append(Paragraph("Compliance by Section", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
elements.append(
|
||||
Paragraph(
|
||||
"The following chart shows compliance percentage for each CIS "
|
||||
"section based on automated checks:",
|
||||
self.styles["normal_center"],
|
||||
)
|
||||
)
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
# Sort sections by pass rate descending for readability
|
||||
sorted_sections = sorted(
|
||||
section_stats.items(),
|
||||
key=lambda item: (
|
||||
(item[1]["passed"] / (item[1]["passed"] + item[1]["failed"]) * 100)
|
||||
if (item[1]["passed"] + item[1]["failed"]) > 0
|
||||
else 100
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
bar_labels = []
|
||||
bar_values = []
|
||||
for section_label, section_data in sorted_sections:
|
||||
total = section_data["passed"] + section_data["failed"]
|
||||
if total == 0:
|
||||
continue
|
||||
pct = (section_data["passed"] / total) * 100
|
||||
bar_labels.append(truncate_text(section_label, 60))
|
||||
bar_values.append(pct)
|
||||
|
||||
if bar_values:
|
||||
bar_buffer = create_horizontal_bar_chart(
|
||||
labels=bar_labels,
|
||||
values=bar_values,
|
||||
xlabel="Compliance (%)",
|
||||
color_func=get_chart_color_for_percentage,
|
||||
label_fontsize=9,
|
||||
)
|
||||
bar_buffer.seek(0)
|
||||
elements.append(Image(bar_buffer, width=6.5 * inch, height=5 * inch))
|
||||
|
||||
# --- Stacked bar: Level 1 vs Level 2 pass/fail ---
|
||||
profile_counts = stats["profile_counts"]
|
||||
has_profile_data = any(
|
||||
(counts["passed"] + counts["failed"]) > 0
|
||||
for counts in profile_counts.values()
|
||||
)
|
||||
if has_profile_data:
|
||||
elements.append(PageBreak())
|
||||
elements.append(Paragraph("Profile Breakdown", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
elements.append(
|
||||
Paragraph(
|
||||
"Distribution of Pass / Fail / Manual across CIS profile levels.",
|
||||
self.styles["normal_center"],
|
||||
)
|
||||
)
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
profile_labels = []
|
||||
pass_series = []
|
||||
fail_series = []
|
||||
manual_series = []
|
||||
for bucket in _PROFILE_BUCKET_ORDER:
|
||||
counts = profile_counts.get(bucket)
|
||||
if not counts:
|
||||
continue
|
||||
total = counts["passed"] + counts["failed"] + counts["manual"]
|
||||
if total == 0:
|
||||
continue
|
||||
profile_labels.append(_profile_badge_text(bucket))
|
||||
pass_series.append(counts["passed"])
|
||||
fail_series.append(counts["failed"])
|
||||
manual_series.append(counts["manual"])
|
||||
|
||||
if profile_labels:
|
||||
stacked_buffer = create_stacked_bar_chart(
|
||||
labels=profile_labels,
|
||||
data_series={
|
||||
"Pass": pass_series,
|
||||
"Fail": fail_series,
|
||||
"Manual": manual_series,
|
||||
},
|
||||
xlabel="Profile",
|
||||
ylabel="Requirements",
|
||||
)
|
||||
stacked_buffer.seek(0)
|
||||
elements.append(Image(stacked_buffer, width=6 * inch, height=4 * inch))
|
||||
|
||||
return elements
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Requirements Index
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def create_requirements_index(self, data: ComplianceData) -> list:
|
||||
"""Create the CIS requirements index grouped by dynamic section."""
|
||||
elements = []
|
||||
|
||||
elements.append(Paragraph("Requirements Index", self.styles["h1"]))
|
||||
elements.append(Spacer(1, 0.1 * inch))
|
||||
|
||||
sections = self._derive_sections(data)
|
||||
by_section: dict[str, list[dict]] = defaultdict(list)
|
||||
for req in data.requirements:
|
||||
meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
section = "Other"
|
||||
profile_bucket = "Other"
|
||||
assessment = ""
|
||||
if meta:
|
||||
section = getattr(meta, "Section", "Other") or "Other"
|
||||
profile_bucket = _normalize_profile(getattr(meta, "Profile", None))
|
||||
assessment_enum = getattr(meta, "AssessmentStatus", None)
|
||||
assessment = getattr(assessment_enum, "value", None) or str(
|
||||
assessment_enum or ""
|
||||
)
|
||||
by_section[section].append(
|
||||
{
|
||||
"id": req.id,
|
||||
"description": truncate_text(req.description, 80),
|
||||
"profile": _profile_badge_text(profile_bucket),
|
||||
"assessment": assessment or "-",
|
||||
"status": (req.status or "").upper(),
|
||||
}
|
||||
)
|
||||
|
||||
columns = [
|
||||
ColumnConfig("ID", 0.9 * inch, "id", align="LEFT"),
|
||||
ColumnConfig("Description", 3.0 * inch, "description", align="LEFT"),
|
||||
ColumnConfig("Profile", 0.9 * inch, "profile"),
|
||||
ColumnConfig("Assessment", 1 * inch, "assessment"),
|
||||
ColumnConfig("Status", 0.9 * inch, "status"),
|
||||
]
|
||||
|
||||
for section in sections:
|
||||
rows = by_section.get(section, [])
|
||||
if not rows:
|
||||
continue
|
||||
elements.append(Paragraph(truncate_text(section, 90), self.styles["h2"]))
|
||||
elements.append(Spacer(1, 0.05 * inch))
|
||||
table = create_data_table(
|
||||
data=rows,
|
||||
columns=columns,
|
||||
header_color=self.config.primary_color,
|
||||
normal_style=self.styles["normal_center"],
|
||||
)
|
||||
elements.append(table)
|
||||
elements.append(Spacer(1, 0.15 * inch))
|
||||
|
||||
return elements
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Detailed findings hook — inject CIS-specific rationale / audit content
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _render_requirement_detail_extras(
|
||||
self, req: RequirementData, data: ComplianceData
|
||||
) -> list:
|
||||
"""Render CIS rationale, impact, audit, remediation and references."""
|
||||
extras = []
|
||||
meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if meta is None:
|
||||
return extras
|
||||
|
||||
field_map = [
|
||||
("Rationale", "RationaleStatement"),
|
||||
("Impact", "ImpactStatement"),
|
||||
("Audit Procedure", "AuditProcedure"),
|
||||
("Remediation", "RemediationProcedure"),
|
||||
("References", "References"),
|
||||
]
|
||||
|
||||
for label, attr_name in field_map:
|
||||
value = getattr(meta, attr_name, None)
|
||||
if not value:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
continue
|
||||
extras.append(Paragraph(f"<b>{label}:</b>", self.styles["h3"]))
|
||||
extras.append(Paragraph(escape_html(text), self.styles["normal"]))
|
||||
extras.append(Spacer(1, 0.08 * inch))
|
||||
|
||||
return extras
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _derive_sections(self, data: ComplianceData) -> list[str]:
|
||||
"""Extract ordered unique Section names from loaded compliance data."""
|
||||
seen: dict[str, bool] = {}
|
||||
for req in data.requirements:
|
||||
meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if meta is None:
|
||||
continue
|
||||
section = getattr(meta, "Section", None) or "Other"
|
||||
if section not in seen:
|
||||
seen[section] = True
|
||||
return list(seen.keys())
|
||||
|
||||
def _compute_statistics(self, data: ComplianceData) -> dict:
|
||||
"""Aggregate all statistics needed for summary and charts.
|
||||
|
||||
Memoized per-``ComplianceData`` instance via ``_stats_cache_*``: the
|
||||
executive summary and the charts section both need the same numbers,
|
||||
so they would otherwise re-iterate the requirements twice. We key on
|
||||
``id(data)`` because ``ComplianceData`` is a dataclass and its
|
||||
instances are not hashable.
|
||||
|
||||
Returns a dict with:
|
||||
- total, passed, failed, manual: int
|
||||
- overall_compliance: float (percentage)
|
||||
- profile_counts: {"L1": {"passed", "failed", "manual"}, ...}
|
||||
- assessment_counts: {"Automated": {...}, "Manual": {...}}
|
||||
- section_stats: {section_name: {"passed", "failed", "manual"}, ...}
|
||||
- top_failing_sections: list[(section_name, stats)] (up to 5)
|
||||
"""
|
||||
cache_key = id(data)
|
||||
if self._stats_cache_key == cache_key and self._stats_cache_value is not None:
|
||||
return self._stats_cache_value
|
||||
stats = self._compute_statistics_uncached(data)
|
||||
self._stats_cache_key = cache_key
|
||||
self._stats_cache_value = stats
|
||||
return stats
|
||||
|
||||
def _compute_statistics_uncached(self, data: ComplianceData) -> dict:
|
||||
"""Actual aggregation kernel; call ``_compute_statistics`` instead."""
|
||||
total = len(data.requirements)
|
||||
passed = sum(1 for r in data.requirements if r.status == StatusChoices.PASS)
|
||||
failed = sum(1 for r in data.requirements if r.status == StatusChoices.FAIL)
|
||||
manual = sum(1 for r in data.requirements if r.status == StatusChoices.MANUAL)
|
||||
|
||||
evaluated = passed + failed
|
||||
overall_compliance = (passed / evaluated * 100) if evaluated > 0 else 100.0
|
||||
|
||||
profile_counts: dict[str, dict[str, int]] = {
|
||||
"L1": {"passed": 0, "failed": 0, "manual": 0},
|
||||
"L2": {"passed": 0, "failed": 0, "manual": 0},
|
||||
"Other": {"passed": 0, "failed": 0, "manual": 0},
|
||||
}
|
||||
assessment_counts: dict[str, dict[str, int]] = {
|
||||
"Automated": {"passed": 0, "failed": 0, "manual": 0},
|
||||
"Manual": {"passed": 0, "failed": 0, "manual": 0},
|
||||
}
|
||||
section_stats: dict[str, dict[str, int]] = defaultdict(
|
||||
lambda: {"passed": 0, "failed": 0, "manual": 0}
|
||||
)
|
||||
|
||||
for req in data.requirements:
|
||||
meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id)
|
||||
if meta is None:
|
||||
continue
|
||||
|
||||
profile_bucket = _normalize_profile(getattr(meta, "Profile", None))
|
||||
assessment_enum = getattr(meta, "AssessmentStatus", None)
|
||||
assessment_value = getattr(assessment_enum, "value", None) or str(
|
||||
assessment_enum or ""
|
||||
)
|
||||
assessment_bucket = (
|
||||
"Automated" if assessment_value == "Automated" else "Manual"
|
||||
)
|
||||
section = getattr(meta, "Section", None) or "Other"
|
||||
|
||||
status_key = {
|
||||
StatusChoices.PASS: "passed",
|
||||
StatusChoices.FAIL: "failed",
|
||||
StatusChoices.MANUAL: "manual",
|
||||
}.get(req.status)
|
||||
if status_key is None:
|
||||
continue
|
||||
|
||||
profile_counts[profile_bucket][status_key] += 1
|
||||
assessment_counts[assessment_bucket][status_key] += 1
|
||||
section_stats[section][status_key] += 1
|
||||
|
||||
# Top 5 sections with lowest pass rate (only sections with evaluated reqs)
|
||||
def _section_rate(item):
|
||||
_, stats_ = item
|
||||
evaluated_ = stats_["passed"] + stats_["failed"]
|
||||
if evaluated_ == 0:
|
||||
return 101 # sort evaluated=0 to the bottom
|
||||
return stats_["passed"] / evaluated_ * 100
|
||||
|
||||
top_failing_sections = sorted(
|
||||
(
|
||||
item
|
||||
for item in section_stats.items()
|
||||
if (item[1]["passed"] + item[1]["failed"]) > 0
|
||||
),
|
||||
key=_section_rate,
|
||||
)[:5]
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"manual": manual,
|
||||
"overall_compliance": overall_compliance,
|
||||
"profile_counts": profile_counts,
|
||||
"assessment_counts": assessment_counts,
|
||||
"section_stats": dict(section_stats),
|
||||
"top_failing_sections": top_failing_sections,
|
||||
}
|
||||
@@ -26,6 +26,52 @@ from .config import (
|
||||
)
|
||||
|
||||
|
||||
def truncate_text(text: str, max_len: int) -> str:
|
||||
"""Truncate ``text`` to ``max_len`` characters, appending an ellipsis if cut.
|
||||
|
||||
Used by report generators that need to squeeze long descriptions, section
|
||||
titles or finding titles into a fixed-width table cell.
|
||||
|
||||
Args:
|
||||
text: Source string. ``None`` and non-string values are treated as empty.
|
||||
max_len: Maximum output length including the ellipsis. Values < 4 are
|
||||
clamped so the result never grows beyond ``max_len``.
|
||||
|
||||
Returns:
|
||||
The original string if short enough, otherwise ``text[: max_len - 3] + "..."``.
|
||||
When ``max_len < 4`` a plain substring of length ``max_len`` is returned
|
||||
so callers never get a string longer than they asked for.
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
text = str(text)
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
if max_len < 4:
|
||||
return text[:max_len]
|
||||
return text[: max_len - 3] + "..."
|
||||
|
||||
|
||||
def escape_html(text: str) -> str:
|
||||
"""Escape the minimal HTML entities required for safe ReportLab Paragraph rendering.
|
||||
|
||||
ReportLab's ``Paragraph`` parses a small HTML subset, so raw ``<``, ``>``
|
||||
and ``&`` in user-provided content (rationale, remediation, etc.) would
|
||||
break layout or be interpreted as tags. This helper mirrors
|
||||
``html.escape`` but avoids pulling in the stdlib dependency and keeps the
|
||||
output deterministic.
|
||||
|
||||
Args:
|
||||
text: Untrusted source string.
|
||||
|
||||
Returns:
|
||||
A string safe to embed inside a ReportLab Paragraph.
|
||||
"""
|
||||
return (
|
||||
str(text or "").replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
)
|
||||
|
||||
|
||||
def get_color_for_risk_level(risk_level: int) -> colors.Color:
|
||||
"""
|
||||
Get color based on risk level.
|
||||
@@ -429,8 +475,15 @@ def create_data_table(
|
||||
else:
|
||||
value = item.get(col.field, "")
|
||||
|
||||
# Wrap every string cell in Paragraph so the data rows keep the
|
||||
# caller-supplied font/colour/alignment. Skipping Paragraph for
|
||||
# short cells (a tempting micro-optimisation) breaks visual
|
||||
# consistency: ReportLab Table falls back to Helvetica/black for
|
||||
# raw strings, mixing fonts within the same table.
|
||||
# ``escape_html`` keeps ``<``/``>``/``&`` in resource names from
|
||||
# breaking Paragraph's mini-HTML parser.
|
||||
if normal_style and isinstance(value, str):
|
||||
value = Paragraph(value, normal_style)
|
||||
value = Paragraph(escape_html(value), normal_style)
|
||||
row.append(value)
|
||||
table_data.append(row)
|
||||
|
||||
@@ -462,17 +515,26 @@ def create_data_table(
|
||||
for idx, col in enumerate(columns):
|
||||
styles.append(("ALIGN", (idx, 0), (idx, -1), col.align))
|
||||
|
||||
# Alternate row backgrounds - skip for very large tables as it adds memory overhead
|
||||
# Alternate row backgrounds: single O(1) ROWBACKGROUNDS style entry.
|
||||
# The previous implementation appended N per-row BACKGROUND commands,
|
||||
# which scaled the TableStyle list linearly with row count. ReportLab
|
||||
# cycles through the colour list row-by-row so the visual is identical.
|
||||
# The ALTERNATE_ROWS_MAX_SIZE cap is preserved to mirror legacy
|
||||
# behaviour (very large tables stay plain), but the memory cost of the
|
||||
# styles list is now constant regardless of row count.
|
||||
if (
|
||||
alternate_rows
|
||||
and len(table_data) > 1
|
||||
and len(table_data) <= ALTERNATE_ROWS_MAX_SIZE
|
||||
):
|
||||
for i in range(1, len(table_data)):
|
||||
if i % 2 == 0:
|
||||
styles.append(
|
||||
("BACKGROUND", (0, i), (-1, i), colors.Color(0.98, 0.98, 0.98))
|
||||
)
|
||||
styles.append(
|
||||
(
|
||||
"ROWBACKGROUNDS",
|
||||
(0, 1),
|
||||
(-1, -1),
|
||||
[colors.white, colors.Color(0.98, 0.98, 0.98)],
|
||||
)
|
||||
)
|
||||
|
||||
table.setStyle(TableStyle(styles))
|
||||
return table
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from reportlab.lib import colors
|
||||
@@ -23,6 +24,47 @@ ALTERNATE_ROWS_MAX_SIZE = 200
|
||||
# Larger = fewer queries but more memory per batch
|
||||
FINDINGS_BATCH_SIZE = 2000
|
||||
|
||||
# Maximum rows per findings sub-table. ReportLab resolves layout per Flowable;
|
||||
# splitting a huge findings list into multiple smaller tables keeps the peak
|
||||
# memory of doc.build() bounded. A single 15k-row LongTable would force
|
||||
# ReportLab to compute all column widths/row heights/page-breaks at once and
|
||||
# OOM the worker; 300-row chunks are rendered contiguously with negligible
|
||||
# visual impact.
|
||||
FINDINGS_TABLE_CHUNK_SIZE = 300
|
||||
|
||||
# Maximum findings rendered per check in the detailed-findings section.
|
||||
#
|
||||
# Product behaviour: compliance PDFs render at most ``MAX_FINDINGS_PER_CHECK``
|
||||
# **failed** findings per check (PASS rows are excluded at SQL level by the
|
||||
# ``only_failed`` flag that all four list-rendering frameworks default to:
|
||||
# ThreatScore, NIS2, CSA, CIS; ENS does not render finding tables). Above
|
||||
# this cap each affected check renders an in-PDF banner
|
||||
# ("Showing first 100 of N failed findings for this check. Use the CSV
|
||||
# or JSON export for the full list") so the reader knows the table is
|
||||
# truncated and where to find the full data.
|
||||
#
|
||||
# Why a cap exists at all:
|
||||
# * ``FindingOutput.transform_api_finding`` is O(N) per finding (Pydantic
|
||||
# v1 validation + nested model construction).
|
||||
# * ReportLab resolves layout per Flowable; thousands of sub-tables make
|
||||
# ``doc.build()`` very slow and grow the PDF unboundedly.
|
||||
# * A human-readable executive/auditor PDF does not need 12,000 rows for
|
||||
# one check; that is forensic data and lives in the CSV/JSON exports.
|
||||
#
|
||||
# Why 100 specifically:
|
||||
# * Covers ~99% of real scans without truncation (most checks emit far
|
||||
# fewer than 100 findings even in enterprise estates).
|
||||
# * Worst-case rendered rows = 100 × ~500 checks = 50k rows across all
|
||||
# frameworks, which keeps RSS bounded and a 5-framework run completes
|
||||
# in minutes instead of hours.
|
||||
#
|
||||
# Override at runtime via ``DJANGO_PDF_MAX_FINDINGS_PER_CHECK``:
|
||||
# * Set to ``0`` to disable the cap entirely (load every finding; only
|
||||
# advisable for small scans).
|
||||
# * Set to a larger value (e.g. ``500``) for forensic detail in big runs;
|
||||
# watch RSS in the Celery worker.
|
||||
MAX_FINDINGS_PER_CHECK = int(os.environ.get("DJANGO_PDF_MAX_FINDINGS_PER_CHECK", "100"))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Base colors
|
||||
@@ -313,6 +355,32 @@ FRAMEWORK_REGISTRY: dict[str, FrameworkConfig] = {
|
||||
has_niveles=False,
|
||||
has_weight=False,
|
||||
),
|
||||
"cis": FrameworkConfig(
|
||||
name="cis",
|
||||
display_name="CIS Benchmark",
|
||||
logo_filename=None,
|
||||
primary_color=COLOR_BLUE,
|
||||
secondary_color=COLOR_LIGHT_BLUE,
|
||||
bg_color=COLOR_BG_BLUE,
|
||||
attribute_fields=[
|
||||
"Section",
|
||||
"SubSection",
|
||||
"Profile",
|
||||
"AssessmentStatus",
|
||||
"Description",
|
||||
"RationaleStatement",
|
||||
"ImpactStatement",
|
||||
"RemediationProcedure",
|
||||
"AuditProcedure",
|
||||
"References",
|
||||
],
|
||||
sections=None, # Derived dynamically per CIS variant (section names differ across versions/providers)
|
||||
language="en",
|
||||
has_risk_levels=False,
|
||||
has_dimensions=False,
|
||||
has_niveles=False,
|
||||
has_weight=False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -336,5 +404,7 @@ def get_framework_config(compliance_id: str) -> FrameworkConfig | None:
|
||||
return FRAMEWORK_REGISTRY["nis2"]
|
||||
if "csa" in compliance_lower or "ccm" in compliance_lower:
|
||||
return FRAMEWORK_REGISTRY["csa_ccm"]
|
||||
if compliance_lower.startswith("cis_") or "cis" in compliance_lower:
|
||||
return FRAMEWORK_REGISTRY["cis"]
|
||||
|
||||
return None
|
||||
|
||||
@@ -10,16 +10,29 @@ from typing import Any
|
||||
|
||||
import sentry_sdk
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||||
from config.env import env
|
||||
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
|
||||
from django.db import IntegrityError, OperationalError
|
||||
from django.db.models import Case, Count, IntegerField, Max, Min, Prefetch, Q, Sum, When
|
||||
from django.db.models import (
|
||||
Case,
|
||||
Count,
|
||||
Exists,
|
||||
IntegerField,
|
||||
Max,
|
||||
Min,
|
||||
OuterRef,
|
||||
Prefetch,
|
||||
Q,
|
||||
Sum,
|
||||
When,
|
||||
)
|
||||
from django.utils import timezone as django_timezone
|
||||
from tasks.jobs.queries import (
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||||
)
|
||||
from tasks.utils import CustomEncoder
|
||||
from tasks.utils import CustomEncoder, batched
|
||||
|
||||
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
from api.constants import SEVERITY_ORDER
|
||||
@@ -189,8 +202,9 @@ def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
|
||||
"iam_inline_policy_allows_privilege_escalation",
|
||||
},
|
||||
"ec2-imdsv1": {
|
||||
"ec2_instance_imdsv2_enabled"
|
||||
}, # AWS only - IMDSv1 enabled findings
|
||||
"ec2_instance_imdsv2_enabled",
|
||||
"ec2_instance_account_imdsv2_enabled",
|
||||
}, # AWS only - instance-level IMDSv1 exposure and account IMDS defaults
|
||||
}
|
||||
for category_name, check_ids in attack_surface_check_mappings.items():
|
||||
if check_ids is None:
|
||||
@@ -1197,11 +1211,39 @@ def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
muted_changed=agg["muted_changed"],
|
||||
)
|
||||
for agg in aggregation
|
||||
if agg["resources__service"] is not None
|
||||
and agg["resources__region"] is not None
|
||||
}
|
||||
# Delete first so re-runs (e.g. post-mute reaggregation) don't hit
|
||||
# the `unique_scan_summary` constraint.
|
||||
ScanSummary.objects.filter(tenant_id=tenant_id, scan_id=scan_id).delete()
|
||||
ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000)
|
||||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||||
# `unique_scan_summary`; race-safe under concurrent writers.
|
||||
ScanSummary.objects.bulk_create(
|
||||
scan_aggregations,
|
||||
batch_size=3000,
|
||||
update_conflicts=True,
|
||||
unique_fields=[
|
||||
"tenant",
|
||||
"scan",
|
||||
"check_id",
|
||||
"service",
|
||||
"severity",
|
||||
"region",
|
||||
],
|
||||
update_fields=[
|
||||
"_pass",
|
||||
"fail",
|
||||
"muted",
|
||||
"total",
|
||||
"new",
|
||||
"changed",
|
||||
"unchanged",
|
||||
"fail_new",
|
||||
"fail_changed",
|
||||
"pass_new",
|
||||
"pass_changed",
|
||||
"muted_new",
|
||||
"muted_changed",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _aggregate_findings_by_region(
|
||||
@@ -1546,13 +1588,24 @@ def aggregate_attack_surface(tenant_id: str, scan_id: str):
|
||||
)
|
||||
)
|
||||
|
||||
# Bulk create overview records
|
||||
if overview_objects:
|
||||
with rls_transaction(tenant_id):
|
||||
AttackSurfaceOverview.objects.bulk_create(overview_objects, batch_size=500)
|
||||
logger.info(
|
||||
f"Created {len(overview_objects)} attack surface overview records for scan {scan_id}"
|
||||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||||
# `unique_attack_surface_per_scan`; race-safe under concurrent writers.
|
||||
AttackSurfaceOverview.objects.bulk_create(
|
||||
overview_objects,
|
||||
batch_size=500,
|
||||
update_conflicts=True,
|
||||
unique_fields=["tenant_id", "scan_id", "attack_surface_type"],
|
||||
update_fields=[
|
||||
"total_findings",
|
||||
"failed_findings",
|
||||
"muted_failed_findings",
|
||||
],
|
||||
)
|
||||
logger.info(
|
||||
f"Upserted {len(overview_objects)} attack surface overview records for scan {scan_id}"
|
||||
)
|
||||
else:
|
||||
logger.info(f"No attack surface overview records created for scan {scan_id}")
|
||||
|
||||
@@ -2030,3 +2083,169 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
}
|
||||
|
||||
|
||||
def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dict:
|
||||
"""Zero failed_findings_count for resources missing from a completed full-scope scan.
|
||||
|
||||
Resources that exist in the database for the scan's provider but were not
|
||||
touched by this scan are treated as ephemeral. We keep their historical
|
||||
findings, but reset the denormalized counter that drives the Resources page
|
||||
sort so they stop ranking at the top.
|
||||
|
||||
Skipped (no-op) when:
|
||||
- The scan is not in COMPLETED state.
|
||||
- The scan ran with any scoping filter in scanner_args (partial scope).
|
||||
|
||||
Query design (must scale to 500k+ resources per provider):
|
||||
Phase 1 — collect ephemeral IDs with one anti-join read.
|
||||
Outer filter ``(tenant_id, provider_id, failed_findings_count > 0)``
|
||||
uses ``resources_tenant_provider_idx``. The correlated
|
||||
``NOT EXISTS`` subquery hits the implicit unique index
|
||||
``(tenant_id, scan_id, resource_id)`` on ``ResourceScanSummary``.
|
||||
``NOT EXISTS`` (vs ``NOT IN``) is null-safe and lets the planner
|
||||
choose between hash anti-join and indexed nested-loop anti-join.
|
||||
``.iterator(chunk_size=...)`` skips the queryset cache so memory
|
||||
stays bounded while streaming UUIDs.
|
||||
Phase 2 — UPDATE in fixed-size batches.
|
||||
One large UPDATE would hold row-exclusive locks for seconds and
|
||||
create a WAL spike. Batched UPDATEs by ``id__in`` (~1k rows each)
|
||||
hit the primary key, keep each lock window ~50ms, bound WAL chunks,
|
||||
and let other writers proceed between batches.
|
||||
``failed_findings_count__gt=0`` in the UPDATE is idempotent under
|
||||
concurrent scans and skips no-op rewrites.
|
||||
Reads use the primary DB, not the replica: ``ResourceScanSummary`` rows
|
||||
were written by the same scan task that triggered this one, so replica
|
||||
lag could falsely classify scanned resources as ephemeral.
|
||||
|
||||
Scope detection (``Scan.is_full_scope()``) derives the set of scoping
|
||||
scanner_args from ``prowler.lib.scan.scan.Scan.__init__`` via
|
||||
introspection, so the API can never drift from the SDK's filter
|
||||
contract. Imported scans are also rejected by trigger — they may only
|
||||
cover a partial slice of resources.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
scan = Scan.objects.filter(tenant_id=tenant_id, id=scan_id).first()
|
||||
|
||||
if scan is None:
|
||||
logger.warning(f"Scan {scan_id} not found")
|
||||
return {"status": "skipped", "reason": "scan not found"}
|
||||
|
||||
if scan.state != StateChoices.COMPLETED:
|
||||
logger.info(f"Scan {scan_id} not completed; skipping ephemeral reset")
|
||||
return {"status": "skipped", "reason": "scan not completed"}
|
||||
|
||||
if not scan.is_full_scope():
|
||||
logger.info(
|
||||
f"Scan {scan_id} ran with scoping filters; skipping ephemeral reset"
|
||||
)
|
||||
return {"status": "skipped", "reason": "partial scan scope"}
|
||||
|
||||
# Race protection: if a newer completed full-scope scan exists for this
|
||||
# provider, our ResourceScanSummary set is stale relative to the resources'
|
||||
# current failed_findings_count values (which the newer scan already
|
||||
# refreshed). Wiping based on the older scan would zero counts the newer
|
||||
# scan just set. Skip and let the newer scan's reset task do the work; if
|
||||
# this task was delayed in the queue, that's the correct outcome.
|
||||
# `completed_at__isnull=False` is required: Postgres orders NULL first in
|
||||
# DESC, so a sibling COMPLETED scan with a missing completed_at would sort
|
||||
# as "newest" and incorrectly cause us to skip.
|
||||
with rls_transaction(tenant_id):
|
||||
latest_full_scope_scan_id = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=scan.provider_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("-completed_at", "-inserted_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
)
|
||||
if latest_full_scope_scan_id != scan.id:
|
||||
logger.info(
|
||||
f"Scan {scan_id} is not the latest completed scan for provider "
|
||||
f"{scan.provider_id}; skipping ephemeral reset"
|
||||
)
|
||||
return {"status": "skipped", "reason": "newer scan exists"}
|
||||
|
||||
# Defensive gate: ResourceScanSummary rows are written by perform_prowler_scan
|
||||
# via best-effort bulk_create. If those writes failed silently (or the scan
|
||||
# genuinely produced resources but no summaries were persisted), the
|
||||
# ~Exists(in_scan) anti-join below would classify EVERY resource for this
|
||||
# provider as ephemeral and zero their counts. Bail loudly instead.
|
||||
with rls_transaction(tenant_id):
|
||||
summaries_present = ResourceScanSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists()
|
||||
if scan.unique_resource_count > 0 and not summaries_present:
|
||||
logger.error(
|
||||
f"Scan {scan_id} reports {scan.unique_resource_count} unique "
|
||||
f"resources but no ResourceScanSummary rows are persisted; "
|
||||
f"skipping ephemeral reset to avoid wiping valid counts"
|
||||
)
|
||||
return {"status": "skipped", "reason": "summaries missing"}
|
||||
|
||||
# Stays on the primary DB intentionally. ResourceScanSummary rows are
|
||||
# written by perform_prowler_scan in the same chain that triggered this
|
||||
# task, so replica lag could return an empty/partial summary set; a stale
|
||||
# read here would classify every Resource as ephemeral and wipe valid
|
||||
# failed_findings_count values on the primary. Same rationale as
|
||||
# update_provider_compliance_scores below in this module.
|
||||
# Materializing the ID list (rather than streaming the iterator into
|
||||
# batched UPDATEs) is intentional: it lets the UPDATEs run in their own
|
||||
# short rls_transactions instead of one long transaction holding row locks
|
||||
# on every batch. At 500k UUIDs the peak memory is ~40 MB — acceptable for
|
||||
# a Celery worker — and is the better trade-off versus a multi-second
|
||||
# write-lock window blocking concurrent scans.
|
||||
with rls_transaction(tenant_id):
|
||||
in_scan = ResourceScanSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_id=OuterRef("pk"),
|
||||
)
|
||||
ephemeral_ids = list(
|
||||
Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=scan.provider_id,
|
||||
failed_findings_count__gt=0,
|
||||
)
|
||||
.filter(~Exists(in_scan))
|
||||
.values_list("id", flat=True)
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
)
|
||||
|
||||
if not ephemeral_ids:
|
||||
logger.info(f"No ephemeral resources for scan {scan_id}")
|
||||
return {
|
||||
"status": "completed",
|
||||
"scan_id": str(scan_id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
"reset": 0,
|
||||
}
|
||||
|
||||
total_updated = 0
|
||||
for batch, _ in batched(ephemeral_ids, DJANGO_FINDINGS_BATCH_SIZE):
|
||||
# batched() always yields a final tuple, which is empty when the input
|
||||
# length is an exact multiple of the batch size. Skip it so we don't
|
||||
# issue a no-op UPDATE ... WHERE id IN ().
|
||||
if not batch:
|
||||
continue
|
||||
with rls_transaction(tenant_id):
|
||||
total_updated += Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id__in=batch,
|
||||
failed_findings_count__gt=0,
|
||||
).update(failed_findings_count=0)
|
||||
|
||||
logger.info(
|
||||
f"Ephemeral resource reset for scan {scan_id}: "
|
||||
f"{total_updated} resources zeroed for provider {scan.provider_id}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"scan_id": str(scan_id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
"reset": total_updated,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||||
from django.db.models import Count, Q
|
||||
from django.db.models import Count, F, Q, Window
|
||||
from django.db.models.functions import RowNumber
|
||||
from tasks.jobs.reports.config import MAX_FINDINGS_PER_CHECK
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
@@ -154,6 +156,8 @@ def _load_findings_for_requirement_checks(
|
||||
check_ids: list[str],
|
||||
prowler_provider,
|
||||
findings_cache: dict[str, list[FindingOutput]] | None = None,
|
||||
total_counts_out: dict[str, int] | None = None,
|
||||
only_failed_findings: bool = False,
|
||||
) -> dict[str, list[FindingOutput]]:
|
||||
"""
|
||||
Load findings for specific check IDs on-demand with optional caching.
|
||||
@@ -178,6 +182,23 @@ def _load_findings_for_requirement_checks(
|
||||
prowler_provider: The initialized Prowler provider instance.
|
||||
findings_cache (dict, optional): Cache of already loaded findings.
|
||||
If provided, checks are first looked up in cache before querying database.
|
||||
total_counts_out (dict, optional): If provided, populated with
|
||||
``{check_id: total_findings_in_db}`` BEFORE any per-check cap is
|
||||
applied. Lets callers render a "Showing first N of M" banner for
|
||||
truncated checks. Only populated for ``check_ids`` actually
|
||||
queried (cache hits keep whatever value the caller already had).
|
||||
When ``only_failed_findings=True`` the total is FAIL-only.
|
||||
only_failed_findings (bool): When True, push the ``status=FAIL``
|
||||
filter down into the SQL query so PASS rows are never loaded
|
||||
from the DB nor pydantic-transformed. This matches the
|
||||
``only_failed`` requirement-level filter applied at PDF render
|
||||
time: a requirement marked FAIL because 1/1000 findings failed
|
||||
shouldn't render a table of 999 PASS rows. That hides the
|
||||
actual failure under noise and wastes the per-check cap on
|
||||
irrelevant data. NOTE: the findings cache stores whatever the
|
||||
first caller asked for, so all callers in a single
|
||||
``generate_compliance_reports`` run MUST pass the same flag
|
||||
(which they do: it threads from ``only_failed`` defaults).
|
||||
|
||||
Returns:
|
||||
dict[str, list[FindingOutput]]: Dictionary mapping check_id to list of FindingOutput objects.
|
||||
@@ -222,17 +243,88 @@ def _load_findings_for_requirement_checks(
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Use iterator with chunk_size for memory-efficient streaming
|
||||
# chunk_size controls how many rows Django fetches from DB at once
|
||||
findings_queryset = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
check_id__in=check_ids_to_load,
|
||||
)
|
||||
.order_by("check_id", "uid")
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
base_qs = Finding.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
check_id__in=check_ids_to_load,
|
||||
)
|
||||
if only_failed_findings:
|
||||
# Push the FAIL filter down into SQL: DB returns ~N×FAIL
|
||||
# rows instead of N×ALL, and we never spend pydantic CPU on
|
||||
# PASS findings the PDF would never render.
|
||||
base_qs = base_qs.filter(status=StatusChoices.FAIL)
|
||||
|
||||
# Aggregate totals once so we (a) know which checks need capping
|
||||
# and (b) can surface "Showing first N of M" in the PDF banner.
|
||||
# Cheap: a single COUNT grouped by check_id.
|
||||
totals: dict[str, int] = {
|
||||
row["check_id"]: row["total"]
|
||||
for row in base_qs.values("check_id").annotate(total=Count("id"))
|
||||
}
|
||||
if total_counts_out is not None:
|
||||
total_counts_out.update(totals)
|
||||
|
||||
cap = MAX_FINDINGS_PER_CHECK
|
||||
checks_over_cap = (
|
||||
{cid for cid, n in totals.items() if n > cap} if cap > 0 else set()
|
||||
)
|
||||
|
||||
# Use iterator with chunk_size for memory-efficient streaming.
|
||||
# FindingOutput.transform_api_finding (prowler/lib/outputs/finding.py)
|
||||
# reads finding.resources.first() and resource.tags.all() per
|
||||
# finding, which without prefetch generates 2N queries per chunk.
|
||||
# prefetch_related runs once per iterator chunk (Django >=4.1) and
|
||||
# collapses that into a constant 2 extra queries per chunk.
|
||||
if checks_over_cap:
|
||||
# Two-step query so we can both cap rows per check AND attach
|
||||
# prefetch_related on the streamed results:
|
||||
#
|
||||
# 1) ``ranked`` annotates every matching finding with a
|
||||
# per-check row number via a window function. The
|
||||
# partition keeps numbering independent per check, and
|
||||
# ordering by ``uid`` makes the "first N" selection
|
||||
# deterministic across runs (same scan → same rows).
|
||||
#
|
||||
# 2) The outer ``Finding.all_objects.filter(id__in=...)``
|
||||
# keeps only IDs whose row number is within the cap and
|
||||
# re-opens a plain queryset on it. Django cannot combine
|
||||
# ``Window`` annotations with ``prefetch_related`` on the
|
||||
# same queryset (the window is evaluated post-aggregation
|
||||
# and the prefetch loader fights with it), so the inner
|
||||
# SELECT becomes a subquery and the outer queryset is
|
||||
# free to prefetch resources/tags as usual.
|
||||
#
|
||||
# PostgreSQL only materialises
|
||||
# ``cap * |checks_over_cap| + sum(uncapped)`` rows for the
|
||||
# window step, vs the full table scan the previous path did.
|
||||
ranked = base_qs.annotate(
|
||||
rn=Window(
|
||||
expression=RowNumber(),
|
||||
partition_by=[F("check_id")],
|
||||
order_by=F("uid").asc(),
|
||||
)
|
||||
)
|
||||
findings_queryset = (
|
||||
Finding.all_objects.filter(
|
||||
id__in=ranked.filter(rn__lte=cap).values("id")
|
||||
)
|
||||
.prefetch_related("resources", "resources__tags")
|
||||
.order_by("check_id", "uid")
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
)
|
||||
logger.info(
|
||||
"Per-check cap=%d active for %d checks (max %d each); "
|
||||
"skipping transform for surplus rows",
|
||||
cap,
|
||||
len(checks_over_cap),
|
||||
cap,
|
||||
)
|
||||
else:
|
||||
findings_queryset = (
|
||||
base_qs.prefetch_related("resources", "resources__tags")
|
||||
.order_by("check_id", "uid")
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
)
|
||||
|
||||
# Pre-initialize empty lists for all check_ids to load
|
||||
# This avoids repeated dict lookups and 'if not in' checks
|
||||
@@ -248,7 +340,11 @@ def _load_findings_for_requirement_checks(
|
||||
findings_count += 1
|
||||
|
||||
logger.info(
|
||||
f"Loaded {findings_count} findings for {len(check_ids_to_load)} checks"
|
||||
"Loaded %d findings for %d checks (truncated %d checks total=%d)",
|
||||
findings_count,
|
||||
len(check_ids_to_load),
|
||||
len(checks_over_cap),
|
||||
sum(totals.values()),
|
||||
)
|
||||
|
||||
# Build result dict using cache references (no data duplication)
|
||||
@@ -258,3 +354,40 @@ def _load_findings_for_requirement_checks(
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _get_compliance_check_ids(compliance_obj) -> set[str]:
|
||||
"""Return the union of all check_ids referenced by a compliance framework.
|
||||
|
||||
Used by the master report orchestrator to know which checks each
|
||||
framework consumes from the shared ``findings_cache``, so that once a
|
||||
framework finishes the entries no other pending framework needs can be
|
||||
evicted from the cache (PROWLER-1733).
|
||||
|
||||
Args:
|
||||
compliance_obj: A loaded Compliance framework object exposing a
|
||||
``Requirements`` iterable, each requirement carrying ``Checks``.
|
||||
``None`` is treated as "no checks" rather than raising, so the
|
||||
caller can pass ``frameworks_bulk.get(...)`` directly without
|
||||
an extra existence check.
|
||||
|
||||
Returns:
|
||||
Set of check_id strings (empty if ``compliance_obj`` is ``None``).
|
||||
"""
|
||||
if compliance_obj is None:
|
||||
return set()
|
||||
checks: set[str] = set()
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or []
|
||||
try:
|
||||
# Defensive: Mock objects (used in unit tests) return another Mock
|
||||
# for any attribute access, which is truthy but not iterable. Treat
|
||||
# any non-iterable Requirements value as "no checks".
|
||||
for req in requirements:
|
||||
req_checks = getattr(req, "Checks", None) or []
|
||||
try:
|
||||
checks.update(req_checks)
|
||||
except TypeError:
|
||||
continue
|
||||
except TypeError:
|
||||
return set()
|
||||
return checks
|
||||
|
||||
+123
-16
@@ -20,8 +20,8 @@ from tasks.jobs.backfill import (
|
||||
backfill_finding_group_summaries,
|
||||
backfill_provider_compliance_scores,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
aggregate_scan_category_summaries,
|
||||
aggregate_scan_resource_group_summaries,
|
||||
)
|
||||
from tasks.jobs.connection import (
|
||||
check_integration_connection,
|
||||
@@ -46,7 +46,11 @@ from tasks.jobs.lighthouse_providers import (
|
||||
refresh_lighthouse_provider_models,
|
||||
)
|
||||
from tasks.jobs.muting import mute_historical_findings
|
||||
from tasks.jobs.report import generate_compliance_reports_job
|
||||
from tasks.jobs.report import (
|
||||
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
|
||||
_cleanup_stale_tmp_output_directories,
|
||||
generate_compliance_reports_job,
|
||||
)
|
||||
from tasks.jobs.scan import (
|
||||
aggregate_attack_surface,
|
||||
aggregate_daily_severity,
|
||||
@@ -54,6 +58,7 @@ from tasks.jobs.scan import (
|
||||
aggregate_findings,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
reset_ephemeral_resource_findings_count,
|
||||
update_provider_compliance_scores,
|
||||
)
|
||||
from tasks.utils import (
|
||||
@@ -64,7 +69,7 @@ from tasks.utils import (
|
||||
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.db_utils import delete_related_daily_task, rls_transaction
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
@@ -73,6 +78,7 @@ from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
@@ -154,6 +160,13 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
generate_outputs_task.si(
|
||||
scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id
|
||||
),
|
||||
# post-scan task — runs in the parallel group so a
|
||||
# failure cannot cascade into reports or integrations. Its only
|
||||
# prerequisite is that perform_prowler_scan has committed
|
||||
# ResourceScanSummary, which is true by the time this chain fires.
|
||||
reset_ephemeral_resource_findings_count_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
),
|
||||
),
|
||||
group(
|
||||
# Use optimized task that generates both reports with shared queries
|
||||
@@ -169,10 +182,25 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
).apply_async()
|
||||
|
||||
if can_provider_run_attack_paths_scan(tenant_id, provider_id):
|
||||
perform_attack_paths_scan_task.apply_async(
|
||||
# Row is normally created upstream, so this is a safeguard so we can attach the task id below
|
||||
attack_paths_scan = attack_paths_db_utils.retrieve_attack_paths_scan(
|
||||
tenant_id, scan_id
|
||||
)
|
||||
if attack_paths_scan is None:
|
||||
attack_paths_scan = attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id, scan_id, provider_id
|
||||
)
|
||||
|
||||
# Persist the Celery task id so the periodic cleanup can revoke scans stuck in SCHEDULED
|
||||
result = perform_attack_paths_scan_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
|
||||
if attack_paths_scan and result:
|
||||
attack_paths_db_utils.set_attack_paths_scan_task_id(
|
||||
tenant_id, attack_paths_scan.id, result.task_id
|
||||
)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="provider-connection-check")
|
||||
@set_tenant
|
||||
@@ -246,6 +274,17 @@ def perform_scan_task(
|
||||
Returns:
|
||||
dict: The result of the scan execution, typically including the status and results of the performed checks.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
if not Provider.objects.filter(pk=provider_id).exists():
|
||||
logger.warning(
|
||||
"scan-perform skipped: provider %s no longer exists "
|
||||
"(tenant=%s, scan=%s)",
|
||||
provider_id,
|
||||
tenant_id,
|
||||
scan_id,
|
||||
)
|
||||
return None
|
||||
|
||||
result = perform_prowler_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
@@ -282,6 +321,16 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
task_id = self.request.id
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
if not Provider.objects.filter(pk=provider_id).exists():
|
||||
logger.warning(
|
||||
"scheduled scan-perform skipped: provider %s no longer exists "
|
||||
"(tenant=%s)",
|
||||
provider_id,
|
||||
tenant_id,
|
||||
)
|
||||
delete_related_daily_task(provider_id)
|
||||
return None
|
||||
|
||||
periodic_task_instance = PeriodicTask.objects.get(
|
||||
name=f"scan-perform-scheduled-{provider_id}"
|
||||
)
|
||||
@@ -374,7 +423,8 @@ class AttackPathsScanRLSTask(RLSTask):
|
||||
SDK initialization, or Neo4j configuration errors during setup).
|
||||
"""
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, _einfo):
|
||||
def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002
|
||||
del args # Required by Celery's Task.on_failure signature; not used.
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
scan_id = kwargs.get("scan_id")
|
||||
|
||||
@@ -440,6 +490,19 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
scan_id (str): The scan identifier.
|
||||
provider_id (str): The provider_id id to be used in generating outputs.
|
||||
"""
|
||||
try:
|
||||
_cleanup_stale_tmp_output_directories(
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY,
|
||||
max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS,
|
||||
exclude_scan=(tenant_id, scan_id),
|
||||
)
|
||||
except Exception as error:
|
||||
logger.warning(
|
||||
"Skipping stale tmp cleanup before output generation for scan %s: %s",
|
||||
scan_id,
|
||||
error,
|
||||
)
|
||||
|
||||
# Check if the scan has findings
|
||||
if not ScanSummary.objects.filter(scan_id=scan_id).exists():
|
||||
logger.info(f"No findings found for scan {scan_id}")
|
||||
@@ -659,9 +722,9 @@ def backfill_finding_group_summaries_task(tenant_id: str, days: int = None):
|
||||
return backfill_finding_group_summaries(tenant_id=tenant_id, days=days)
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-category-summaries", queue="backfill")
|
||||
@shared_task(name="scan-category-summaries", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
|
||||
def aggregate_scan_category_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanCategorySummary for a completed scan.
|
||||
|
||||
@@ -671,12 +734,12 @@ def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
"""
|
||||
return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
return aggregate_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill")
|
||||
@shared_task(name="scan-resource-group-summaries", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
|
||||
def aggregate_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanGroupSummary for a completed scan.
|
||||
|
||||
@@ -686,7 +749,7 @@ def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
"""
|
||||
return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
return aggregate_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-provider-compliance-scores", queue="backfill")
|
||||
@@ -758,6 +821,32 @@ def aggregate_daily_severity_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_daily_severity(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-reset-ephemeral-resources", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def reset_ephemeral_resource_findings_count_task(tenant_id: str, scan_id: str):
|
||||
"""Reset failed_findings_count for resources missing from a completed full-scope scan.
|
||||
|
||||
Failures are swallowed and returned as a status: this task lives inside the
|
||||
post-scan group, and Celery propagates group-member exceptions into the next
|
||||
chain step — meaning a crash here would block compliance reports and
|
||||
integrations. The reset is purely cosmetic (UI sort optimization), so a
|
||||
bad run is logged and absorbed rather than allowed to cascade.
|
||||
"""
|
||||
try:
|
||||
return reset_ephemeral_resource_findings_count(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — intentionally broad
|
||||
logger.exception(
|
||||
f"reset_ephemeral_resource_findings_count failed for scan {scan_id}: {exc}"
|
||||
)
|
||||
return {
|
||||
"status": "failed",
|
||||
"scan_id": str(scan_id),
|
||||
"reason": str(exc),
|
||||
}
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-finding-group-summaries", queue="overview")
|
||||
@set_tenant(keep_tenant=True)
|
||||
@handle_provider_deletion
|
||||
@@ -778,12 +867,16 @@ def reaggregate_all_finding_group_summaries_task(tenant_id: str):
|
||||
limit. To keep the pre-aggregated tables consistent with that update,
|
||||
this task re-runs the same per-scan aggregation pipeline that scan
|
||||
completion runs on the latest completed scan of every (provider, day)
|
||||
pair, rebuilding the three tables that power the read endpoints:
|
||||
pair, rebuilding the tables that power the read endpoints:
|
||||
|
||||
- `ScanSummary` and `DailySeveritySummary` -> `/overviews/findings`,
|
||||
`/overviews/findings-severity`, `/overviews/services`.
|
||||
- `FindingGroupDailySummary` -> `/finding-groups` and
|
||||
`/finding-groups/latest`.
|
||||
- `ScanGroupSummary` -> `/overviews/resource-groups` (resource
|
||||
inventory).
|
||||
- `ScanCategorySummary` -> `/overviews/categories`.
|
||||
- `AttackSurfaceOverview` -> `/overviews/attack-surfaces`.
|
||||
|
||||
Per-scan pipelines are dispatched in parallel via a Celery group so
|
||||
wallclock scales with the worker pool.
|
||||
@@ -815,8 +908,8 @@ def reaggregate_all_finding_group_summaries_task(tenant_id: str):
|
||||
len(scan_ids),
|
||||
)
|
||||
# DailySeveritySummary reads from ScanSummary, so ScanSummary must be
|
||||
# recomputed first; FindingGroupDailySummary reads from Finding
|
||||
# directly and can run in parallel with the severity step.
|
||||
# recomputed first; the other aggregators read Finding directly and
|
||||
# can run in parallel with the severity step.
|
||||
group(
|
||||
chain(
|
||||
perform_scan_summary_task.si(tenant_id=tenant_id, scan_id=scan_id),
|
||||
@@ -827,6 +920,15 @@ def reaggregate_all_finding_group_summaries_task(tenant_id: str):
|
||||
aggregate_finding_group_summaries_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
),
|
||||
aggregate_scan_resource_group_summaries_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
),
|
||||
aggregate_scan_category_summaries_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
),
|
||||
aggregate_attack_surface_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
),
|
||||
),
|
||||
)
|
||||
for scan_id in scan_ids
|
||||
@@ -1000,13 +1102,17 @@ def jira_integration_task(
|
||||
@handle_provider_deletion
|
||||
def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: str):
|
||||
"""
|
||||
Optimized task to generate ThreatScore, ENS, NIS2, and CSA CCM reports with shared queries.
|
||||
Optimized task to generate ThreatScore, ENS, NIS2, CSA CCM and CIS reports with shared queries.
|
||||
|
||||
This task is more efficient than running separate report tasks because it reuses database queries:
|
||||
- Provider object fetched once (instead of multiple times)
|
||||
- Requirement statistics aggregated once (instead of multiple times)
|
||||
- Can reduce database load by up to 50-70%
|
||||
|
||||
CIS emits a single PDF per run: the one matching the highest CIS version
|
||||
available for the scan's provider, picked dynamically from
|
||||
``Compliance.get_bulk`` (no hard-coded provider → version mapping).
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
@@ -1023,6 +1129,7 @@ def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id:
|
||||
generate_ens=True,
|
||||
generate_nis2=True,
|
||||
generate_csa=True,
|
||||
generate_cis=True,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -135,7 +135,7 @@ class TestAttackPathsRun:
|
||||
assert result == ingestion_result
|
||||
mock_retrieve_scan.assert_called_once_with(str(tenant.id), str(scan.id))
|
||||
mock_starting.assert_called_once()
|
||||
config = mock_starting.call_args[0][2]
|
||||
config = mock_starting.call_args[0][1]
|
||||
assert config.neo4j_database == "tenant-db"
|
||||
mock_get_db_name.assert_has_calls(
|
||||
[call(attack_paths_scan.id, temporary=True), call(provider.tenant_id)]
|
||||
@@ -2732,3 +2732,143 @@ class TestCleanupStaleAttackPathsScans:
|
||||
assert result["cleaned_up_count"] == 2
|
||||
# Worker should be pinged exactly once — cache prevents second ping
|
||||
mock_alive.assert_called_once_with("shared-worker@host")
|
||||
|
||||
# `SCHEDULED` state cleanup
|
||||
def _create_scheduled_scan(
|
||||
self,
|
||||
tenant,
|
||||
provider,
|
||||
*,
|
||||
age_minutes,
|
||||
parent_state,
|
||||
with_task=True,
|
||||
):
|
||||
"""Create a SCHEDULED AttackPathsScan with a parent Scan in `parent_state`.
|
||||
|
||||
`age_minutes` controls how far in the past `started_at` is set, so
|
||||
callers can place rows safely past the cleanup cutoff.
|
||||
"""
|
||||
parent_scan = Scan.objects.create(
|
||||
name="Parent Prowler scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=parent_state,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
ap_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=parent_scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
started_at=datetime.now(tz=timezone.utc) - timedelta(minutes=age_minutes),
|
||||
)
|
||||
|
||||
task_result = None
|
||||
if with_task:
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(ap_scan.id),
|
||||
task_name="attack-paths-scan-perform",
|
||||
status="PENDING",
|
||||
)
|
||||
task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
task_runner_task=task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
ap_scan.task = task
|
||||
ap_scan.save(update_fields=["task_id"])
|
||||
|
||||
return ap_scan, task_result
|
||||
|
||||
@patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready")
|
||||
@patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database")
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.cleanup.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.cleanup._revoke_task")
|
||||
def test_cleans_up_scheduled_scan_when_parent_is_terminal(
|
||||
self,
|
||||
mock_revoke,
|
||||
mock_drop_db,
|
||||
mock_recover,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
ap_scan, task_result = self._create_scheduled_scan(
|
||||
tenant,
|
||||
provider,
|
||||
age_minutes=24 * 60 * 3, # 3 days, safely past any threshold
|
||||
parent_state=StateChoices.FAILED,
|
||||
)
|
||||
|
||||
result = cleanup_stale_attack_paths_scans()
|
||||
|
||||
assert result["cleaned_up_count"] == 1
|
||||
assert str(ap_scan.id) in result["scan_ids"]
|
||||
|
||||
ap_scan.refresh_from_db()
|
||||
assert ap_scan.state == StateChoices.FAILED
|
||||
assert ap_scan.progress == 100
|
||||
assert ap_scan.completed_at is not None
|
||||
assert ap_scan.ingestion_exceptions == {
|
||||
"global_error": "Scan never started — cleaned up by periodic task"
|
||||
}
|
||||
|
||||
# SCHEDULED revoke must NOT terminate a running worker
|
||||
mock_revoke.assert_called_once()
|
||||
assert mock_revoke.call_args.kwargs == {"terminate": False}
|
||||
|
||||
# Temp DB never created for SCHEDULED, so no drop attempted
|
||||
mock_drop_db.assert_not_called()
|
||||
# Tenant Neo4j data is untouched in this path
|
||||
mock_recover.assert_not_called()
|
||||
|
||||
task_result.refresh_from_db()
|
||||
assert task_result.status == "FAILURE"
|
||||
assert task_result.date_done is not None
|
||||
|
||||
@patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready")
|
||||
@patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database")
|
||||
@patch(
|
||||
"tasks.jobs.attack_paths.cleanup.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
@patch("tasks.jobs.attack_paths.cleanup._revoke_task")
|
||||
def test_skips_scheduled_scan_when_parent_still_in_flight(
|
||||
self,
|
||||
mock_revoke,
|
||||
mock_drop_db,
|
||||
mock_recover,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
ap_scan, _ = self._create_scheduled_scan(
|
||||
tenant,
|
||||
provider,
|
||||
age_minutes=24 * 60 * 3,
|
||||
parent_state=StateChoices.EXECUTING,
|
||||
)
|
||||
|
||||
result = cleanup_stale_attack_paths_scans()
|
||||
|
||||
assert result["cleaned_up_count"] == 0
|
||||
|
||||
ap_scan.refresh_from_db()
|
||||
assert ap_scan.state == StateChoices.SCHEDULED
|
||||
mock_revoke.assert_not_called()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user