mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-17 13:03:14 +00:00
Compare commits
115 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4ffb1dd852 | |||
| 7b8ce51263 | |||
| 262dfda0aa | |||
| 8bc42a5ded | |||
| 406712ffa3 | |||
| 6be201b8f7 | |||
| c2d7187a0b | |||
| e690e5e86b | |||
| e4d5ca11b3 | |||
| 181197177c | |||
| f21304c6a8 | |||
| 0cf48a2c35 | |||
| 6b4fb934f8 | |||
| d1ed1eddef | |||
| 0c9f4f6578 | |||
| e1f20487ce | |||
| 26b8c6b663 | |||
| 3960827a9c | |||
| e419771b04 | |||
| 94ce76d679 | |||
| 28c064a9b7 | |||
| eeb02453d1 | |||
| cb4b889b20 | |||
| f1e42d1681 | |||
| ca7ce5a8c3 | |||
| 810d8d7686 | |||
| dd1895d2c4 | |||
| b5bb85c956 | |||
| 36fe48dbc5 | |||
| 1f6fe3ab5b | |||
| 24ba4cf193 | |||
| e5bbffd47c | |||
| 566167489b | |||
| 3cb360e9ae | |||
| 24e3182329 | |||
| 49309b43d3 | |||
| 6db8ce672c | |||
| 9465b82747 | |||
| 383d2b218f | |||
| dccd674cf9 | |||
| a679865cce | |||
| 15bfa39b23 | |||
| dc3433aaf0 | |||
| 25fc285966 | |||
| 9022a3a138 | |||
| ca443b8ff1 | |||
| 79e066d3f5 | |||
| 56831a7392 | |||
| 2e82f1564f | |||
| a394c0fdf6 | |||
| 20eca78767 | |||
| bba594a1db | |||
| 65f00a197b | |||
| ce27053c2d | |||
| 610febb5d5 | |||
| c4378d5992 | |||
| f1d741214a | |||
| 285974b7d4 | |||
| 989c3b174e | |||
| 75f95559d6 | |||
| e085e14247 | |||
| 368d3a2661 | |||
| 3c8fde25ee | |||
| ec0bb53839 | |||
| bfb3fcea4c | |||
| 61cd4aea3f | |||
| 01b49f0743 | |||
| 4a5a49b5bb | |||
| a21cb64a94 | |||
| 9a50dffaa0 | |||
| e710ebff1c | |||
| b3caee88e4 | |||
| d9f90e50b8 | |||
| 58efb719fa | |||
| 355b7071aa | |||
| b994b0b14e | |||
| 6c559fbb8d | |||
| b2d74711d9 | |||
| 7e60e8f8da | |||
| 62955dd16b | |||
| 1f7caa6394 | |||
| 662e7e9e18 | |||
| e3013d9918 | |||
| 0ea2f6d67e | |||
| 7692a1d76a | |||
| 1c9afc714e | |||
| 466f1a3d73 | |||
| 061fbaa7bb | |||
| 28b045302f | |||
| 5a2226c02c | |||
| 6f172a5c19 | |||
| a7d180ea5b | |||
| d4bbc8b5ad | |||
| a5bc226f11 | |||
| 3a3d9d6146 | |||
| bcd282d3d0 | |||
| eb7949c884 | |||
| e60a4462e5 | |||
| f7f8747512 | |||
| d573af911d | |||
| cf9beb8234 | |||
| 7f67eac1bf | |||
| a652e28b4a | |||
| 1b17304c4a | |||
| c2cef99b33 | |||
| a769e37615 | |||
| 9d2a8d9108 | |||
| e05519ff9f | |||
| 67b26072f8 | |||
| 2222082631 | |||
| 8b0cb4b981 | |||
| 9422eff8ab | |||
| e3c4368d32 | |||
| 2a641b39c8 | |||
| 02b713572b |
@@ -145,7 +145,7 @@ SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.29.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.31.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: 'OSV-Scanner'
|
||||
description: 'Install osv-scanner and scan a lockfile, failing on HIGH/CRITICAL/UNKNOWN severity findings. Posts/updates a PR comment with findings on pull_request events (requires pull-requests: write).'
|
||||
description: 'Install osv-scanner and scan a lockfile, failing on CRITICAL severity findings. Posts/updates a PR comment with findings on pull_request events (requires pull-requests: write).'
|
||||
author: 'Prowler'
|
||||
|
||||
inputs:
|
||||
@@ -7,9 +7,9 @@ inputs:
|
||||
description: 'Path to the lockfile to scan, relative to the repository root (e.g. uv.lock, api/uv.lock, ui/pnpm-lock.yaml).'
|
||||
required: true
|
||||
severity-levels:
|
||||
description: 'Comma-separated severity levels that fail the scan. Default: HIGH,CRITICAL,UNKNOWN.'
|
||||
description: 'Comma-separated severity levels that fail the scan. Default: CRITICAL.'
|
||||
required: false
|
||||
default: 'HIGH,CRITICAL,UNKNOWN'
|
||||
default: 'CRITICAL'
|
||||
version:
|
||||
description: 'osv-scanner release tag to install. When overriding, you MUST also override binary-sha256.'
|
||||
required: false
|
||||
|
||||
@@ -43,8 +43,17 @@ runs:
|
||||
if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
LATEST_COMMIT=$(curl -sf \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/prowler-cloud/prowler/commits/master" \
|
||||
| jq -er '.sha') || {
|
||||
echo "::error::Failed to fetch latest prowler/master commit from the GitHub API (HTTP error or missing .sha). Check the GITHUB_TOKEN and API rate limits."
|
||||
exit 1
|
||||
}
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
@@ -54,8 +63,17 @@ runs:
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
LATEST_COMMIT=$(curl -sf \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/prowler-cloud/prowler/commits/master" \
|
||||
| jq -er '.sha') || {
|
||||
echo "::error::Failed to fetch latest prowler/master commit from the GitHub API (HTTP error or missing .sha). Check the GITHUB_TOKEN and API rate limits."
|
||||
exit 1
|
||||
}
|
||||
echo "Latest commit hash: $LATEST_COMMIT"
|
||||
sed -i "s|\(git = \"https://github\.com/prowler-cloud/prowler\.git?rev=master\)#[a-f0-9]\{40\}\"|\1#${LATEST_COMMIT}\"|g" uv.lock
|
||||
echo "Updated uv.lock entry:"
|
||||
|
||||
@@ -63,7 +63,7 @@ runs:
|
||||
exit-code: '0'
|
||||
scanners: 'vuln'
|
||||
timeout: '5m'
|
||||
version: 'v0.69.2'
|
||||
version: 'v0.71.0'
|
||||
|
||||
- name: Run Trivy vulnerability scan (SARIF)
|
||||
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
|
||||
@@ -76,7 +76,7 @@ runs:
|
||||
exit-code: '0'
|
||||
scanners: 'vuln'
|
||||
timeout: '5m'
|
||||
version: 'v0.69.2'
|
||||
version: 'v0.71.0'
|
||||
|
||||
- name: Upload Trivy results to GitHub Security tab
|
||||
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
|
||||
|
||||
@@ -6,8 +6,7 @@
|
||||
# - .github/workflows/api-security.yml, sdk-security.yml, ui-security.yml
|
||||
#
|
||||
# Severity levels (comma-separated) are read from OSV_SEVERITY_LEVELS.
|
||||
# Default: HIGH,CRITICAL,UNKNOWN — preserves prior .safety-policy.yml policy
|
||||
# (ignore-cvss-severity-below: 7 + ignore-cvss-unknown-severity: False).
|
||||
# Default: CRITICAL — only CVSS >= 9.0 findings fail the scan.
|
||||
# osv-scanner has no native CVSS threshold (google/osv-scanner#1400, closed
|
||||
# not-planned). Severity is derived from $group.max_severity (numeric CVSS
|
||||
# score string) which osv-scanner emits per group.
|
||||
@@ -33,7 +32,7 @@ set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel)"
|
||||
CONFIG="${ROOT}/osv-scanner.toml"
|
||||
SEVERITY_LEVELS="${OSV_SEVERITY_LEVELS:-HIGH,CRITICAL,UNKNOWN}"
|
||||
SEVERITY_LEVELS="${OSV_SEVERITY_LEVELS:-CRITICAL}"
|
||||
|
||||
for bin in osv-scanner jq; do
|
||||
if ! command -v "${bin}" >/dev/null 2>&1; then
|
||||
|
||||
@@ -12,9 +12,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -134,5 +131,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -16,13 +16,6 @@ on:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
paths:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-tests.yml'
|
||||
- '.github/workflows/api-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
@@ -12,9 +12,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/**'
|
||||
- '.github/workflows/mcp-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -127,5 +124,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -15,12 +15,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/pyproject.toml'
|
||||
- 'mcp_server/uv.lock'
|
||||
- '.github/workflows/mcp-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -30,7 +24,6 @@ permissions: {}
|
||||
|
||||
jobs:
|
||||
mcp-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
- '3.10'
|
||||
- '3.11'
|
||||
- '3.12'
|
||||
- '3.13'
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
|
||||
@@ -15,12 +15,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'Dockerfile*'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -111,25 +105,14 @@ jobs:
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files: |
|
||||
prowler/**
|
||||
Dockerfile*
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
.github/workflows/sdk-container-checks.yml
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
.env
|
||||
docker-compose*
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
@@ -153,5 +136,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -19,16 +19,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'prowler/**'
|
||||
- 'tests/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- '.github/workflows/sdk-tests.yml'
|
||||
- '.github/workflows/sdk-security.yml'
|
||||
- '.github/actions/setup-python-uv/**'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -71,27 +61,18 @@ jobs:
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files:
|
||||
./**
|
||||
files: |
|
||||
prowler/**
|
||||
tests/**
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
.github/workflows/sdk-tests.yml
|
||||
.github/workflows/sdk-security.yml
|
||||
.github/actions/setup-python-uv/**
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
.env
|
||||
docker-compose*
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
- '3.10'
|
||||
- '3.11'
|
||||
- '3.12'
|
||||
- '3.13'
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -589,6 +590,33 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-stackit
|
||||
files: ./stackit_coverage.xml
|
||||
|
||||
# External Provider (dynamic loading)
|
||||
- name: Check if External Provider files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-external
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/common/**
|
||||
./prowler/config/**
|
||||
./prowler/lib/**
|
||||
./tests/providers/external/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run External Provider tests
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/common --cov=./prowler/config --cov=./prowler/lib --cov-report=xml:external_coverage.xml tests/providers/external
|
||||
|
||||
- name: Upload External Provider coverage to Codecov
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-external
|
||||
files: ./external_coverage.xml
|
||||
|
||||
# Lib
|
||||
- name: Check if Lib files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -12,9 +12,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/**'
|
||||
- '.github/workflows/ui-container-checks.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -132,5 +129,5 @@ jobs:
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
fail-on-critical: 'false'
|
||||
fail-on-critical: 'true'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -77,6 +77,11 @@ jobs:
|
||||
E2E_ALIBABACLOUD_ACCESS_KEY_ID: ${{ secrets.E2E_ALIBABACLOUD_ACCESS_KEY_ID }}
|
||||
E2E_ALIBABACLOUD_ACCESS_KEY_SECRET: ${{ secrets.E2E_ALIBABACLOUD_ACCESS_KEY_SECRET }}
|
||||
E2E_ALIBABACLOUD_ROLE_ARN: ${{ secrets.E2E_ALIBABACLOUD_ROLE_ARN }}
|
||||
E2E_OKTA_DOMAIN: ${{ secrets.E2E_OKTA_DOMAIN }}
|
||||
E2E_OKTA_CLIENT_ID: ${{ secrets.E2E_OKTA_CLIENT_ID }}
|
||||
E2E_OKTA_BASE64_PRIVATE_KEY: ${{ secrets.E2E_OKTA_BASE64_PRIVATE_KEY }}
|
||||
E2E_VERCEL_TEAM_ID: ${{ secrets.E2E_VERCEL_TEAM_ID }}
|
||||
E2E_VERCEL_API_TOKEN: ${{ secrets.E2E_VERCEL_API_TOKEN }}
|
||||
# Pass E2E paths from impact analysis
|
||||
E2E_TEST_PATHS: ${{ needs.impact-analysis.outputs.ui-e2e }}
|
||||
RUN_ALL_TESTS: ${{ needs.impact-analysis.outputs.run-all }}
|
||||
@@ -134,7 +139,17 @@ jobs:
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
# which lags behind PR changes; build locally so E2E exercises the API image
|
||||
# produced by this PR.
|
||||
run: docker build -t prowlercloud/prowler-api:latest ./api
|
||||
#
|
||||
# The image installs the SDK from git@master (api/uv.lock), so a PR changing BOTH the SDK
|
||||
# and the API would run against the OLD SDK and crash on startup. Overlay the checkout's
|
||||
# SDK source so both run together. New SDK dependencies still need an api/uv.lock bump.
|
||||
run: |
|
||||
docker build -t prowlercloud/prowler-api:pr-base ./api
|
||||
docker build -t prowlercloud/prowler-api:latest -f - prowler <<'DOCKERFILE'
|
||||
FROM prowlercloud/prowler-api:pr-base
|
||||
RUN rm -rf /home/prowler/.venv/lib/python3.12/site-packages/prowler
|
||||
COPY --chown=prowler:prowler . /home/prowler/.venv/lib/python3.12/site-packages/prowler
|
||||
DOCKERFILE
|
||||
|
||||
- name: Start API services
|
||||
run: |
|
||||
|
||||
@@ -15,12 +15,6 @@ on:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'ui/package.json'
|
||||
- 'ui/pnpm-lock.yaml'
|
||||
- '.github/workflows/ui-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -30,7 +24,6 @@ permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
|
||||
@@ -131,6 +131,10 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run healthcheck
|
||||
|
||||
- name: Check product-tour alignment
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run tour:check
|
||||
|
||||
- name: Run pnpm audit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm run audit
|
||||
|
||||
@@ -7,6 +7,10 @@
|
||||
# P50 — dependency validation
|
||||
|
||||
default_install_hook_types: [pre-commit]
|
||||
# Hooks run on commit only by default;
|
||||
# NOTE: default_stages does NOT override a hook's manifest stages, so fixers shipping pre-push in their
|
||||
# manifest need an explicit stages: ["pre-commit"] below to stay off push.
|
||||
default_stages: [pre-commit]
|
||||
|
||||
repos:
|
||||
## GENERAL (prek built-in — no external repo needed)
|
||||
@@ -21,13 +25,16 @@ repos:
|
||||
- id: check-json
|
||||
priority: 10
|
||||
- id: end-of-file-fixer
|
||||
stages: ["pre-commit"]
|
||||
priority: 0
|
||||
- id: trailing-whitespace
|
||||
stages: ["pre-commit"]
|
||||
priority: 0
|
||||
- id: no-commit-to-branch
|
||||
priority: 10
|
||||
- id: pretty-format-json
|
||||
args: ["--autofix", --no-sort-keys, --no-ensure-ascii]
|
||||
stages: ["pre-commit"]
|
||||
priority: 10
|
||||
|
||||
## TOML
|
||||
@@ -82,6 +89,7 @@ repos:
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
stages: ["pre-commit"]
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# Trivy ignore file for prowlercloud/prowler SDK container image.
|
||||
# Each entry below documents (a) the affected package and why it ships in the
|
||||
# image, (b) why the CVE is not exploitable in Prowler's runtime, and (c) the
|
||||
# upstream fix status. Entries carry an expiry so they auto-force re-review.
|
||||
# Entries are scoped per-package so suppressions cannot drift onto unrelated
|
||||
# packages that may be assigned the same CVE in the future.
|
||||
#
|
||||
# Scanned by: .github/actions/trivy-scan via .github/workflows/sdk-container-checks.yml
|
||||
|
||||
# CVE-2026-42496 — perl-archive-tar path traversal via crafted symlinks.
|
||||
# CVE-2026-8376 — perl heap buffer overflow when compiling regex.
|
||||
# Packages: perl, perl-base, perl-modules-5.36, libperl5.36.
|
||||
# Why ignored: perl-base is part of Debian's "Essential: yes" set; it cannot be
|
||||
# removed without breaking dpkg. The Prowler SDK does not invoke perl at runtime;
|
||||
# neither vulnerable code path (Archive::Tar parsing or regex compilation of
|
||||
# attacker-controlled input) is reachable from Prowler. No Debian bookworm fix
|
||||
# is available yet.
|
||||
CVE-2026-42496 pkg:perl exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-42496 pkg:libperl5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:libperl5.36 exp:2026-07-15
|
||||
|
||||
# CVE-2025-7458 — SQLite integer overflow.
|
||||
# Package: libsqlite3-0.
|
||||
# Why ignored: transitive dependency of CPython's stdlib sqlite3 module. The
|
||||
# Prowler SDK does not open user-supplied SQLite databases; SQLite usage is
|
||||
# internal and bounded. No Debian bookworm fix is available.
|
||||
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
|
||||
|
||||
# CVE-2026-43185 — Linux kernel ksmbd signedness bug.
|
||||
# Package: linux-libc-dev.
|
||||
# Why ignored: linux-libc-dev ships kernel headers for build-time compilation,
|
||||
# not a running kernel. Containers execute against the host kernel, so these
|
||||
# headers are inert at runtime. The upstream fix landed in kernel 7.0-rc2 and
|
||||
# has not been backported to Debian's 6.1 LTS line.
|
||||
CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
|
||||
|
||||
# CVE-2023-45853 — zlib MiniZip integer overflow / heap overflow in
|
||||
# zipOpenNewFileInZip4_64.
|
||||
# Packages: zlib1g, zlib1g-dev.
|
||||
# Why ignored: Debian Security Tracker status for bookworm is <ignored>, with
|
||||
# the published rationale "contrib/minizip not built and src:zlib not producing
|
||||
# binary packages" — i.e. the vulnerable symbol is not present in the libz.so
|
||||
# shipped by Debian. Real-not-affected, not unpatched. Upstream fix is in
|
||||
# zlib 1.3.1, available in Debian trixie (13); migrating the base image would
|
||||
# clear it fully.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2023-45853
|
||||
CVE-2023-45853 pkg:zlib1g exp:2026-07-15
|
||||
CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
|
||||
|
||||
# --- API container image (api/Dockerfile) ---
|
||||
# The entries below are specific to the Prowler API image, which ships
|
||||
# PowerShell and additional build tooling on top of the same bookworm base.
|
||||
|
||||
# CVE-2026-7210 — CPython/Expat hash-flooding denial of service in
|
||||
# `xml.parsers.expat` and `xml.etree.ElementTree`.
|
||||
# Packages: the Debian system Python 3.11 (python3.11*, libpython3.11*).
|
||||
# Why ignored: the API runs under the Python 3.12 interpreter shipped in its
|
||||
# `.venv`; the system `python3.11` is only present because `python3-dev` is
|
||||
# pulled in to compile native extensions (xmlsec, lxml) and is never executed
|
||||
# at runtime. The vulnerable path requires parsing attacker-controlled XML with
|
||||
# the affected interpreter, which Prowler does not do with the system Python.
|
||||
# Full mitigation also needs libexpat >= 2.8.0; no Debian bookworm fix yet.
|
||||
CVE-2026-7210 pkg:python3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
|
||||
|
||||
# CVE-2026-33278 — Unbound DNSSEC validator use-after-free (DoS, possible RCE).
|
||||
# CVE-2026-42960 — Unbound DNS cache poisoning via promiscuous additional records.
|
||||
# Package: libunbound8.
|
||||
# Why ignored: libunbound8 is a transitive apt dependency of the TLS/networking
|
||||
# stack (GnuTLS DANE support); only the shared library ships in the image. Both
|
||||
# vulnerabilities require operating a live Unbound recursive DNSSEC validator
|
||||
# that processes attacker-influenced DNS responses. Prowler never starts an
|
||||
# Unbound resolver, so neither code path is reachable. No Debian bookworm fix yet.
|
||||
CVE-2026-33278 pkg:libunbound8 exp:2026-07-15
|
||||
CVE-2026-42960 pkg:libunbound8 exp:2026-07-15
|
||||
@@ -12,139 +12,141 @@ Use these skills for detailed patterns on-demand:
|
||||
|
||||
### Generic Skills (Any Project)
|
||||
|
||||
| Skill | Description | URL |
|
||||
| ------------ | ----------------------------------------------- | -------------------------------------- |
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) |
|
||||
| `nextjs-16` | App Router, Server Actions, proxy.ts, streaming | [SKILL.md](skills/nextjs-16/SKILL.md) |
|
||||
| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) |
|
||||
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
|
||||
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
|
||||
| `django-drf` | ViewSets, Serializers, Filters | [SKILL.md](skills/django-drf/SKILL.md) |
|
||||
| `jsonapi` | Strict JSON:API v1.1 spec compliance | [SKILL.md](skills/jsonapi/SKILL.md) |
|
||||
| `zod-4` | New API (z.email(), z.uuid()) | [SKILL.md](skills/zod-4/SKILL.md) |
|
||||
| `zustand-5` | Persist, selectors, slices | [SKILL.md](skills/zustand-5/SKILL.md) |
|
||||
| `ai-sdk-5` | UIMessage, streaming, LangChain | [SKILL.md](skills/ai-sdk-5/SKILL.md) |
|
||||
| `vitest` | Unit testing, React Testing Library | [SKILL.md](skills/vitest/SKILL.md) |
|
||||
| `tdd` | Test-Driven Development workflow | [SKILL.md](skills/tdd/SKILL.md) |
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) |
|
||||
| `nextjs-16` | App Router, Server Actions, proxy.ts, streaming | [SKILL.md](skills/nextjs-16/SKILL.md) |
|
||||
| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) |
|
||||
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
|
||||
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
|
||||
| `django-drf` | ViewSets, Serializers, Filters | [SKILL.md](skills/django-drf/SKILL.md) |
|
||||
| `jsonapi` | Strict JSON:API v1.1 spec compliance | [SKILL.md](skills/jsonapi/SKILL.md) |
|
||||
| `zod-4` | New API (z.email(), z.uuid()) | [SKILL.md](skills/zod-4/SKILL.md) |
|
||||
| `zustand-5` | Persist, selectors, slices | [SKILL.md](skills/zustand-5/SKILL.md) |
|
||||
| `ai-sdk-5` | UIMessage, streaming, LangChain | [SKILL.md](skills/ai-sdk-5/SKILL.md) |
|
||||
| `vitest` | Unit testing, React Testing Library | [SKILL.md](skills/vitest/SKILL.md) |
|
||||
| `tdd` | Test-Driven Development workflow | [SKILL.md](skills/tdd/SKILL.md) |
|
||||
|
||||
### Prowler-Specific Skills
|
||||
|
||||
| Skill | Description | URL |
|
||||
| ---------------------------- | ------------------------------------------------------ | ------------------------------------------------------ |
|
||||
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
|
||||
| `prowler-api` | Django + RLS + JSON:API patterns | [SKILL.md](skills/prowler-api/SKILL.md) |
|
||||
| `prowler-ui` | Next.js + shadcn conventions | [SKILL.md](skills/prowler-ui/SKILL.md) |
|
||||
| `prowler-ui-motion` | shadcn/Radix visible microinteraction conventions | [SKILL.md](skills/prowler-ui-motion/SKILL.md) |
|
||||
| `prowler-ui-skeletons` | shadcn skeleton loading and content reveal conventions | [SKILL.md](skills/prowler-ui-skeletons/SKILL.md) |
|
||||
| `prowler-sdk-check` | Create new security checks | [SKILL.md](skills/prowler-sdk-check/SKILL.md) |
|
||||
| `prowler-mcp` | MCP server tools and models | [SKILL.md](skills/prowler-mcp/SKILL.md) |
|
||||
| `prowler-test-sdk` | SDK testing (pytest + moto) | [SKILL.md](skills/prowler-test-sdk/SKILL.md) |
|
||||
| `prowler-test-api` | API testing (pytest-django + RLS) | [SKILL.md](skills/prowler-test-api/SKILL.md) |
|
||||
| `prowler-test-ui` | E2E testing (Playwright) | [SKILL.md](skills/prowler-test-ui/SKILL.md) |
|
||||
| `prowler-compliance` | Compliance framework structure | [SKILL.md](skills/prowler-compliance/SKILL.md) |
|
||||
| `prowler-compliance-review` | Review compliance framework PRs | [SKILL.md](skills/prowler-compliance-review/SKILL.md) |
|
||||
| `prowler-provider` | Add new cloud providers | [SKILL.md](skills/prowler-provider/SKILL.md) |
|
||||
| `prowler-changelog` | Changelog entries (keepachangelog.com) | [SKILL.md](skills/prowler-changelog/SKILL.md) |
|
||||
| `prowler-ci` | CI checks and PR gates (GitHub Actions) | [SKILL.md](skills/prowler-ci/SKILL.md) |
|
||||
| `prowler-commit` | Professional commits (conventional-commits) | [SKILL.md](skills/prowler-commit/SKILL.md) |
|
||||
| `prowler-pr` | Pull request conventions | [SKILL.md](skills/prowler-pr/SKILL.md) |
|
||||
| `prowler-docs` | Documentation style guide | [SKILL.md](skills/prowler-docs/SKILL.md) |
|
||||
| `django-migration-psql` | Django migration best practices for PostgreSQL | [SKILL.md](skills/django-migration-psql/SKILL.md) |
|
||||
| `postgresql-indexing` | PostgreSQL indexing, EXPLAIN, monitoring, maintenance | [SKILL.md](skills/postgresql-indexing/SKILL.md) |
|
||||
| `prowler-attack-paths-query` | Create Attack Paths openCypher queries | [SKILL.md](skills/prowler-attack-paths-query/SKILL.md) |
|
||||
| `gh-aw` | GitHub Agentic Workflows (gh-aw) | [SKILL.md](skills/gh-aw/SKILL.md) |
|
||||
| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) |
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
|
||||
| `prowler-api` | Django + RLS + JSON:API patterns | [SKILL.md](skills/prowler-api/SKILL.md) |
|
||||
| `prowler-ui` | Next.js + shadcn conventions | [SKILL.md](skills/prowler-ui/SKILL.md) |
|
||||
| `prowler-sdk-check` | Create new security checks | [SKILL.md](skills/prowler-sdk-check/SKILL.md) |
|
||||
| `prowler-mcp` | MCP server tools and models | [SKILL.md](skills/prowler-mcp/SKILL.md) |
|
||||
| `prowler-test-sdk` | SDK testing (pytest + moto) | [SKILL.md](skills/prowler-test-sdk/SKILL.md) |
|
||||
| `prowler-test-api` | API testing (pytest-django + RLS) | [SKILL.md](skills/prowler-test-api/SKILL.md) |
|
||||
| `prowler-test-ui` | E2E testing (Playwright) | [SKILL.md](skills/prowler-test-ui/SKILL.md) |
|
||||
| `prowler-compliance` | Compliance framework structure | [SKILL.md](skills/prowler-compliance/SKILL.md) |
|
||||
| `prowler-compliance-review` | Review compliance framework PRs | [SKILL.md](skills/prowler-compliance-review/SKILL.md) |
|
||||
| `prowler-provider` | Add new cloud providers | [SKILL.md](skills/prowler-provider/SKILL.md) |
|
||||
| `prowler-changelog` | Changelog entries (keepachangelog.com) | [SKILL.md](skills/prowler-changelog/SKILL.md) |
|
||||
| `prowler-ci` | CI checks and PR gates (GitHub Actions) | [SKILL.md](skills/prowler-ci/SKILL.md) |
|
||||
| `prowler-commit` | Professional commits (conventional-commits) | [SKILL.md](skills/prowler-commit/SKILL.md) |
|
||||
| `prowler-pr` | Pull request conventions | [SKILL.md](skills/prowler-pr/SKILL.md) |
|
||||
| `prowler-docs` | Documentation style guide | [SKILL.md](skills/prowler-docs/SKILL.md) |
|
||||
| `django-migration-psql` | Django migration best practices for PostgreSQL | [SKILL.md](skills/django-migration-psql/SKILL.md) |
|
||||
| `postgresql-indexing` | PostgreSQL indexing, EXPLAIN, monitoring, maintenance | [SKILL.md](skills/postgresql-indexing/SKILL.md) |
|
||||
| `prowler-attack-paths-query` | Create Attack Paths openCypher queries | [SKILL.md](skills/prowler-attack-paths-query/SKILL.md) |
|
||||
| `prowler-tour` | Keep product-tour definitions aligned with the UI | [SKILL.md](skills/prowler-tour/SKILL.md) |
|
||||
| `gh-aw` | GitHub Agentic Workflows (gh-aw) | [SKILL.md](skills/gh-aw/SKILL.md) |
|
||||
| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) |
|
||||
|
||||
### Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
| Action | Skill |
|
||||
| --------------------------------------------------------------------------------------------------------------------- | ---------------------------- |
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding a compliance output formatter (per-provider class + table dispatcher) | `prowler-compliance` |
|
||||
| Adding indexes or constraints to database tables | `django-migration-psql` |
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-16` |
|
||||
| Auditing check-to-requirement mappings as a cloud auditor | `prowler-compliance` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Configuring MCP servers in agentic workflows | `gh-aw` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
| Create a PR with gh pr create | `prowler-pr` |
|
||||
| Creating API endpoints | `jsonapi` |
|
||||
| Creating Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Creating GitHub Agentic Workflows | `gh-aw` |
|
||||
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
|
||||
| Creating Zod schemas | `zod-4` |
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating new checks | `prowler-sdk-check` |
|
||||
| Creating new skills | `skill-creator` |
|
||||
| Creating or reviewing Django migrations | `django-migration-psql` |
|
||||
| Creating/modifying Prowler UI components | `prowler-ui` |
|
||||
| Creating/modifying UI motion, transitions, or microinteractions | `prowler-ui-motion` |
|
||||
| Creating/modifying skeletons, loading states, or Suspense fallbacks | `prowler-ui-skeletons` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
| Debug why a GitHub Actions job is failing | `prowler-ci` |
|
||||
| Debugging gh-aw compilation errors | `gh-aw` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| Fixing bug | `tdd` |
|
||||
| Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs) | `prowler-compliance` |
|
||||
| General Prowler development questions | `prowler` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Implementing feature | `tdd` |
|
||||
| Importing Copilot Custom Agents into workflows | `gh-aw` |
|
||||
| Inspect PR CI checks and gates (.github/workflows/\*) | `prowler-ci` |
|
||||
| Inspect PR CI workflows (.github/workflows/\*): conventional-commit, pr-check-changelog, pr-conflict-checker, labeler | `prowler-pr` |
|
||||
| Mapping checks to compliance controls | `prowler-compliance` |
|
||||
| Mocking AWS with moto in tests | `prowler-test-sdk` |
|
||||
| Modifying API responses | `jsonapi` |
|
||||
| Modifying component | `tdd` |
|
||||
| Modifying gh-aw workflow frontmatter or safe-outputs | `gh-aw` |
|
||||
| Refactoring code | `tdd` |
|
||||
| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` |
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Testing hooks or utilities | `vitest` |
|
||||
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
|
||||
| Understand CODEOWNERS/labeler-based automation | `prowler-ci` |
|
||||
| Understand PR title conventional-commit validation | `prowler-ci` |
|
||||
| Understand changelog gate and no-changelog label behavior | `prowler-ci` |
|
||||
| Understand review ownership with CODEOWNERS | `prowler-pr` |
|
||||
| Update CHANGELOG.md in any component | `prowler-changelog` |
|
||||
| Updating README.md provider statistics table | `prowler-readme-table` |
|
||||
| Updating checks, services, compliance, or categories count in README.md | `prowler-readme-table` |
|
||||
| Updating existing Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Updating existing checks and metadata | `prowler-sdk-check` |
|
||||
| Using Zustand stores | `zustand-5` |
|
||||
| Working on MCP server tools | `prowler-mcp` |
|
||||
| Working on Prowler UI structure (actions/adapters/types/hooks) | `prowler-ui` |
|
||||
| Working on task | `tdd` |
|
||||
| Working with Prowler UI test helpers/pages | `prowler-test-ui` |
|
||||
| Working with Tailwind classes | `tailwind-4` |
|
||||
| Writing Playwright E2E tests | `playwright` |
|
||||
| Writing Prowler API tests | `prowler-test-api` |
|
||||
| Writing Prowler SDK tests | `prowler-test-sdk` |
|
||||
| Writing Prowler UI E2E tests | `prowler-test-ui` |
|
||||
| Writing Python tests with pytest | `pytest` |
|
||||
| Writing React component tests | `vitest` |
|
||||
| Writing React components | `react-19` |
|
||||
| Writing TypeScript types/interfaces | `typescript` |
|
||||
| Writing Vitest tests | `vitest` |
|
||||
| Writing data backfill or data migration | `django-migration-psql` |
|
||||
| Writing documentation | `prowler-docs` |
|
||||
| Writing unit tests for UI | `vitest` |
|
||||
| Action | Skill |
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding a compliance output formatter (per-provider class + table dispatcher) | `prowler-compliance` |
|
||||
| Adding indexes or constraints to database tables | `django-migration-psql` |
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding privilege escalation detection queries | `prowler-attack-paths-query` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| Adding, updating, or removing a tour definition (*.tour.ts) | `prowler-tour` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-16` |
|
||||
| Auditing check-to-requirement mappings as a cloud auditor | `prowler-compliance` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Changing button labels or section headings on a tour-covered page | `prowler-tour` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Configuring MCP servers in agentic workflows | `gh-aw` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
| Create a PR with gh pr create | `prowler-pr` |
|
||||
| Creating API endpoints | `jsonapi` |
|
||||
| Creating Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Creating GitHub Agentic Workflows | `gh-aw` |
|
||||
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
|
||||
| Creating Zod schemas | `zod-4` |
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating new checks | `prowler-sdk-check` |
|
||||
| Creating new skills | `skill-creator` |
|
||||
| Creating or reviewing Django migrations | `django-migration-psql` |
|
||||
| Creating/modifying Prowler UI components | `prowler-ui` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
| Debug why a GitHub Actions job is failing | `prowler-ci` |
|
||||
| Debugging gh-aw compilation errors | `gh-aw` |
|
||||
| Editing a UI file containing data-tour-id attributes | `prowler-tour` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| Fixing bug | `tdd` |
|
||||
| Fixing compliance JSON bugs (duplicate IDs, empty Section, stale refs) | `prowler-compliance` |
|
||||
| General Prowler development questions | `prowler` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Implementing feature | `tdd` |
|
||||
| Importing Copilot Custom Agents into workflows | `gh-aw` |
|
||||
| Inspect PR CI checks and gates (.github/workflows/*) | `prowler-ci` |
|
||||
| Inspect PR CI workflows (.github/workflows/*): conventional-commit, pr-check-changelog, pr-conflict-checker, labeler | `prowler-pr` |
|
||||
| Mapping checks to compliance controls | `prowler-compliance` |
|
||||
| Mocking AWS with moto in tests | `prowler-test-sdk` |
|
||||
| Modifying API responses | `jsonapi` |
|
||||
| Modifying component | `tdd` |
|
||||
| Modifying gh-aw workflow frontmatter or safe-outputs | `gh-aw` |
|
||||
| Refactoring code | `tdd` |
|
||||
| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` |
|
||||
| Renaming or removing a data-tour-id attribute value | `prowler-tour` |
|
||||
| Restructuring routes or layouts covered by a tour | `prowler-tour` |
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Testing hooks or utilities | `vitest` |
|
||||
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
|
||||
| Understand CODEOWNERS/labeler-based automation | `prowler-ci` |
|
||||
| Understand PR title conventional-commit validation | `prowler-ci` |
|
||||
| Understand changelog gate and no-changelog label behavior | `prowler-ci` |
|
||||
| Understand review ownership with CODEOWNERS | `prowler-pr` |
|
||||
| Update CHANGELOG.md in any component | `prowler-changelog` |
|
||||
| Updating README.md provider statistics table | `prowler-readme-table` |
|
||||
| Updating checks, services, compliance, or categories count in README.md | `prowler-readme-table` |
|
||||
| Updating existing Attack Paths queries | `prowler-attack-paths-query` |
|
||||
| Updating existing checks and metadata | `prowler-sdk-check` |
|
||||
| Using Zustand stores | `zustand-5` |
|
||||
| Working on MCP server tools | `prowler-mcp` |
|
||||
| Working on Prowler UI structure (actions/adapters/types/hooks) | `prowler-ui` |
|
||||
| Working on task | `tdd` |
|
||||
| Working with Prowler UI test helpers/pages | `prowler-test-ui` |
|
||||
| Working with Tailwind classes | `tailwind-4` |
|
||||
| Writing Playwright E2E tests | `playwright` |
|
||||
| Writing Prowler API tests | `prowler-test-api` |
|
||||
| Writing Prowler SDK tests | `prowler-test-sdk` |
|
||||
| Writing Prowler UI E2E tests | `prowler-test-ui` |
|
||||
| Writing Python tests with pytest | `pytest` |
|
||||
| Writing React component tests | `vitest` |
|
||||
| Writing React components | `react-19` |
|
||||
| Writing TypeScript types/interfaces | `typescript` |
|
||||
| Writing Vitest tests | `vitest` |
|
||||
| Writing data backfill or data migration | `django-migration-psql` |
|
||||
| Writing documentation | `prowler-docs` |
|
||||
| Writing unit tests for UI | `vitest` |
|
||||
|
||||
---
|
||||
|
||||
@@ -152,13 +154,13 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
Prowler is an open-source cloud security assessment tool supporting AWS, Azure, GCP, Kubernetes, GitHub, M365, and more.
|
||||
|
||||
| Component | Location | Tech Stack |
|
||||
| ---------- | ------------- | -------------------------------- |
|
||||
| SDK | `prowler/` | Python 3.10+, uv |
|
||||
| API | `api/` | Django 5.1, DRF, Celery |
|
||||
| UI | `ui/` | Next.js 16, React 19, Tailwind 4 |
|
||||
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
|
||||
| Dashboard | `dashboard/` | Dash, Plotly |
|
||||
| Component | Location | Tech Stack |
|
||||
|-----------|----------|------------|
|
||||
| SDK | `prowler/` | Python 3.10+, uv |
|
||||
| API | `api/` | Django 5.1, DRF, Celery |
|
||||
| UI | `ui/` | Next.js 16, React 19, Tailwind 4 |
|
||||
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
|
||||
| Dashboard | `dashboard/` | Dash, Plotly |
|
||||
|
||||
---
|
||||
|
||||
@@ -184,7 +186,6 @@ Follow conventional-commit style: `<type>[scope]: <description>`
|
||||
**Types:** `feat`, `fix`, `docs`, `chore`, `perf`, `refactor`, `style`, `test`
|
||||
|
||||
Before creating a PR:
|
||||
|
||||
1. Complete checklist in `.github/pull_request_template.md`
|
||||
2. Run all relevant tests and linters
|
||||
3. Link screenshots for UI changes
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/prowler"
|
||||
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
@@ -6,7 +6,7 @@ LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ARG TRIVY_VERSION=0.71.0
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
|
||||
@@ -122,7 +122,7 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
| Vercel | 26 | 6 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 4 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 7 | 2 | 0 | 3 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
@@ -153,6 +153,8 @@ Prowler App offers flexible installation methods tailored to various environment
|
||||
|
||||
#### Commands
|
||||
|
||||
_macOS/Linux:_
|
||||
|
||||
``` console
|
||||
VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name)
|
||||
curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/docker-compose.yml"
|
||||
@@ -161,6 +163,16 @@ curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${V
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
_Windows PowerShell:_
|
||||
|
||||
``` powershell
|
||||
$VERSION = (Invoke-RestMethod -Uri "https://api.github.com/repos/prowler-cloud/prowler/releases/latest").tag_name
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/docker-compose.yml" -OutFile "docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/.env" -OutFile ".env"
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
|
||||
|
||||
+79
-1
@@ -2,11 +2,89 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.30.0] (Prowler UNRELEASED)
|
||||
## [1.32.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Provider group filters for API endpoints that support cloud provider filtering, including exact and `__in` variants [(#11573)](https://github.com/prowler-cloud/prowler/pull/11573)
|
||||
- Provider filters for `GET /api/v1/compliance-overviews`, `/metadata`, and `/requirements`, using latest completed scans per matching provider [(#11587)](https://github.com/prowler-cloud/prowler/pull/11587)
|
||||
- Server-Sent Events (SSE) infrastructure for the API: a base viewset, a tenant-aware channel manager, and channel-name helpers backed by `django-eventstream` over Valkey Pub/Sub and served through the Gunicorn ASGI worker, so feature endpoints can stream events to clients over a single long-lived connection [(#11556)](https://github.com/prowler-cloud/prowler/pull/11556)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `aiohttp` to 3.14.0 and `idna` to 3.15, patching known CVEs [(#11596)](https://github.com/prowler-cloud/prowler/pull/11596)
|
||||
- Container base image to `python:3.12.13-slim-bookworm` and `trivy` to 0.71.0, patching OS and Go module CVEs [(#11596)](https://github.com/prowler-cloud/prowler/pull/11596)
|
||||
- `trivy` binary bumped to 0.71.0 patching embedded `golang.org/x/crypto`, `golang.org/x/net`, and Go `stdlib` CVEs [(#11592)](https://github.com/prowler-cloud/prowler/pull/11592)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.2] (Prowler v5.30.2)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- `scan-compliance-overviews` task now streams the findings aggregation and the requirement-row writes so it runs faster and its peak memory no longer grows with the number of regions and frameworks [(#11591)](https://github.com/prowler-cloud/prowler/pull/11591)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.1] (Prowler v5.30.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `compliance-overviews/attributes` now resolves the provider from the scan, so multi-provider universal frameworks (e.g. CSA CCM) return the check IDs of the scan's provider and Azure/GCP requirement details show their findings instead of appearing empty [(#11546)](https://github.com/prowler-cloud/prowler/pull/11546)
|
||||
- Attack Paths: `drop_subgraph` now deletes relationships first and then nodes in batches, using less memory on Neo4j when clearing a dense provider graph [(#11557)](https://github.com/prowler-cloud/prowler/pull/11557)
|
||||
- OCI scans now use API key credentials with the configured region instead of falling back to `/home/prowler/.oci/config` [(#11558)](https://github.com/prowler-cloud/prowler/pull/11558)
|
||||
|
||||
---
|
||||
|
||||
## [1.31.0] (Prowler v5.30.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Opt-in automatic recovery of allowlisted idempotent background tasks whose worker died during a deploy or crash: when enabled via `DJANGO_TASK_RECOVERY_ENABLED` (off by default), stuck summary and deletion tasks are detected and re-run instead of staying pending forever (scan and Jira tasks are excluded), with a `reconcile_orphan_tasks` management command for on-demand recovery [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- DORA compliance framework support [(#11131)](https://github.com/prowler-cloud/prowler/pull/11131)
|
||||
- Label Postgres connections with `application_name="<component>:<alias>"` (component injected per process via `DJANGO_APP_COMPONENT`) so connections are attributable by component in `pg_stat_activity` [(#11494)](https://github.com/prowler-cloud/prowler/pull/11494)
|
||||
- DISA Okta IDaaS STIG V1R2 compliance framework export support for the Okta provider [(#11428)](https://github.com/prowler-cloud/prowler/pull/11428)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Allowlisted idempotent background tasks are no longer lost when a worker is stopped or crashes mid-task; tasks with external side effects are marked terminal instead of blindly re-running [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- SAML logins no longer wipe a user's roles when the IdP does not send the `userType` attribute; existing roles are kept, and when `userType` names a role that does not exist it is now created with read-only access (visibility over all providers, no management permissions) instead of no permissions at all [(#11520)](https://github.com/prowler-cloud/prowler/pull/11520)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Workers now shut down gracefully on deploy or restart, finishing or re-queueing in-flight tasks instead of being force-killed and leaving them stuck [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- Resource `name` is now stored and refreshed on every scan, so resources no longer keep an empty name [(#11476)](https://github.com/prowler-cloud/prowler/pull/11476)
|
||||
- Compliance catalog now warms in background during startup. `compliance-overviews/attributes` returns `503` while warming, so the first request after a deploy no longer trips the API timeout [(#11530)](https://github.com/prowler-cloud/prowler/pull/11530)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `dulwich` from 0.23.0 to 1.2.5 and `pyjwt` from 2.12.1 to 2.13.0, patching `GHSA-897w-fcg9-f6xj` (arbitrary file write) and `PYSEC-2026-179` (HMAC/JWK key confusion) [(#11499)](https://github.com/prowler-cloud/prowler/pull/11499)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.3] (Prowler v5.29.3)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- API startup no longer crashes when Neo4j is unreachable, as the Neo4j driver now connects lazily on first use rather than during app initialization [(#11491)](https://github.com/prowler-cloud/prowler/pull/11491)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.1] (Prowler v5.29.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `GET /api/v1/findings` N+1 query loading `resources__tags` when listing findings [(#11420)](https://github.com/prowler-cloud/prowler/pull/11420)
|
||||
- Clean up the scan tmp output directory when `scan-report` fails so partial files do not accumulate and fill the worker disk (`No space left on device`) [(#11421)](https://github.com/prowler-cloud/prowler/pull/11421)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.0] (Prowler v5.29.0)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Scan finding ingestion: bulk-resolve `Resource`/`ResourceTag` rows, replace per-mapping `SELECT FOR UPDATE` with deferred `ResourceTagMapping.bulk_create(ignore_conflicts=True)`, wrap each micro-batch in a single `rls_transaction`, and raise `SCAN_DB_BATCH_SIZE` to 1000 [(#11249)](https://github.com/prowler-cloud/prowler/pull/11249)
|
||||
- Faster `GET /api/v1/finding-groups/latest` aggregation on tenants where one recent scan holds most findings [(#11380)](https://github.com/prowler-cloud/prowler/pull/11380)
|
||||
|
||||
---
|
||||
|
||||
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
|
||||
ARG TRIVY_VERSION=0.70.0
|
||||
ARG TRIVY_VERSION=0.71.0
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
|
||||
@@ -21,13 +21,19 @@ apply_fixtures() {
|
||||
}
|
||||
|
||||
start_dev_server() {
|
||||
echo "Starting the development server..."
|
||||
uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
echo "Starting the development server (Gunicorn ASGI, debug + reload)..."
|
||||
# Same server/worker as prod (config.asgi via the native `asgi` worker), so
|
||||
# SSE streams run on the event loop exactly as they do in production. DEBUG is
|
||||
# on so guniconf's `reload = DEBUG` hot-reloads edited code (and flips
|
||||
# `preload_app` off so reload actually takes).
|
||||
export DJANGO_DEBUG="${DJANGO_DEBUG:-True}"
|
||||
export DJANGO_BIND_ADDRESS="${DJANGO_BIND_ADDRESS:-0.0.0.0}"
|
||||
exec uv run gunicorn -c config/guniconf.py config.asgi:application
|
||||
}
|
||||
|
||||
start_prod_server() {
|
||||
echo "Starting the Gunicorn server..."
|
||||
uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
exec uv run gunicorn -c config/guniconf.py config.asgi:application
|
||||
}
|
||||
|
||||
resolve_worker_hostname() {
|
||||
@@ -47,7 +53,7 @@ resolve_worker_hostname() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
uv run python -m celery -A config.celery worker \
|
||||
exec uv run python -m celery -A config.celery worker \
|
||||
-n "$(resolve_worker_hostname)" \
|
||||
-l "${DJANGO_LOGGING_LEVEL:-info}" \
|
||||
-Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \
|
||||
@@ -56,7 +62,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
exec uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
manage_db_partitions() {
|
||||
@@ -68,6 +74,15 @@ manage_db_partitions() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Identify this process to Postgres (application_name=<component>:<alias>) so
|
||||
# connections are attributable by component in pg_stat_activity. Web tiers
|
||||
# report "api"; everything else uses the launch subcommand.
|
||||
case "$1" in
|
||||
prod|dev) DJANGO_APP_COMPONENT="api" ;;
|
||||
*) DJANGO_APP_COMPONENT="$1" ;;
|
||||
esac
|
||||
export DJANGO_APP_COMPONENT
|
||||
|
||||
case "$1" in
|
||||
dev)
|
||||
apply_migrations
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# Orphan Celery task recovery
|
||||
|
||||
When a worker is terminated mid-task (a deploy, an OOM kill, a node eviction), the
|
||||
task it was running can be left non-terminal forever: the `TaskResult` stays
|
||||
`STARTED` and nothing re-runs it. This page describes the mechanisms that detect and
|
||||
recover allowlisted idempotent orphans so pending-task alerts do not fire. Scan tasks
|
||||
are not auto-recovered (re-running a scan is not safe to do automatically); the
|
||||
watchdog covers the summary/aggregation and deletion tasks.
|
||||
|
||||
## How recovery works
|
||||
|
||||
1. **Durable delivery.** The broker is configured so a task message is acknowledged
|
||||
only after the task finishes (`task_acks_late`), one task is reserved at a time
|
||||
(`worker_prefetch_multiplier = 1`), and an abruptly-lost worker re-queues its task
|
||||
(`task_reject_on_worker_lost`). On `SIGTERM` the worker is given a soft-shutdown
|
||||
window (`worker_soft_shutdown_timeout`) to finish or re-queue in-flight work
|
||||
before it is force-killed. `scan-perform`, `scan-perform-scheduled` and
|
||||
`integration-jira` opt out of redelivery with `acks_late=False`, so a crash drops
|
||||
them rather than re-running and duplicating findings or Jira issues. Other
|
||||
non-recovered side-effect tasks keep `acks_late=True`, so the broker can still
|
||||
re-deliver them after a worker loss: the S3 upload rebuilds from worker-local files
|
||||
that did not survive the crash and so no-ops, but Security Hub re-reads findings from
|
||||
the DB and re-sends them to AWS.
|
||||
|
||||
2. **Periodic watchdog.** A Beat task, `reconcile-orphan-tasks`, runs every couple of
|
||||
minutes (a `django_celery_beat` periodic task created by migration). For each
|
||||
in-flight task result with an allowlisted idempotent task name, it pings the
|
||||
worker recorded on the task's `TaskResult`:
|
||||
- worker responds -> the task is still running, leave it alone;
|
||||
- worker is gone (and the task started before a short grace window) -> it is a
|
||||
real orphan: the stale task is revoked and marked terminal (clearing the
|
||||
pending/started alert), and the task is re-enqueued from its stored name and
|
||||
kwargs.
|
||||
|
||||
The re-run is safe because only tasks with proven idempotency are allowlisted: the
|
||||
summary/aggregation tasks clear and re-write their own rows, and deletions are
|
||||
idempotent. Scan tasks and external side effects are excluded: re-running a scan is
|
||||
not safe to do automatically, Jira sends would create duplicate issues, the S3
|
||||
upload rebuilds from worker-local files that do not survive a crash, and
|
||||
report/Security Hub recovery is out of scope.
|
||||
|
||||
3. **Recovery cap.** A per-task Valkey counter limits how often the same task is
|
||||
re-enqueued. After `--max-attempts` recoveries (default 3) the orphan is marked
|
||||
terminal instead of re-enqueued, so a task that repeatedly kills its worker cannot
|
||||
loop forever.
|
||||
|
||||
A Postgres advisory lock ensures that, even with multiple API/worker replicas, only
|
||||
one reconciliation runs at a time; the others no-op.
|
||||
|
||||
## On-demand command
|
||||
|
||||
The same logic is available as a management command, useful right after a deploy or
|
||||
for manual intervention:
|
||||
|
||||
```bash
|
||||
python manage.py reconcile_orphan_tasks # recover now
|
||||
python manage.py reconcile_orphan_tasks --dry-run # report orphans, change nothing
|
||||
python manage.py reconcile_orphan_tasks --grace-minutes 5 --max-attempts 3
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings have safe defaults; override via environment variables.
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER` | `1` | Tasks reserved per worker process. |
|
||||
| `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT` | `60` | Seconds the worker drains/re-queues on `SIGTERM` before force-kill. |
|
||||
| `DJANGO_CELERY_TASK_TIME_LIMIT` | `21600` (6h) | Hard limit for most tasks; connection checks are capped at 120s. |
|
||||
| `DJANGO_CELERY_TASK_SOFT_TIME_LIMIT` | hard - 600 | Soft limit; raises `SoftTimeLimitExceeded` for cleanup. |
|
||||
| `DJANGO_CELERY_LONG_TASK_TIME_LIMIT` | `172800` (48h) | Hard limit for scans and provider/tenant deletions, which can legitimately run for more than a day. |
|
||||
| `DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT` | long hard - 600 | Soft limit for the long-running tasks above. |
|
||||
| `DJANGO_TASK_RECOVERY_ENABLED` | `false` | Master switch for orphan-task recovery, disabled by default (opt-in); set to `true` to enable. When off, no orphan is detected, marked terminal, or re-enqueued (attack-paths stale cleanup still runs). |
|
||||
| `DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED` | `true` | Auto re-enqueue orphaned scan summary/aggregation tasks. |
|
||||
| `DJANGO_TASK_RECOVERY_DELETIONS_ENABLED` | `true` | Auto re-enqueue orphaned provider/tenant deletion tasks. |
|
||||
|
||||
Recovery is opt-in: with the master flag off (the default) the sweep does nothing.
|
||||
Once enabled, the per-group flags default to on, so every group recovers unless you
|
||||
turn one off; a task whose group flag is off is marked terminal instead of
|
||||
re-enqueued.
|
||||
|
||||
Turning recovery off only disables this watchdog sweep; it does not change Celery's
|
||||
broker-level redelivery (`task_acks_late`/`task_reject_on_worker_lost`), which still
|
||||
re-delivers tasks that keep `acks_late=True` on worker loss, independently of this flag.
|
||||
|
||||
`task_acks_late` and `task_reject_on_worker_lost` are enabled in `config/celery.py`.
|
||||
|
||||
## Deployment requirement
|
||||
|
||||
Two conditions must both hold for the soft shutdown to actually drain work:
|
||||
|
||||
1. **The worker must receive `SIGTERM`.** The container entrypoint `exec`s the
|
||||
Celery process so it runs as PID 1; otherwise `SIGTERM` from `docker stop`/ECS
|
||||
hits the entrypoint shell, never reaches Celery, and the worker is hard-killed
|
||||
(SIGKILL) at the grace deadline without draining. Custom entrypoints must
|
||||
preserve the `exec`.
|
||||
2. **The orchestrator must give the worker enough time** before force-killing it.
|
||||
Set the stop grace period to exceed `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT`
|
||||
plus a margin:
|
||||
- **docker-compose:** `stop_grace_period` on the worker services (set to `120s`).
|
||||
- **AWS ECS:** the worker container `stopTimeout` (configured in the deployment
|
||||
repository).
|
||||
|
||||
If either condition is missing, long tasks are still recovered by the watchdog,
|
||||
but they are cut mid-run on every deploy instead of draining.
|
||||
+24
-13
@@ -41,7 +41,8 @@ dependencies = [
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"defusedxml==0.7.1",
|
||||
"gunicorn==23.0.0",
|
||||
"django-eventstream==5.3.3",
|
||||
"gunicorn==26.0.0",
|
||||
"lxml==6.1.0",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@master",
|
||||
"psycopg2-binary==2.9.9",
|
||||
@@ -68,7 +69,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.30.0"
|
||||
version = "1.32.0"
|
||||
|
||||
[tool.uv]
|
||||
# Transitive pins matching master to avoid silent drift; bump deliberately.
|
||||
@@ -79,7 +80,7 @@ constraint-dependencies = [
|
||||
"aiobotocore==2.25.1",
|
||||
"aiofiles==24.1.0",
|
||||
"aiohappyeyeballs==2.6.1",
|
||||
"aiohttp==3.13.5",
|
||||
"aiohttp==3.14.0",
|
||||
"aioitertools==0.13.0",
|
||||
"aiosignal==1.4.0",
|
||||
"alibabacloud-actiontrail20200706==2.4.1",
|
||||
@@ -124,9 +125,8 @@ constraint-dependencies = [
|
||||
"astroid==3.2.4",
|
||||
"async-timeout==5.0.1",
|
||||
"attrs==25.4.0",
|
||||
"authlib==1.6.9",
|
||||
"authlib==1.6.12",
|
||||
"autopep8==2.3.2",
|
||||
"awsipranges==0.3.3",
|
||||
"azure-cli-core==2.83.0",
|
||||
"azure-cli-telemetry==1.1.0",
|
||||
"azure-common==1.1.28",
|
||||
@@ -209,6 +209,7 @@ constraint-dependencies = [
|
||||
"django-celery-results==2.6.0",
|
||||
"django-cors-headers==4.4.0",
|
||||
"django-environ==0.11.2",
|
||||
"django-eventstream==5.3.3",
|
||||
"django-filter==24.3",
|
||||
"django-guid==3.5.0",
|
||||
"django-postgres-extra==2.0.9",
|
||||
@@ -226,7 +227,7 @@ constraint-dependencies = [
|
||||
"drf-simple-apikey==2.2.1",
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"dulwich==0.23.0",
|
||||
"dulwich==1.2.5",
|
||||
"duo-client==5.5.0",
|
||||
"durationpy==0.10",
|
||||
"email-validator==2.2.0",
|
||||
@@ -253,7 +254,7 @@ constraint-dependencies = [
|
||||
"grpc-google-iam-v1==0.14.3",
|
||||
"grpcio==1.76.0",
|
||||
"grpcio-status==1.76.0",
|
||||
"gunicorn==23.0.0",
|
||||
"gunicorn==26.0.0",
|
||||
"h11==0.16.0",
|
||||
"h2==4.3.0",
|
||||
"hpack==4.1.0",
|
||||
@@ -262,8 +263,8 @@ constraint-dependencies = [
|
||||
"httpx==0.28.1",
|
||||
"humanfriendly==10.0",
|
||||
"hyperframe==6.1.0",
|
||||
"iamdata==0.1.202602021",
|
||||
"idna==3.11",
|
||||
"iamdata==0.1.202605131",
|
||||
"idna==3.15",
|
||||
"importlib-metadata==8.7.1",
|
||||
"inflection==0.5.1",
|
||||
"iniconfig==2.3.0",
|
||||
@@ -315,7 +316,7 @@ constraint-dependencies = [
|
||||
"neo4j==6.1.0",
|
||||
"nest-asyncio==1.6.0",
|
||||
"nltk==3.9.4",
|
||||
"numpy==2.0.2",
|
||||
"numpy==2.2.6",
|
||||
"oauthlib==3.3.1",
|
||||
"oci==2.169.0",
|
||||
"openai==1.109.1",
|
||||
@@ -344,7 +345,7 @@ constraint-dependencies = [
|
||||
"psutil==7.2.2",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"py-deviceid==0.1.1",
|
||||
"py-iam-expand==0.1.0",
|
||||
"py-iam-expand==0.3.0",
|
||||
"py-ocsf-models==0.8.1",
|
||||
"pyasn1==0.6.3",
|
||||
"pyasn1-modules==0.4.2",
|
||||
@@ -354,7 +355,7 @@ constraint-dependencies = [
|
||||
"pydantic-core==2.41.5",
|
||||
"pygithub==2.8.0",
|
||||
"pygments==2.20.0",
|
||||
"pyjwt==2.12.1",
|
||||
"pyjwt==2.13.0",
|
||||
"pylint==3.2.5",
|
||||
"pymsalruntime==0.18.1",
|
||||
"pynacl==1.6.2",
|
||||
@@ -443,7 +444,17 @@ constraint-dependencies = [
|
||||
# The microsoft-kiota-http security bump to 1.9.9 (GHSA-7j59-v9qr-6fq9) requires
|
||||
# microsoft-kiota-abstractions>=1.9.9, which a constraint cannot satisfy against the
|
||||
# SDK's hard pin; override it to the patched, kiota-aligned version.
|
||||
#
|
||||
# prowler@master hard-pins dulwich==0.23.0 and pyjwt==2.12.1 in [project.dependencies].
|
||||
# dulwich 1.2.5 patches GHSA-897w-fcg9-f6xj (arbitrary file write) and pyjwt 2.13.0
|
||||
# patches PYSEC-2026-179 (HMAC/JWK key-confusion); a constraint cannot satisfy these
|
||||
# against the SDK's hard pins, so override them to the patched versions until the SDK
|
||||
# bump propagates to the pinned master rev. pyjwt keeps the [crypto] extra because an
|
||||
# override replaces the whole requirement; bare pyjwt would drop it from the consumers
|
||||
# that request pyjwt[crypto] and leave cryptography (needed for RS256) only transitive.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2",
|
||||
"microsoft-kiota-abstractions==1.9.9"
|
||||
"microsoft-kiota-abstractions==1.9.9",
|
||||
"dulwich==1.2.5",
|
||||
"pyjwt[crypto]==2.13.0"
|
||||
]
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
@@ -30,7 +32,6 @@ class ApiConfig(AppConfig):
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.attack_paths import database as graph_database
|
||||
|
||||
# Generate required cryptographic keys if not present, but only if:
|
||||
# `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
|
||||
@@ -41,37 +42,8 @@ class ApiConfig(AppConfig):
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
# Commands that don't need Neo4j
|
||||
SKIP_NEO4J_DJANGO_COMMANDS = [
|
||||
"makemigrations",
|
||||
"migrate",
|
||||
"pgpartition",
|
||||
"check",
|
||||
"help",
|
||||
"showmigrations",
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook)
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
(
|
||||
"manage.py" in sys.argv[0]
|
||||
and sys.argv[1] in SKIP_NEO4J_DJANGO_COMMANDS
|
||||
)
|
||||
or "celery" in sys.argv[0]
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)"
|
||||
)
|
||||
|
||||
else:
|
||||
graph_database.init_driver()
|
||||
|
||||
# Neo4j driver is initialized at API startup (see api.attack_paths.database)
|
||||
# It remains lazy for Celery workers and selected Django commands
|
||||
# Neo4j driver is created lazily on first use (see api.attack_paths.database).
|
||||
# App init never contacts Neo4j, so a Neo4j outage cannot block API startup.
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
|
||||
@@ -1,22 +1,24 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Iterator
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
@@ -28,6 +30,9 @@ READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
|
||||
# Shorter than CONN_ACQUISITION_TIMEOUT — the driver requires acquisition to be
|
||||
# the longer of the two (it may include opening a new connection).
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
@@ -58,15 +63,24 @@ def init_driver() -> neo4j.Driver:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
_driver.verify_connectivity()
|
||||
# Publish the singleton only after connectivity is verified so a
|
||||
# failed probe does not leave an unverified driver behind. Close the
|
||||
# driver on failure so a repeatedly-probed outage cannot leak pools.
|
||||
try:
|
||||
driver.verify_connectivity()
|
||||
except Exception:
|
||||
driver.close()
|
||||
raise
|
||||
_driver = driver
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
@@ -161,7 +175,8 @@ def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
"""
|
||||
Delete all nodes for a provider from the tenant database.
|
||||
|
||||
Uses batched deletion to avoid memory issues with large graphs.
|
||||
Deletes relationships then nodes in batches (not `DETACH DELETE`) so a dense
|
||||
provider's graph cannot exceed Neo4j's transaction memory limit.
|
||||
Silently returns 0 if the database doesn't exist.
|
||||
"""
|
||||
provider_label = get_provider_label(provider_id)
|
||||
@@ -169,13 +184,28 @@ def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
# Phase 1: delete relationships incident to provider nodes in batches.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (:`{provider_label}`)-[r]-()
|
||||
WITH DISTINCT r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_rels_count", 0)
|
||||
|
||||
# Phase 2: delete the now relationship-free nodes in batches.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DETACH DELETE n
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
|
||||
@@ -93,3 +93,30 @@ class CombinedJWTOrAPIKeyAuthentication(BaseAuthentication):
|
||||
|
||||
# Default fallback
|
||||
return self.jwt_auth.authenticate(request)
|
||||
|
||||
|
||||
class SSEAuthentication(CombinedJWTOrAPIKeyAuthentication):
|
||||
"""JWT/API-Key auth that also accepts `?access_token=<jwt>`.
|
||||
|
||||
Browser `EventSource` is the only widely available SSE client API
|
||||
and it cannot set the `Authorization` header (its constructor takes
|
||||
only a URL and `withCredentials`). To keep browser SSE clients on
|
||||
the same auth stack as the rest of the API, SSE endpoints additionally
|
||||
accept a JWT via the `?access_token=<jwt>` query parameter — the
|
||||
standard parameter name defined in RFC 6750 Section 2.3 for bearer tokens.
|
||||
"""
|
||||
|
||||
def authenticate(self, request: Request):
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
if auth_header:
|
||||
return super().authenticate(request)
|
||||
|
||||
raw_token = request.query_params.get("access_token")
|
||||
if not raw_token:
|
||||
# No header and no query token — let the default path raise
|
||||
# the canonical AuthenticationFailed via the parent class.
|
||||
return super().authenticate(request)
|
||||
|
||||
validated_token = self.jwt_auth.get_validated_token(raw_token)
|
||||
user = self.jwt_auth.get_user(validated_token)
|
||||
return user, validated_token
|
||||
|
||||
@@ -1,11 +1,26 @@
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
|
||||
|
||||
# Per-process readiness flags for the background compliance warm-up.
|
||||
# `STARTED` is set as soon as warming begins (only happens under Gunicorn via
|
||||
# the post_fork hook); `WARMED` is set when it finishes. The attributes
|
||||
# endpoint checks both: it returns 503 only while warming is in progress.
|
||||
# Under `runserver` warming never runs, so `STARTED` stays clear and the
|
||||
# endpoint keeps lazy-loading as before.
|
||||
COMPLIANCE_WARMING_STARTED = threading.Event()
|
||||
COMPLIANCE_WARMED = threading.Event()
|
||||
|
||||
|
||||
class LazyComplianceTemplate(Mapping):
|
||||
"""Lazy-load compliance templates per provider on first access."""
|
||||
@@ -94,25 +109,22 @@ PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""List compliance frameworks the API can load for `provider_type`.
|
||||
"""List compliance framework identifiers available for `provider_type`.
|
||||
|
||||
The list is sourced from `Compliance.get_bulk` so that the names
|
||||
returned here are guaranteed to be loadable by the bulk loader. This
|
||||
prevents downstream key mismatches (e.g. CSV report generation iterating
|
||||
framework names and looking them up in the bulk dict).
|
||||
Includes both per-provider frameworks and universal top-level frameworks
|
||||
(e.g. ``dora``, ``csa_ccm_4.0``).
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type for which to retrieve
|
||||
available compliance frameworks (e.g., "aws", "azure", "gcp", "m365").
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type
|
||||
(e.g., "aws", "azure", "gcp", "m365").
|
||||
|
||||
Returns:
|
||||
list[str]: A list of framework identifiers (e.g., "cis_1.4_aws", "mitre_attack_azure") available
|
||||
for the given provider.
|
||||
list[str]: Framework identifiers (e.g., "cis_1.4_aws", "dora").
|
||||
"""
|
||||
global AVAILABLE_COMPLIANCE_FRAMEWORKS
|
||||
if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS:
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = list(
|
||||
Compliance.get_bulk(provider_type).keys()
|
||||
get_bulk_compliance_frameworks_universal(provider_type).keys()
|
||||
)
|
||||
|
||||
return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type]
|
||||
@@ -139,18 +151,14 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
|
||||
"""
|
||||
Retrieve the Prowler compliance data for a specified provider type.
|
||||
|
||||
This function fetches the compliance frameworks and their associated
|
||||
requirements for the given cloud provider.
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The provider type
|
||||
(e.g., 'aws', 'azure') for which to retrieve compliance data.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary mapping compliance framework names to their respective
|
||||
Compliance objects for the specified provider.
|
||||
dict: Mapping of framework name to `ComplianceFramework` for the provider.
|
||||
"""
|
||||
return Compliance.get_bulk(provider_type)
|
||||
return get_bulk_compliance_frameworks_universal(provider_type)
|
||||
|
||||
|
||||
def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
|
||||
@@ -179,6 +187,56 @@ def _ensure_provider_loaded(provider_type: Provider.ProviderChoices) -> None:
|
||||
PROWLER_CHECKS._cache[provider_type] = checks
|
||||
|
||||
|
||||
def warm_compliance_caches(
|
||||
provider_types: Iterable[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Eagerly populate the per-process compliance caches at server startup.
|
||||
|
||||
Moves the cold-cache catalog load off the request thread so the first
|
||||
request does not trip the Gunicorn worker timeout. Reads only on-disk
|
||||
metadata (no database access). Each provider is warmed in isolation;
|
||||
failures are logged and fall back to lazy loading.
|
||||
|
||||
Args:
|
||||
provider_types (Iterable[str] | None): Subset to warm. Defaults to all.
|
||||
|
||||
Returns:
|
||||
list[str]: Provider types that could not be warmed.
|
||||
"""
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
provider_types = list(provider_types)
|
||||
|
||||
COMPLIANCE_WARMING_STARTED.set()
|
||||
logger.info("Compliance cache warm-up started for providers: %s", provider_types)
|
||||
|
||||
failed = []
|
||||
for provider_type in provider_types:
|
||||
try:
|
||||
get_compliance_frameworks(provider_type)
|
||||
_ensure_provider_loaded(provider_type)
|
||||
# Prowler check loading may sys.exit (SystemExit, not Exception).
|
||||
except (Exception, SystemExit):
|
||||
logger.warning(
|
||||
"Failed to warm compliance caches for provider '%s'; "
|
||||
"loading lazily on first request",
|
||||
provider_type,
|
||||
exc_info=True,
|
||||
)
|
||||
failed.append(provider_type)
|
||||
|
||||
# Mark as warmed even when some providers failed: a failed provider falls
|
||||
# back to a single-provider lazy load, which stays under the worker timeout.
|
||||
COMPLIANCE_WARMED.set()
|
||||
logger.info(
|
||||
"Compliance cache warm-up finished (providers warmed: %d, failed: %s)",
|
||||
len(provider_types) - len(failed),
|
||||
failed,
|
||||
)
|
||||
return failed
|
||||
|
||||
|
||||
def load_prowler_checks(
|
||||
prowler_compliance, provider_types: Iterable[str] | None = None
|
||||
):
|
||||
@@ -209,8 +267,8 @@ def load_prowler_checks(
|
||||
for compliance_name, compliance_data in prowler_compliance.get(
|
||||
provider_type, {}
|
||||
).items():
|
||||
for requirement in compliance_data.Requirements:
|
||||
for check in requirement.Checks:
|
||||
for requirement in compliance_data.requirements:
|
||||
for check in requirement.checks.get(provider_type, []):
|
||||
try:
|
||||
checks[provider_type][check].add(compliance_name)
|
||||
except KeyError:
|
||||
@@ -290,24 +348,40 @@ def generate_compliance_overview_template(
|
||||
requirements_status = {"passed": 0, "failed": 0, "manual": 0}
|
||||
total_requirements = 0
|
||||
|
||||
for requirement in compliance_data.Requirements:
|
||||
for requirement in compliance_data.requirements:
|
||||
total_requirements += 1
|
||||
total_checks = len(requirement.Checks)
|
||||
checks_dict = {check: None for check in requirement.Checks}
|
||||
provider_check_list = list(requirement.checks.get(provider_type, []))
|
||||
total_checks = len(provider_check_list)
|
||||
checks_dict = {check: None for check in provider_check_list}
|
||||
|
||||
req_status_val = "MANUAL" if total_checks == 0 else "PASS"
|
||||
|
||||
# MITRE attrs are wrapped under `_raw_attributes` by the
|
||||
# universal adapter — unwrap so consumers see the flat list.
|
||||
requirement_attributes = requirement.attributes
|
||||
if (
|
||||
isinstance(requirement_attributes, dict)
|
||||
and "_raw_attributes" in requirement_attributes
|
||||
):
|
||||
attributes_payload = list(requirement_attributes["_raw_attributes"])
|
||||
elif isinstance(requirement_attributes, dict):
|
||||
attributes_payload = (
|
||||
[dict(requirement_attributes)] if requirement_attributes else []
|
||||
)
|
||||
else:
|
||||
attributes_payload = [
|
||||
dict(attribute) for attribute in requirement_attributes
|
||||
]
|
||||
|
||||
# Build requirement dictionary
|
||||
requirement_dict = {
|
||||
"name": requirement.Name or requirement.Id,
|
||||
"description": requirement.Description,
|
||||
"tactics": getattr(requirement, "Tactics", []),
|
||||
"subtechniques": getattr(requirement, "SubTechniques", []),
|
||||
"platforms": getattr(requirement, "Platforms", []),
|
||||
"technique_url": getattr(requirement, "TechniqueURL", ""),
|
||||
"attributes": [
|
||||
dict(attribute) for attribute in requirement.Attributes
|
||||
],
|
||||
"name": requirement.name or requirement.id,
|
||||
"description": requirement.description,
|
||||
"tactics": requirement.tactics or [],
|
||||
"subtechniques": requirement.sub_techniques or [],
|
||||
"platforms": requirement.platforms or [],
|
||||
"technique_url": requirement.technique_url or "",
|
||||
"attributes": attributes_payload,
|
||||
"checks": checks_dict,
|
||||
"checks_status": {
|
||||
"pass": 0,
|
||||
@@ -325,15 +399,15 @@ def generate_compliance_overview_template(
|
||||
requirements_status["passed"] += 1
|
||||
|
||||
# Add requirement to compliance requirements
|
||||
compliance_requirements[requirement.Id] = requirement_dict
|
||||
compliance_requirements[requirement.id] = requirement_dict
|
||||
|
||||
# Build compliance dictionary
|
||||
compliance_dict = {
|
||||
"framework": compliance_data.Framework,
|
||||
"name": compliance_data.Name,
|
||||
"version": compliance_data.Version,
|
||||
"framework": compliance_data.framework,
|
||||
"name": compliance_data.name,
|
||||
"version": compliance_data.version,
|
||||
"provider": provider_type,
|
||||
"description": compliance_data.Description,
|
||||
"description": compliance_data.description,
|
||||
"requirements": compliance_requirements,
|
||||
"requirements_status": requirements_status,
|
||||
"total_requirements": total_requirements,
|
||||
|
||||
@@ -187,6 +187,32 @@ class UpstreamServiceUnavailableError(APIException):
|
||||
)
|
||||
|
||||
|
||||
class ComplianceWarmingError(APIException):
|
||||
"""Compliance catalog is still warming (503 Service Unavailable).
|
||||
|
||||
Returned by the compliance attributes endpoint while the per-process
|
||||
catalog warm-up is in progress, so the request thread never triggers the
|
||||
slow cold load that would trip the Gunicorn worker timeout.
|
||||
"""
|
||||
|
||||
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
default_detail = (
|
||||
"Compliance data is still loading. Please try again in a few seconds."
|
||||
)
|
||||
default_code = "compliance_warming"
|
||||
|
||||
def __init__(self, detail=None):
|
||||
super().__init__(
|
||||
detail=[
|
||||
{
|
||||
"detail": detail or self.default_detail,
|
||||
"status": str(self.status_code),
|
||||
"code": self.default_code,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class UpstreamInternalError(APIException):
|
||||
"""Unexpected error communicating with provider (500 Internal Server Error).
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ class BaseProviderFilter(FilterSet):
|
||||
"""
|
||||
Abstract base filter for models with direct FK to Provider.
|
||||
|
||||
Provides standard provider_id and provider_type filters.
|
||||
Provides standard provider_id, provider_type, and provider_groups filters.
|
||||
Subclasses must define Meta.model.
|
||||
"""
|
||||
|
||||
@@ -116,6 +116,16 @@ class BaseProviderFilter(FilterSet):
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
@@ -126,7 +136,7 @@ class BaseScanProviderFilter(FilterSet):
|
||||
"""
|
||||
Abstract base filter for models with FK to Scan (and Scan has FK to Provider).
|
||||
|
||||
Provides standard provider_id and provider_type filters via scan relationship.
|
||||
Provides standard provider_id, provider_type, and provider_groups filters via scan relationship.
|
||||
Subclasses must define Meta.model.
|
||||
"""
|
||||
|
||||
@@ -140,6 +150,16 @@ class BaseScanProviderFilter(FilterSet):
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
@@ -160,6 +180,16 @@ class CommonFindingFilters(FilterSet):
|
||||
provider_type__in = ChoiceInFilter(
|
||||
choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
provider_uid = CharFilter(field_name="scan__provider__uid", lookup_expr="exact")
|
||||
provider_uid__in = CharInFilter(field_name="scan__provider__uid", lookup_expr="in")
|
||||
provider_uid__icontains = CharFilter(
|
||||
@@ -370,6 +400,12 @@ class ProviderFilter(FilterSet):
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider_groups__id", lookup_expr="exact", distinct=True
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider_groups__id", lookup_expr="in", distinct=True
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Provider
|
||||
@@ -395,6 +431,16 @@ class ProviderRelationshipFilterSet(FilterSet):
|
||||
provider_type__in = ChoiceInFilter(
|
||||
choices=Provider.ProviderChoices.choices, field_name="provider__provider"
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
provider_uid = CharFilter(field_name="provider__uid", lookup_expr="exact")
|
||||
provider_uid__in = CharInFilter(field_name="provider__uid", lookup_expr="in")
|
||||
provider_uid__icontains = CharFilter(
|
||||
@@ -1001,6 +1047,16 @@ class FindingGroupSummaryFilter(_CheckTitleToCheckIdMixin, FilterSet):
|
||||
field_name="provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = CharInFilter(field_name="provider__provider", lookup_expr="in")
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = FindingGroupDailySummary
|
||||
@@ -1101,6 +1157,16 @@ class LatestFindingGroupSummaryFilter(_CheckTitleToCheckIdMixin, FilterSet):
|
||||
field_name="provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = CharInFilter(field_name="provider__provider", lookup_expr="in")
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = FindingGroupDailySummary
|
||||
@@ -1280,12 +1346,19 @@ class RoleFilter(FilterSet):
|
||||
}
|
||||
|
||||
|
||||
class ComplianceOverviewFilter(FilterSet):
|
||||
class ComplianceOverviewFilter(BaseScanProviderFilter):
|
||||
"""
|
||||
Keep provider filters in the schema while runtime filtering resolves scans first.
|
||||
|
||||
Compliance overview provider filters are applied to the latest completed scans
|
||||
in the viewset, then this filterset handles the remaining compliance fields.
|
||||
"""
|
||||
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
scan_id = UUIDFilter(field_name="scan_id", required=True)
|
||||
scan_id = UUIDFilter(field_name="scan_id")
|
||||
region = CharFilter(field_name="region")
|
||||
|
||||
class Meta:
|
||||
class Meta(BaseScanProviderFilter.Meta):
|
||||
model = ComplianceRequirementOverview
|
||||
fields = {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
@@ -1306,6 +1379,16 @@ class ScanSummaryFilter(FilterSet):
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
region = CharFilter(field_name="region")
|
||||
|
||||
class Meta:
|
||||
@@ -1329,6 +1412,16 @@ class DailySeveritySummaryFilter(FilterSet):
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
date_from = DateFilter(method="filter_noop")
|
||||
date_to = DateFilter(method="filter_noop")
|
||||
|
||||
@@ -1585,6 +1678,16 @@ class ThreatScoreSnapshotFilter(FilterSet):
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
compliance_id = CharFilter(field_name="compliance_id", lookup_expr="exact")
|
||||
compliance_id__in = CharInFilter(field_name="compliance_id", lookup_expr="in")
|
||||
|
||||
@@ -1628,6 +1731,16 @@ class ResourceGroupOverviewFilter(FilterSet):
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
provider_groups = UUIDFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="exact",
|
||||
distinct=True,
|
||||
)
|
||||
provider_groups__in = UUIDInFilter(
|
||||
field_name="scan__provider__provider_groups__id",
|
||||
lookup_expr="in",
|
||||
distinct=True,
|
||||
)
|
||||
resource_group = CharFilter(field_name="resource_group", lookup_expr="exact")
|
||||
resource_group__in = CharInFilter(field_name="resource_group", lookup_expr="in")
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from tasks.jobs.orphan_recovery import reconcile_orphans
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Recover orphaned allowlisted Celery tasks whose worker is gone and mark "
|
||||
"other stale task results terminal. Single-flight via a Postgres advisory lock."
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--grace-minutes",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Skip tasks started within this window (worker may still register).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-attempts",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Give up re-running a task after this many recovery attempts; it is then left terminal instead of re-enqueued.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect and report orphans without revoking or re-enqueuing.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
result = reconcile_orphans(
|
||||
grace_minutes=options["grace_minutes"],
|
||||
max_attempts=options["max_attempts"],
|
||||
dry_run=options["dry_run"],
|
||||
)
|
||||
|
||||
if not result.get("acquired"):
|
||||
self.stdout.write("Reconcile skipped: another run holds the lock.")
|
||||
return
|
||||
|
||||
if result.get("enabled") is False:
|
||||
message = (
|
||||
"Task recovery is disabled (DJANGO_TASK_RECOVERY_ENABLED is off); "
|
||||
"no orphans were recovered."
|
||||
)
|
||||
if result.get("attack_paths") is not None:
|
||||
message += " Attack-paths stale cleanup still ran."
|
||||
self.stdout.write(message)
|
||||
return
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
"Orphan reconcile complete: "
|
||||
f"recovered={len(result.get('recovered', []))} "
|
||||
f"failed={len(result.get('failed', []))} "
|
||||
f"skipped(in-flight)={len(result.get('skipped', []))}"
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
TASK_NAME = "reconcile-orphan-tasks"
|
||||
INTERVAL_MINUTES = 2
|
||||
|
||||
|
||||
def create_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
schedule, _ = IntervalSchedule.objects.get_or_create(
|
||||
every=INTERVAL_MINUTES,
|
||||
period="minutes",
|
||||
)
|
||||
|
||||
PeriodicTask.objects.update_or_create(
|
||||
name=TASK_NAME,
|
||||
defaults={
|
||||
"task": TASK_NAME,
|
||||
"interval": schedule,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def delete_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
PeriodicTask.objects.filter(name=TASK_NAME).delete()
|
||||
|
||||
# Clean up the schedule if no other task references it
|
||||
IntervalSchedule.objects.filter(
|
||||
every=INTERVAL_MINUTES,
|
||||
period="minutes",
|
||||
periodictask__isnull=True,
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0093_okta_provider"),
|
||||
("django_celery_beat", "0019_alter_periodictasks_options"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_periodic_task, delete_periodic_task),
|
||||
]
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.30.0
|
||||
version: 1.32.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
@@ -13137,8 +13137,59 @@ paths:
|
||||
responses:
|
||||
'200':
|
||||
description: CSV file containing the compliance report
|
||||
'202':
|
||||
description: The task is in progress
|
||||
'403':
|
||||
description: There is a problem with credentials
|
||||
'404':
|
||||
description: Compliance report not found
|
||||
description: Compliance report not found, or the scan has no reports yet
|
||||
/api/v1/scans/{id}/compliance/{name}/ocsf:
|
||||
get:
|
||||
operationId: scans_compliance_ocsf_retrieve
|
||||
description: Download a specific compliance report as an OCSF JSON file. Only
|
||||
universal frameworks that declare an output configuration produce this artifact
|
||||
(currently 'dora' and 'csa_ccm_4.0'); any other framework returns 404.
|
||||
summary: Retrieve compliance report as OCSF JSON
|
||||
parameters:
|
||||
- in: query
|
||||
name: fields[scan-reports]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- name
|
||||
description: endpoint return only specific fields in the response on a per-type
|
||||
basis by including a fields[TYPE] query parameter.
|
||||
explode: false
|
||||
- in: path
|
||||
name: id
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
description: A UUID string identifying this scan.
|
||||
required: true
|
||||
- in: path
|
||||
name: name
|
||||
schema:
|
||||
type: string
|
||||
description: The compliance report name, like 'dora'
|
||||
required: true
|
||||
tags:
|
||||
- Scan
|
||||
security:
|
||||
- JWT or API Key: []
|
||||
responses:
|
||||
'200':
|
||||
description: OCSF JSON file containing the compliance report
|
||||
'202':
|
||||
description: The task is in progress
|
||||
'403':
|
||||
description: There is a problem with credentials
|
||||
'404':
|
||||
description: Compliance report not found, the framework does not provide
|
||||
an OCSF export, or the scan has no reports yet
|
||||
/api/v1/scans/{id}/csa:
|
||||
get:
|
||||
operationId: scans_csa_retrieve
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Platform Server-Sent Events (SSE) infrastructure.
|
||||
|
||||
Wires `django-eventstream` into the API: a base viewset features
|
||||
subclass to expose an SSE endpoint
|
||||
(:class:`api.sse.base_views.BaseSSEViewSet`), the channel manager that
|
||||
enforces the tenant gate (:class:`api.sse.channelmanager.SSEChannelManager`),
|
||||
and the channel-name helpers (:func:`api.sse.utils.make_channel_name`).
|
||||
"""
|
||||
|
||||
from api.sse.utils import make_channel_name
|
||||
from api.sse.base_views import BaseSSEViewSet
|
||||
|
||||
__all__ = ["BaseSSEViewSet", "make_channel_name"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Base view class for SSE endpoints."""
|
||||
|
||||
from api.authentication import SSEAuthentication
|
||||
from api.base_views import BaseRLSViewSet
|
||||
from django_eventstream.renderers import SSEEventRenderer
|
||||
from django_eventstream.views import events
|
||||
|
||||
|
||||
class BaseSSEViewSet(BaseRLSViewSet):
|
||||
"""Base class for platform SSE endpoints.
|
||||
|
||||
Subclasses override method `get_channels` to declare the channel
|
||||
names the connection should subscribe to — the same way a regular
|
||||
DRF viewset overrides method `get_queryset`. The channel manager
|
||||
reads the result from `request.sse_channels`; there is no other
|
||||
coupling between platform and feature.
|
||||
"""
|
||||
|
||||
authentication_classes = [SSEAuthentication]
|
||||
# Pin the SSE renderer so content negotiation accepts the browser's
|
||||
# `Accept: text/event-stream`.
|
||||
renderer_classes = [SSEEventRenderer]
|
||||
|
||||
def get_channels(self) -> set[str]:
|
||||
"""Return the channels this connection subscribes to.
|
||||
|
||||
Implementations MUST raise the relevant DRF exceptions
|
||||
(`NotAuthenticated`, `PermissionDenied`, `NotFound`) when
|
||||
authorization fails. Returning an empty set would surface as
|
||||
django-eventstream's "No channels specified" which masks the
|
||||
real cause.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_queryset(self):
|
||||
# Most SSE viewsets only need `get_channels` and never call
|
||||
# `get_queryset` (the SSE list path bypasses serialization
|
||||
# entirely). Subclasses that perform their own queryset lookup
|
||||
# inside `get_channels` should override; the default raises
|
||||
# the same error a missing override on a ModelViewSet would.
|
||||
raise NotImplementedError
|
||||
|
||||
def list(self, request, *_args, **kwargs):
|
||||
"""Resolve channels under the regular DRF stack and stream."""
|
||||
request.sse_channels = self.get_channels()
|
||||
return events(request, **kwargs)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Channel manager that wires `django-eventstream` to platform SSE views."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from uuid import UUID
|
||||
|
||||
from django_eventstream.channelmanager import DefaultChannelManager
|
||||
from rest_framework.request import Request
|
||||
|
||||
from api.sse.utils import tenant_id_from_channel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from api.models import User
|
||||
|
||||
|
||||
class SSEChannelManager(DefaultChannelManager):
|
||||
"""Connect `django-eventstream` to the platform's SSE viewsets."""
|
||||
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]: # noqa: vulture
|
||||
"""Return the request's channels scoped to the active JWT tenant.
|
||||
|
||||
Args:
|
||||
request: The authenticated DRF request, carrying `tenant_id`
|
||||
(set by `BaseRLSViewSet`) and `sse_channels` (set by
|
||||
`BaseSSEViewSet.list`).
|
||||
view_kwargs: URL keyword arguments from django-eventstream;
|
||||
unused because channels are resolved on the request.
|
||||
|
||||
Returns:
|
||||
The subset of `request.sse_channels` whose embedded tenant
|
||||
matches the active request tenant.
|
||||
"""
|
||||
try:
|
||||
request_tenant_id = UUID(str(getattr(request, "tenant_id", None)))
|
||||
except (TypeError, ValueError):
|
||||
return set()
|
||||
return {
|
||||
channel
|
||||
for channel in getattr(request, "sse_channels", set())
|
||||
if tenant_id_from_channel(channel) == request_tenant_id
|
||||
}
|
||||
|
||||
def can_read_channel(self, user: "User | None", channel: str) -> bool:
|
||||
"""Re-verify tenant membership once the stream is established.
|
||||
|
||||
Args:
|
||||
user: The connection's authenticated `User`, or `None` for an
|
||||
anonymous connection — django-eventstream passes `None`
|
||||
rather than an `AnonymousUser`.
|
||||
channel: The channel name being read, in the canonical
|
||||
`<prefix>:<tenant_id>:<resource_id>` format.
|
||||
|
||||
Returns:
|
||||
`True` only when `user` is authenticated and a member of the
|
||||
tenant embedded in `channel`; `False` otherwise, including for
|
||||
anonymous connections and malformed channel names.
|
||||
"""
|
||||
if user is None or not user.is_authenticated:
|
||||
return False
|
||||
tenant_id = tenant_id_from_channel(channel)
|
||||
if tenant_id is None:
|
||||
return False
|
||||
return user.is_member_of_tenant(tenant_id)
|
||||
|
||||
def is_channel_reliable(self, channel: str) -> bool:
|
||||
"""Report whether the channel keeps a server-side replay buffer.
|
||||
|
||||
Args:
|
||||
channel: The channel name being queried.
|
||||
|
||||
Returns:
|
||||
`False`, unconditionally. Replay storage is not configured
|
||||
"""
|
||||
return False
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Channel-name convention shared by SSE publishers, consumers, and the
|
||||
channel manager. The format is `<prefix>:<tenant_id>:<resource_id>`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
CHANNEL_SEPARATOR = ":"
|
||||
|
||||
|
||||
def make_channel_name(
|
||||
prefix: str,
|
||||
tenant_id: str | uuid.UUID,
|
||||
resource_id: str | uuid.UUID,
|
||||
) -> str:
|
||||
"""Build the canonical channel name for a resource.
|
||||
|
||||
Args:
|
||||
prefix: Feature-owned prefix (e.g. `"lighthouse-session"`).
|
||||
tenant_id: Tenant the resource belongs to.
|
||||
resource_id: Resource identifier within the tenant.
|
||||
|
||||
Raises:
|
||||
ValueError: If any segment contains `CHANNEL_SEPARATOR`, which
|
||||
would break the `<prefix>:<tenant_id>:<resource_id>` contract
|
||||
and let a crafted name smuggle extra segments past the parser.
|
||||
"""
|
||||
segments = (str(prefix), str(tenant_id), str(resource_id))
|
||||
if any(CHANNEL_SEPARATOR in segment for segment in segments):
|
||||
raise ValueError(
|
||||
f"Channel segments must not contain '{CHANNEL_SEPARATOR}': {segments!r}"
|
||||
)
|
||||
return CHANNEL_SEPARATOR.join(segments)
|
||||
|
||||
|
||||
def tenant_id_from_channel(channel: str) -> uuid.UUID | None:
|
||||
"""Return the tenant UUID embedded in *channel*, or `None` if
|
||||
*channel* does not follow the platform convention.
|
||||
|
||||
A `None` result MUST be treated by callers as "not authorized" or
|
||||
a malformed channel cannot be safely read.
|
||||
"""
|
||||
segments = channel.split(CHANNEL_SEPARATOR)
|
||||
if len(segments) != 3:
|
||||
# Reject non-canonical names
|
||||
return None
|
||||
try:
|
||||
return uuid.UUID(segments[1])
|
||||
except ValueError:
|
||||
return None
|
||||
@@ -182,23 +182,19 @@ def _make_app():
|
||||
return ApiConfig("api", api)
|
||||
|
||||
|
||||
def test_ready_initializes_driver_for_api_process(monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
["gunicorn"],
|
||||
["celery", "-A", "api"],
|
||||
["manage.py", "migrate"],
|
||||
],
|
||||
ids=["api", "celery", "manage_py"],
|
||||
)
|
||||
def test_ready_never_eagerly_initializes_neo4j_driver(monkeypatch, argv):
|
||||
"""ready() must never contact Neo4j; the driver is created lazily on first use."""
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_called_once()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["celery", "-A", "api"])
|
||||
_set_argv(monkeypatch, argv)
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
@@ -208,31 +204,3 @@ def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_manage_py_skip_command(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["manage.py", "migrate"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_when_testing(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, True)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
"""
|
||||
Tests for Neo4j database lazy initialization.
|
||||
|
||||
The Neo4j driver connects on first use by default. API processes may
|
||||
eagerly initialize the driver during app startup, while Celery workers
|
||||
remain lazy. These tests validate the database module behavior itself.
|
||||
The Neo4j driver is created on first use for every process type; app startup
|
||||
never contacts Neo4j. These tests validate the database module behavior itself.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
import pytest
|
||||
|
||||
import api.attack_paths.database as db_module
|
||||
@@ -59,6 +60,32 @@ class TestLazyInitialization:
|
||||
assert result is mock_driver
|
||||
assert db_module._driver is mock_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_leaves_driver_none_when_verify_fails(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""A failed verify_connectivity() must not publish or leak the driver."""
|
||||
mock_driver = MagicMock()
|
||||
mock_driver.verify_connectivity.side_effect = (
|
||||
neo4j.exceptions.ServiceUnavailable("down")
|
||||
)
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
with pytest.raises(neo4j.exceptions.ServiceUnavailable):
|
||||
db_module.init_driver()
|
||||
|
||||
assert db_module._driver is None
|
||||
mock_driver.close.assert_called_once()
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_returns_cached_driver_on_subsequent_calls(
|
||||
@@ -116,21 +143,23 @@ class TestConnectionAcquisitionTimeout:
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
original_driver = db_module._driver
|
||||
original_timeout = db_module.CONN_ACQUISITION_TIMEOUT
|
||||
original_acq_timeout = db_module.CONN_ACQUISITION_TIMEOUT
|
||||
original_conn_timeout = db_module.CONNECTION_TIMEOUT
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = original_timeout
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = original_acq_timeout
|
||||
db_module.CONNECTION_TIMEOUT = original_conn_timeout
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_driver_receives_configured_timeout(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""init_driver() should pass CONN_ACQUISITION_TIMEOUT to the neo4j driver."""
|
||||
"""init_driver() should pass the configured timeouts to the neo4j driver."""
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
@@ -141,11 +170,13 @@ class TestConnectionAcquisitionTimeout:
|
||||
}
|
||||
}
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = 42
|
||||
db_module.CONNECTION_TIMEOUT = 7
|
||||
|
||||
db_module.init_driver()
|
||||
|
||||
_, kwargs = mock_driver_factory.call_args
|
||||
assert kwargs["connection_acquisition_timeout"] == 42
|
||||
assert kwargs["connection_timeout"] == 7
|
||||
|
||||
|
||||
class TestAtexitRegistration:
|
||||
@@ -511,3 +542,84 @@ class TestHasProviderData:
|
||||
):
|
||||
with pytest.raises(db_module.GraphDatabaseQueryException):
|
||||
db_module.has_provider_data("db-tenant-abc", "provider-123")
|
||||
|
||||
|
||||
class TestDropSubgraph:
|
||||
"""Test drop_subgraph two-phase batched deletion of a provider's graph."""
|
||||
|
||||
@staticmethod
|
||||
def _result(count):
|
||||
result = MagicMock()
|
||||
result.single.return_value.get.return_value = count
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _session_ctx(session):
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = session
|
||||
ctx.__exit__.return_value = False
|
||||
return ctx
|
||||
|
||||
def test_deletes_relationships_then_nodes_in_batches(self):
|
||||
session = MagicMock()
|
||||
# Phase 1 (relationships): one full batch then empty.
|
||||
# Phase 2 (nodes): one full batch then empty.
|
||||
session.run.side_effect = [
|
||||
self._result(1000),
|
||||
self._result(0),
|
||||
self._result(1000),
|
||||
self._result(0),
|
||||
]
|
||||
|
||||
with patch(
|
||||
"api.attack_paths.database.get_session",
|
||||
return_value=self._session_ctx(session),
|
||||
):
|
||||
deleted = db_module.drop_subgraph("db-tenant-abc", "provider-123")
|
||||
|
||||
# Only phase-2 node counts contribute to the return value.
|
||||
assert deleted == 1000
|
||||
assert session.run.call_count == 4
|
||||
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
|
||||
# Regression guard: the memory blow-up was caused by DETACH DELETE.
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
|
||||
rel_queries = [query for query in queries if "DELETE r" in query]
|
||||
node_queries = [query for query in queries if "DELETE n" in query]
|
||||
assert rel_queries and node_queries
|
||||
# DISTINCT avoids double-counting relationships matched from both ends.
|
||||
assert all("DISTINCT r" in query for query in rel_queries)
|
||||
|
||||
# Relationships must be fully drained before nodes are deleted.
|
||||
first_node = next(i for i, q in enumerate(queries) if "DELETE n" in q)
|
||||
last_rel = max(i for i, q in enumerate(queries) if "DELETE r" in q)
|
||||
assert last_rel < first_node
|
||||
|
||||
def test_returns_zero_when_database_not_found(self):
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.side_effect = db_module.GraphDatabaseQueryException(
|
||||
message="Database does not exist",
|
||||
code="Neo.ClientError.Database.DatabaseNotFound",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"api.attack_paths.database.get_session",
|
||||
return_value=session_ctx,
|
||||
):
|
||||
assert db_module.drop_subgraph("db-tenant-gone", "provider-123") == 0
|
||||
|
||||
def test_raises_on_other_errors(self):
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.side_effect = db_module.GraphDatabaseQueryException(
|
||||
message="Connection refused",
|
||||
code="Neo.TransientError.General.UnknownError",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"api.attack_paths.database.get_session",
|
||||
return_value=session_ctx,
|
||||
):
|
||||
with pytest.raises(db_module.GraphDatabaseQueryException):
|
||||
db_module.drop_subgraph("db-tenant-abc", "provider-123")
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from django.test import RequestFactory
|
||||
from rest_framework.exceptions import AuthenticationFailed
|
||||
|
||||
from api.authentication import TenantAPIKeyAuthentication
|
||||
from api.authentication import SSEAuthentication, TenantAPIKeyAuthentication
|
||||
from api.db_router import MainRouter
|
||||
from api.models import TenantAPIKey
|
||||
|
||||
@@ -382,3 +382,62 @@ class TestTenantAPIKeyAuthentication:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "API Key has already expired."
|
||||
|
||||
|
||||
class TestSSEAuthentication:
|
||||
"""`SSEAuthentication` adds an `?access_token=<jwt>` fallback for
|
||||
browser `EventSource` clients while keeping the standard
|
||||
`Authorization` header as the authoritative source."""
|
||||
|
||||
def test_header_present_delegates_to_super(self):
|
||||
request = MagicMock()
|
||||
request.headers = {"Authorization": "Bearer header-token"}
|
||||
with patch.object(
|
||||
SSEAuthentication.__bases__[0], "authenticate", return_value=("user", "tok")
|
||||
) as super_auth:
|
||||
result = SSEAuthentication().authenticate(request)
|
||||
super_auth.assert_called_once_with(request)
|
||||
assert result == ("user", "tok")
|
||||
|
||||
def test_no_header_no_query_token_delegates_to_super(self):
|
||||
request = MagicMock()
|
||||
request.headers = {}
|
||||
request.query_params = {}
|
||||
with patch.object(
|
||||
SSEAuthentication.__bases__[0], "authenticate", return_value=None
|
||||
) as super_auth:
|
||||
result = SSEAuthentication().authenticate(request)
|
||||
super_auth.assert_called_once_with(request)
|
||||
assert result is None
|
||||
|
||||
def test_query_token_used_only_as_fallback(self):
|
||||
request = MagicMock()
|
||||
request.headers = {}
|
||||
request.query_params = {"access_token": "query-jwt"}
|
||||
|
||||
jwt_instance = MagicMock()
|
||||
jwt_instance.get_validated_token.return_value = "validated"
|
||||
jwt_instance.get_user.return_value = "query-user"
|
||||
|
||||
with patch.object(SSEAuthentication, "jwt_auth", jwt_instance):
|
||||
user, token = SSEAuthentication().authenticate(request)
|
||||
|
||||
jwt_instance.get_validated_token.assert_called_once_with("query-jwt")
|
||||
assert user == "query-user"
|
||||
assert token == "validated"
|
||||
|
||||
def test_query_token_invalid_raises_authentication_failed(self):
|
||||
request = MagicMock()
|
||||
request.headers = {}
|
||||
request.query_params = {"access_token": "bad-token"}
|
||||
|
||||
jwt_instance = MagicMock()
|
||||
jwt_instance.get_validated_token.side_effect = AuthenticationFailed(
|
||||
"Invalid token"
|
||||
)
|
||||
|
||||
with patch.object(SSEAuthentication, "jwt_auth", jwt_instance):
|
||||
with pytest.raises(AuthenticationFailed):
|
||||
SSEAuthentication().authenticate(request)
|
||||
|
||||
jwt_instance.get_validated_token.assert_called_once_with("bad-token")
|
||||
|
||||
@@ -10,9 +10,12 @@ from api.compliance import (
|
||||
get_prowler_provider_checks,
|
||||
get_prowler_provider_compliance,
|
||||
load_prowler_checks,
|
||||
warm_compliance_caches,
|
||||
)
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
|
||||
|
||||
class TestCompliance:
|
||||
@@ -28,16 +31,16 @@ class TestCompliance:
|
||||
assert set(checks) == {"check1", "check2", "check3"}
|
||||
mock_check_metadata.get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.compliance.Compliance")
|
||||
def test_get_prowler_provider_compliance(self, mock_compliance):
|
||||
@patch("api.compliance.get_bulk_compliance_frameworks_universal")
|
||||
def test_get_prowler_provider_compliance(self, mock_get_bulk):
|
||||
provider_type = Provider.ProviderChoices.AWS
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
mock_get_bulk.return_value = {
|
||||
"compliance1": MagicMock(),
|
||||
"compliance2": MagicMock(),
|
||||
}
|
||||
compliance_data = get_prowler_provider_compliance(provider_type)
|
||||
assert compliance_data == mock_compliance.get_bulk.return_value
|
||||
mock_compliance.get_bulk.assert_called_once_with(provider_type)
|
||||
assert compliance_data == mock_get_bulk.return_value
|
||||
mock_get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.compliance.get_prowler_provider_checks")
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
@@ -51,9 +54,9 @@ class TestCompliance:
|
||||
prowler_compliance = {
|
||||
"aws": {
|
||||
"compliance1": MagicMock(
|
||||
Requirements=[
|
||||
requirements=[
|
||||
MagicMock(
|
||||
Checks=["check1", "check2"],
|
||||
checks={"aws": ["check1", "check2"]},
|
||||
),
|
||||
],
|
||||
),
|
||||
@@ -167,35 +170,38 @@ class TestCompliance:
|
||||
def test_generate_compliance_overview_template(self, mock_provider_choices):
|
||||
mock_provider_choices.values = ["aws"]
|
||||
|
||||
# ``name`` is a reserved MagicMock kwarg (it labels the mock for repr,
|
||||
# it does NOT set a ``.name`` attribute), so it must be assigned
|
||||
# explicitly after construction.
|
||||
requirement1 = MagicMock(
|
||||
Id="requirement1",
|
||||
Name="Requirement 1",
|
||||
Description="Description of requirement 1",
|
||||
Attributes=[],
|
||||
Checks=["check1", "check2"],
|
||||
Tactics=["tactic1"],
|
||||
SubTechniques=["subtechnique1"],
|
||||
Platforms=["platform1"],
|
||||
TechniqueURL="https://example.com",
|
||||
id="requirement1",
|
||||
description="Description of requirement 1",
|
||||
attributes=[],
|
||||
checks={"aws": ["check1", "check2"]},
|
||||
tactics=["tactic1"],
|
||||
sub_techniques=["subtechnique1"],
|
||||
platforms=["platform1"],
|
||||
technique_url="https://example.com",
|
||||
)
|
||||
requirement1.name = "Requirement 1"
|
||||
requirement2 = MagicMock(
|
||||
Id="requirement2",
|
||||
Name="Requirement 2",
|
||||
Description="Description of requirement 2",
|
||||
Attributes=[],
|
||||
Checks=[],
|
||||
Tactics=[],
|
||||
SubTechniques=[],
|
||||
Platforms=[],
|
||||
TechniqueURL="",
|
||||
id="requirement2",
|
||||
description="Description of requirement 2",
|
||||
attributes=[],
|
||||
checks={"aws": []},
|
||||
tactics=[],
|
||||
sub_techniques=[],
|
||||
platforms=[],
|
||||
technique_url="",
|
||||
)
|
||||
requirement2.name = "Requirement 2"
|
||||
compliance1 = MagicMock(
|
||||
Requirements=[requirement1, requirement2],
|
||||
Framework="Framework 1",
|
||||
Version="1.0",
|
||||
Description="Description of compliance1",
|
||||
Name="Compliance 1",
|
||||
requirements=[requirement1, requirement2],
|
||||
framework="Framework 1",
|
||||
version="1.0",
|
||||
description="Description of compliance1",
|
||||
)
|
||||
compliance1.name = "Compliance 1"
|
||||
prowler_compliance = {"aws": {"compliance1": compliance1}}
|
||||
|
||||
template = generate_compliance_overview_template(prowler_compliance)
|
||||
@@ -262,33 +268,43 @@ def reset_compliance_cache():
|
||||
"""Reset the module-level cache so each test starts cold."""
|
||||
previous = dict(compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS)
|
||||
compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.clear()
|
||||
# The warming flags are module-global; clear them so they do not leak
|
||||
# between tests that call warm_compliance_caches.
|
||||
compliance_module.COMPLIANCE_WARMING_STARTED.clear()
|
||||
compliance_module.COMPLIANCE_WARMED.clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.clear()
|
||||
compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.update(previous)
|
||||
compliance_module.COMPLIANCE_WARMING_STARTED.clear()
|
||||
compliance_module.COMPLIANCE_WARMED.clear()
|
||||
|
||||
|
||||
class TestGetComplianceFrameworks:
|
||||
def test_returns_keys_from_compliance_get_bulk(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
with patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk:
|
||||
mock_get_bulk.return_value = {
|
||||
"cis_1.4_aws": MagicMock(),
|
||||
"mitre_attack_aws": MagicMock(),
|
||||
}
|
||||
result = get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
assert sorted(result) == ["cis_1.4_aws", "mitre_attack_aws"]
|
||||
mock_compliance.get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
mock_get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
|
||||
def test_caches_result_per_provider(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
with patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk:
|
||||
mock_get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
# Cached after first call.
|
||||
assert mock_compliance.get_bulk.call_count == 1
|
||||
assert mock_get_bulk.call_count == 1
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_type",
|
||||
@@ -296,17 +312,105 @@ class TestGetComplianceFrameworks:
|
||||
)
|
||||
def test_listing_is_subset_of_bulk(self, reset_compliance_cache, provider_type):
|
||||
"""Regression for CLOUD-API-40S: every name returned by
|
||||
``get_compliance_frameworks`` must be loadable via ``Compliance.get_bulk``.
|
||||
``get_compliance_frameworks`` must be loadable via
|
||||
``get_bulk_compliance_frameworks_universal``.
|
||||
|
||||
A divergence here is what produced ``KeyError: 'csa_ccm_4.0'`` in
|
||||
``generate_outputs_task`` after universal/multi-provider compliance
|
||||
JSONs were introduced at the top-level ``prowler/compliance/`` path.
|
||||
"""
|
||||
bulk_keys = set(Compliance.get_bulk(provider_type).keys())
|
||||
bulk_keys = set(get_bulk_compliance_frameworks_universal(provider_type).keys())
|
||||
listed = set(get_compliance_frameworks(provider_type))
|
||||
|
||||
missing = listed - bulk_keys
|
||||
assert not missing, (
|
||||
f"get_compliance_frameworks({provider_type!r}) returned names not "
|
||||
f"loadable by Compliance.get_bulk: {sorted(missing)}"
|
||||
f"loadable by get_bulk_compliance_frameworks_universal: "
|
||||
f"{sorted(missing)}"
|
||||
)
|
||||
|
||||
|
||||
class TestWarmComplianceCaches:
|
||||
def test_warms_all_provider_types_by_default(self, reset_compliance_cache):
|
||||
provider_types = list(Provider.ProviderChoices.values)
|
||||
with (
|
||||
patch("api.compliance.get_compliance_frameworks") as mock_frameworks,
|
||||
patch("api.compliance._ensure_provider_loaded") as mock_ensure,
|
||||
):
|
||||
warm_compliance_caches()
|
||||
|
||||
warmed = {call.args[0] for call in mock_frameworks.call_args_list}
|
||||
assert warmed == set(provider_types)
|
||||
assert mock_frameworks.call_count == len(provider_types)
|
||||
assert mock_ensure.call_count == len(provider_types)
|
||||
|
||||
def test_warms_only_requested_provider_types(self, reset_compliance_cache):
|
||||
with (
|
||||
patch("api.compliance.get_compliance_frameworks") as mock_frameworks,
|
||||
patch("api.compliance._ensure_provider_loaded") as mock_ensure,
|
||||
):
|
||||
warm_compliance_caches([Provider.ProviderChoices.AWS])
|
||||
|
||||
mock_frameworks.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
mock_ensure.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
|
||||
def test_populates_module_cache(self, reset_compliance_cache):
|
||||
with (
|
||||
patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk,
|
||||
patch("api.compliance._ensure_provider_loaded"),
|
||||
):
|
||||
mock_get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
warm_compliance_caches([Provider.ProviderChoices.AWS])
|
||||
|
||||
assert (
|
||||
Provider.ProviderChoices.AWS
|
||||
in compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS
|
||||
)
|
||||
|
||||
def test_failing_provider_does_not_abort_the_rest(self, reset_compliance_cache):
|
||||
"""A failing provider (even on SystemExit) is isolated; others warm."""
|
||||
providers = [Provider.ProviderChoices.AWS, Provider.ProviderChoices.OKTA]
|
||||
|
||||
def fake_frameworks(provider_type):
|
||||
if provider_type == Provider.ProviderChoices.OKTA:
|
||||
raise SystemExit(1)
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(
|
||||
"api.compliance.get_compliance_frameworks", side_effect=fake_frameworks
|
||||
),
|
||||
patch("api.compliance._ensure_provider_loaded") as mock_ensure,
|
||||
):
|
||||
failed = warm_compliance_caches(providers)
|
||||
|
||||
assert failed == [Provider.ProviderChoices.OKTA]
|
||||
mock_ensure.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
|
||||
def test_sets_readiness_flags(self, reset_compliance_cache):
|
||||
assert not compliance_module.COMPLIANCE_WARMING_STARTED.is_set()
|
||||
assert not compliance_module.COMPLIANCE_WARMED.is_set()
|
||||
|
||||
with (
|
||||
patch("api.compliance.get_compliance_frameworks"),
|
||||
patch("api.compliance._ensure_provider_loaded"),
|
||||
):
|
||||
warm_compliance_caches([Provider.ProviderChoices.AWS])
|
||||
|
||||
assert compliance_module.COMPLIANCE_WARMING_STARTED.is_set()
|
||||
assert compliance_module.COMPLIANCE_WARMED.is_set()
|
||||
|
||||
def test_marks_warmed_even_when_a_provider_fails(self, reset_compliance_cache):
|
||||
"""A failed provider still leaves the caches flagged as warmed."""
|
||||
with (
|
||||
patch(
|
||||
"api.compliance.get_compliance_frameworks",
|
||||
side_effect=SystemExit(1),
|
||||
),
|
||||
patch("api.compliance._ensure_provider_loaded"),
|
||||
):
|
||||
warm_compliance_caches([Provider.ProviderChoices.AWS])
|
||||
|
||||
assert compliance_module.COMPLIANCE_WARMED.is_set()
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from config.django.base import label_postgres_connections
|
||||
|
||||
|
||||
class TestLabelPostgresConnections:
|
||||
def test_labels_postgres_and_skips_neo4j(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "scan")
|
||||
databases = {
|
||||
"default": {"ENGINE": "psqlextra.backend"},
|
||||
"neo4j": {"HOST": "neo4j", "PORT": "7687"},
|
||||
}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["default"]["OPTIONS"]["application_name"] == "scan:default"
|
||||
assert "OPTIONS" not in databases["neo4j"]
|
||||
|
||||
def test_labels_plain_postgresql_backend(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "api")
|
||||
databases = {"saas": {"ENGINE": "django.db.backends.postgresql"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["saas"]["OPTIONS"]["application_name"] == "api:saas"
|
||||
|
||||
def test_defaults_component_to_api_when_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("DJANGO_APP_COMPONENT", raising=False)
|
||||
databases = {"default": {"ENGINE": "psqlextra.backend"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["default"]["OPTIONS"]["application_name"] == "api:default"
|
||||
|
||||
def test_preserves_existing_options(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "worker")
|
||||
databases = {
|
||||
"replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"OPTIONS": {"sslmode": "require"},
|
||||
}
|
||||
}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["replica"]["OPTIONS"] == {
|
||||
"sslmode": "require",
|
||||
"application_name": "worker:replica",
|
||||
}
|
||||
|
||||
def test_truncates_application_name_to_63_bytes(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "c" * 80)
|
||||
databases = {"default": {"ENGINE": "psqlextra.backend"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert len(databases["default"]["OPTIONS"]["application_name"]) == 63
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Tests for the platform SSE infrastructure (``api.sse``).
|
||||
|
||||
Cover the two security-critical platform pieces — the channel-name
|
||||
convention (:mod:`api.sse.utils`) and the tenant gate enforced by
|
||||
:class:`api.sse.channelmanager.SSEChannelManager`. The SSE authentication
|
||||
class lives in :mod:`api.authentication` with the rest of the auth stack,
|
||||
so its tests live in ``test_authentication.py``. Per-feature SSE endpoints
|
||||
add their own tests on top of these.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from django.http import StreamingHttpResponse
|
||||
from rest_framework.test import APIRequestFactory, force_authenticate
|
||||
|
||||
from api.sse.base_views import BaseSSEViewSet
|
||||
from api.sse.channelmanager import SSEChannelManager
|
||||
from api.sse.utils import make_channel_name, tenant_id_from_channel
|
||||
|
||||
|
||||
class TestMakeChannel:
|
||||
def test_round_trips_tenant_id(self):
|
||||
tenant_id = uuid.uuid4()
|
||||
channel = make_channel_name("lighthouse-session", tenant_id, uuid.uuid4())
|
||||
assert tenant_id_from_channel(channel) == tenant_id
|
||||
|
||||
def test_accepts_str_arguments(self):
|
||||
tenant_id = uuid.uuid4()
|
||||
channel = make_channel_name("lighthouse-session", str(tenant_id), "resource-1")
|
||||
assert channel == f"lighthouse-session:{tenant_id}:resource-1"
|
||||
|
||||
def test_prefix_with_hyphen_is_not_split(self):
|
||||
# Prefixes contain hyphens but never colons, so the tenant id is
|
||||
# always the second colon-separated segment.
|
||||
tenant_id = uuid.uuid4()
|
||||
channel = make_channel_name("a-long-hyphenated-prefix", tenant_id, "res")
|
||||
assert tenant_id_from_channel(channel) == tenant_id
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefix, tenant_id, resource_id",
|
||||
[
|
||||
("evil:prefix", uuid.uuid4(), "res"),
|
||||
("prefix", uuid.uuid4(), "res:extra"),
|
||||
("prefix", "tenant:smuggled", "res"),
|
||||
],
|
||||
)
|
||||
def test_rejects_separator_injection(self, prefix, tenant_id, resource_id):
|
||||
# A colon in any segment would let a crafted name smuggle extra
|
||||
# segments past the parser, so construction must fail loudly.
|
||||
with pytest.raises(ValueError):
|
||||
make_channel_name(prefix, tenant_id, resource_id)
|
||||
|
||||
|
||||
class TestTenantIdFromChannel:
|
||||
def test_returns_none_for_too_few_segments(self):
|
||||
assert tenant_id_from_channel("prefix:only") is None
|
||||
assert tenant_id_from_channel("garbage") is None
|
||||
|
||||
def test_returns_none_for_too_many_segments(self):
|
||||
# A valid tenant UUID in position 1 must not authorize a
|
||||
# non-canonical name that carries extra segments.
|
||||
tenant_id = uuid.uuid4()
|
||||
assert tenant_id_from_channel(f"prefix:{tenant_id}:resource:extra") is None
|
||||
|
||||
def test_returns_none_for_non_uuid_tenant_segment(self):
|
||||
assert tenant_id_from_channel("prefix:not-a-uuid:resource") is None
|
||||
|
||||
def test_parses_valid_channel(self):
|
||||
tenant_id = uuid.uuid4()
|
||||
assert tenant_id_from_channel(f"prefix:{tenant_id}:resource") == tenant_id
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestSSEChannelManager:
|
||||
def test_member_can_read_own_tenant_channel(
|
||||
self, create_test_user, tenants_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
channel = make_channel_name("lighthouse-session", tenant.id, uuid.uuid4())
|
||||
assert SSEChannelManager().can_read_channel(create_test_user, channel)
|
||||
|
||||
def test_non_member_cannot_read_other_tenant_channel(
|
||||
self, create_test_user, tenants_fixture
|
||||
):
|
||||
# create_test_user is a member of tenant1 and tenant2 but not tenant3.
|
||||
foreign_tenant = tenants_fixture[2]
|
||||
channel = make_channel_name(
|
||||
"lighthouse-session", foreign_tenant.id, uuid.uuid4()
|
||||
)
|
||||
assert not SSEChannelManager().can_read_channel(create_test_user, channel)
|
||||
|
||||
def test_anonymous_user_is_rejected(self, tenants_fixture):
|
||||
channel = make_channel_name(
|
||||
"lighthouse-session", tenants_fixture[0].id, uuid.uuid4()
|
||||
)
|
||||
assert not SSEChannelManager().can_read_channel(None, channel)
|
||||
|
||||
anon = MagicMock(is_authenticated=False)
|
||||
assert not SSEChannelManager().can_read_channel(anon, channel)
|
||||
|
||||
def test_malformed_channel_is_rejected(self, create_test_user, tenants_fixture):
|
||||
assert not SSEChannelManager().can_read_channel(create_test_user, "garbage")
|
||||
|
||||
def test_get_channels_for_request_returns_active_tenant_channels(self):
|
||||
tenant_id = uuid.uuid4()
|
||||
own = make_channel_name("prefix", tenant_id, "resource")
|
||||
request = MagicMock()
|
||||
request.tenant_id = str(tenant_id)
|
||||
request.sse_channels = {own}
|
||||
assert SSEChannelManager().get_channels_for_request(request, {}) == {own}
|
||||
|
||||
def test_get_channels_for_request_drops_other_tenant_channels(self):
|
||||
# Fail-closed: a channel for a tenant other than the active JWT
|
||||
# tenant is dropped before reaching django-eventstream, even if the
|
||||
# viewset mistakenly stashed it. This is the primary tenant gate that
|
||||
# binds authorization to request.tenant_id, not just membership.
|
||||
active_tenant = uuid.uuid4()
|
||||
own = make_channel_name("prefix", active_tenant, "resource")
|
||||
foreign = make_channel_name("prefix", uuid.uuid4(), "resource")
|
||||
request = MagicMock()
|
||||
request.tenant_id = str(active_tenant)
|
||||
request.sse_channels = {own, foreign}
|
||||
assert SSEChannelManager().get_channels_for_request(request, {}) == {own}
|
||||
|
||||
def test_get_channels_for_request_drops_malformed_channels(self):
|
||||
request = MagicMock()
|
||||
request.tenant_id = str(uuid.uuid4())
|
||||
request.sse_channels = {"garbage", "prefix:not-a-uuid:resource"}
|
||||
assert SSEChannelManager().get_channels_for_request(request, {}) == set()
|
||||
|
||||
def test_get_channels_for_request_without_tenant_returns_empty(self):
|
||||
# No active tenant on the request (auth/RLS never ran) → fail closed,
|
||||
# regardless of any channels stashed on it.
|
||||
request = MagicMock(spec=[])
|
||||
assert SSEChannelManager().get_channels_for_request(request, {}) == set()
|
||||
|
||||
def test_get_channels_for_request_defaults_to_empty(self):
|
||||
# A request that never went through BaseSSEViewSet.list has no
|
||||
# sse_channels attribute; the manager must not raise.
|
||||
request = object()
|
||||
assert SSEChannelManager().get_channels_for_request(request, {}) == set()
|
||||
|
||||
def test_channel_is_not_reliable(self):
|
||||
# v1 ships without server-side replay storage.
|
||||
assert (
|
||||
SSEChannelManager().is_channel_reliable("prefix:tenant:resource") is False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBaseSSEViewSet:
|
||||
"""End-to-end check that the base viewset opens a stream.
|
||||
|
||||
``BaseSSEViewSet.list`` hands the DRF ``Request`` straight to
|
||||
django-eventstream's ``events()``, which is written for a plain
|
||||
Django request. This drives a real request through the full DRF
|
||||
stack (authentication, RLS, content negotiation, channel manager)
|
||||
and asserts the result is an SSE stream, so the DRF/Django request
|
||||
mismatch cannot regress silently.
|
||||
"""
|
||||
|
||||
def test_list_opens_event_stream(self, create_test_user, tenants_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
channel = make_channel_name("test-sse", tenant.id, uuid.uuid4())
|
||||
seen_tenant_ids = []
|
||||
|
||||
class _StreamingSSEViewSet(BaseSSEViewSet):
|
||||
def get_channels(self):
|
||||
# Reached only after dispatch/initial ran, so the RLS
|
||||
# tenant context is already on the request.
|
||||
seen_tenant_ids.append(self.request.tenant_id)
|
||||
return {channel}
|
||||
|
||||
request = APIRequestFactory().get("/api/v1/test-sse/stream")
|
||||
force_authenticate(
|
||||
request, user=create_test_user, token={"tenant_id": str(tenant.id)}
|
||||
)
|
||||
|
||||
view = _StreamingSSEViewSet.as_view({"get": "list"})
|
||||
response = view(request)
|
||||
|
||||
# A StreamingHttpResponse (not the plain HttpResponse used for SSE
|
||||
# error envelopes) means events() accepted the DRF request, the
|
||||
# channel manager handed it a non-empty channel set, and the
|
||||
# stream was opened end to end.
|
||||
assert isinstance(response, StreamingHttpResponse)
|
||||
assert response.status_code == 200
|
||||
assert response["Content-Type"] == "text/event-stream"
|
||||
assert seen_tenant_ids == [str(tenant.id)]
|
||||
@@ -357,6 +357,30 @@ class TestGetProwlerProviderKwargs:
|
||||
expected_result = {**secret_dict, **expected_extra_kwargs}
|
||||
assert result == expected_result
|
||||
|
||||
def test_get_prowler_provider_kwargs_oraclecloud_converts_region_string_to_set(
|
||||
self,
|
||||
):
|
||||
secret_dict = {
|
||||
"user": "ocid1.user.oc1..fake",
|
||||
"fingerprint": "00:11:22:33:44:55:66:77",
|
||||
"key_content": "-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----",
|
||||
"tenancy": "ocid1.tenancy.oc1..fake",
|
||||
"region": "us-ashburn-1",
|
||||
"pass_phrase": "fake-passphrase",
|
||||
}
|
||||
secret_mock = MagicMock()
|
||||
secret_mock.secret = secret_dict
|
||||
|
||||
provider = MagicMock()
|
||||
provider.provider = Provider.ProviderChoices.ORACLECLOUD.value
|
||||
provider.secret = secret_mock
|
||||
provider.uid = "ocid1.tenancy.oc1..fake"
|
||||
|
||||
result = get_prowler_provider_kwargs(provider)
|
||||
|
||||
expected_result = {**secret_dict, "region": {"us-ashburn-1"}}
|
||||
assert result == expected_result
|
||||
|
||||
def test_get_prowler_provider_kwargs_with_mutelist(self):
|
||||
provider_uid = "provider_uid"
|
||||
secret_dict = {"key": "value"}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -243,6 +243,12 @@ def get_prowler_provider_kwargs(
|
||||
**prowler_provider_kwargs,
|
||||
"filter_accounts": [provider.uid],
|
||||
}
|
||||
elif provider.provider == Provider.ProviderChoices.ORACLECLOUD.value:
|
||||
if isinstance(prowler_provider_kwargs.get("region"), str):
|
||||
prowler_provider_kwargs = {
|
||||
**prowler_provider_kwargs,
|
||||
"region": {prowler_provider_kwargs["region"]},
|
||||
}
|
||||
elif provider.provider == Provider.ProviderChoices.OPENSTACK.value:
|
||||
# clouds_yaml_content, clouds_yaml_cloud and provider_id are validated
|
||||
# in the provider itself, so it's not needed here.
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import uuid
|
||||
|
||||
from django.http import QueryDict
|
||||
from django.urls import reverse
|
||||
from django_celery_results.models import TaskResult
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import ValidationError
|
||||
from rest_framework.response import Response
|
||||
|
||||
from api.exceptions import (
|
||||
@@ -8,7 +12,7 @@ from api.exceptions import (
|
||||
TaskInProgressException,
|
||||
TaskNotFoundException,
|
||||
)
|
||||
from api.models import StateChoices, Task
|
||||
from api.models import Provider, StateChoices, Task
|
||||
from api.v1.serializers import TaskSerializer
|
||||
|
||||
|
||||
@@ -74,6 +78,162 @@ class PaginateByPkMixin:
|
||||
return self.get_paginated_response(serialized)
|
||||
|
||||
|
||||
class JsonApiFilterMixin:
|
||||
"""Shared helpers for manually applying django-filter to JSON:API params."""
|
||||
|
||||
jsonapi_filter_replace_dots = False
|
||||
|
||||
def _normalize_jsonapi_params(
|
||||
self,
|
||||
query_params,
|
||||
exclude_keys=None,
|
||||
replace_dots=None,
|
||||
):
|
||||
exclude_keys = exclude_keys or set()
|
||||
if replace_dots is None:
|
||||
replace_dots = self.jsonapi_filter_replace_dots
|
||||
|
||||
normalized = QueryDict(mutable=True)
|
||||
for key, values in query_params.lists():
|
||||
normalized_key = (
|
||||
key[7:-1] if key.startswith("filter[") and key.endswith("]") else key
|
||||
)
|
||||
if replace_dots:
|
||||
normalized_key = normalized_key.replace(".", "__")
|
||||
if normalized_key not in exclude_keys:
|
||||
normalized.setlist(normalized_key, values)
|
||||
return normalized
|
||||
|
||||
def _apply_filterset(
|
||||
self,
|
||||
queryset,
|
||||
filterset_class,
|
||||
exclude_keys=None,
|
||||
replace_dots=None,
|
||||
):
|
||||
normalized_params = self._normalize_jsonapi_params(
|
||||
self.request.query_params,
|
||||
exclude_keys=set(exclude_keys or []),
|
||||
replace_dots=replace_dots,
|
||||
)
|
||||
filterset = filterset_class(normalized_params, queryset=queryset)
|
||||
if not filterset.is_valid():
|
||||
raise ValidationError(filterset.errors)
|
||||
return filterset.qs
|
||||
|
||||
|
||||
class ProviderFilterParamsMixin(JsonApiFilterMixin):
|
||||
"""Shared extraction of provider filters from JSON:API query params."""
|
||||
|
||||
PROVIDER_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"provider_id",
|
||||
"provider_id__in",
|
||||
"provider_type",
|
||||
"provider_type__in",
|
||||
"provider_groups",
|
||||
"provider_groups__in",
|
||||
}
|
||||
)
|
||||
PROVIDER_FILTER_DOT_ALIAS_KEYS = frozenset(
|
||||
{
|
||||
"provider_id.in",
|
||||
"provider_type.in",
|
||||
"provider_groups.in",
|
||||
}
|
||||
)
|
||||
PROVIDER_FILTER_QUERY_KEYS = PROVIDER_FILTER_KEYS | PROVIDER_FILTER_DOT_ALIAS_KEYS
|
||||
|
||||
def _csv_filter_values(self, value):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
|
||||
def _validate_uuid_filter_values(self, field_name, values):
|
||||
try:
|
||||
for value in values:
|
||||
uuid.UUID(str(value))
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
raise ValidationError({field_name: ["Enter a valid UUID."]})
|
||||
|
||||
def _has_provider_filters(self, include_dot_aliases=False):
|
||||
provider_filter_keys = (
|
||||
self.PROVIDER_FILTER_QUERY_KEYS
|
||||
if include_dot_aliases
|
||||
else self.PROVIDER_FILTER_KEYS
|
||||
)
|
||||
return any(
|
||||
self.request.query_params.get(f"filter[{key}]")
|
||||
for key in provider_filter_keys
|
||||
)
|
||||
|
||||
def _extract_provider_filters_from_params(
|
||||
self,
|
||||
*,
|
||||
validate_uuids=False,
|
||||
include_dot_aliases=False,
|
||||
):
|
||||
params = self.request.query_params
|
||||
filters = {}
|
||||
valid_provider_types = {
|
||||
choice[0] for choice in Provider.ProviderChoices.choices
|
||||
}
|
||||
|
||||
provider_id = params.get("filter[provider_id]")
|
||||
if provider_id:
|
||||
if validate_uuids:
|
||||
self._validate_uuid_filter_values("provider_id", [provider_id])
|
||||
filters["provider_id"] = provider_id
|
||||
|
||||
provider_id_in = params.get("filter[provider_id__in]")
|
||||
if include_dot_aliases:
|
||||
provider_id_in = provider_id_in or params.get("filter[provider_id.in]")
|
||||
if provider_id_in:
|
||||
values = self._csv_filter_values(provider_id_in)
|
||||
if validate_uuids:
|
||||
self._validate_uuid_filter_values("provider_id__in", values)
|
||||
filters["provider_id__in"] = values
|
||||
|
||||
provider_type = params.get("filter[provider_type]")
|
||||
if provider_type:
|
||||
if provider_type not in valid_provider_types:
|
||||
raise ValidationError(
|
||||
{"provider_type": f"Invalid choice: {provider_type}"}
|
||||
)
|
||||
filters["provider__provider"] = provider_type
|
||||
|
||||
provider_type_in = params.get("filter[provider_type__in]")
|
||||
if include_dot_aliases:
|
||||
provider_type_in = provider_type_in or params.get(
|
||||
"filter[provider_type.in]"
|
||||
)
|
||||
if provider_type_in:
|
||||
values = self._csv_filter_values(provider_type_in)
|
||||
invalid = [value for value in values if value not in valid_provider_types]
|
||||
if invalid:
|
||||
raise ValidationError(
|
||||
{"provider_type__in": f"Invalid choices: {', '.join(invalid)}"}
|
||||
)
|
||||
filters["provider__provider__in"] = values
|
||||
|
||||
provider_groups = params.get("filter[provider_groups]")
|
||||
if provider_groups:
|
||||
if validate_uuids:
|
||||
self._validate_uuid_filter_values("provider_groups", [provider_groups])
|
||||
filters["provider__provider_groups__id"] = provider_groups
|
||||
|
||||
provider_groups_in = params.get("filter[provider_groups__in]")
|
||||
if include_dot_aliases:
|
||||
provider_groups_in = provider_groups_in or params.get(
|
||||
"filter[provider_groups.in]"
|
||||
)
|
||||
if provider_groups_in:
|
||||
values = self._csv_filter_values(provider_groups_in)
|
||||
if validate_uuids:
|
||||
self._validate_uuid_filter_values("provider_groups__in", values)
|
||||
filters["provider__provider_groups__id__in"] = values
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
class TaskManagementMixin:
|
||||
"""
|
||||
Mixin to manage task status checking.
|
||||
|
||||
+501
-257
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,61 @@ celery_app.conf.result_backend_transport_options = {
|
||||
}
|
||||
celery_app.conf.visibility_timeout = BROKER_VISIBILITY_TIMEOUT
|
||||
|
||||
# Durable delivery: keep the message until the task finishes, so a worker killed
|
||||
# mid-task (deploy/OOM/eviction) does not silently drop it. Reserve one task at a
|
||||
# time so a crash exposes at most one extra reserved message.
|
||||
celery_app.conf.task_acks_late = True
|
||||
celery_app.conf.task_reject_on_worker_lost = True
|
||||
celery_app.conf.worker_prefetch_multiplier = env.int(
|
||||
"DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER", default=1
|
||||
)
|
||||
# On SIGTERM, give the worker time to finish or re-queue in-flight tasks before
|
||||
# it is forcefully killed (Celery 5.5+ soft shutdown).
|
||||
celery_app.conf.worker_soft_shutdown_timeout = env.int(
|
||||
"DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT", default=60
|
||||
)
|
||||
# Bound execution so a blocked task cannot pin a worker forever. Connection
|
||||
# checks get a tight limit; scans and provider/tenant deletions can legitimately
|
||||
# run for more than a day on large tenants, so they get a much higher cap.
|
||||
# The default for every other task is set as the global limit, not as a "*"
|
||||
# annotation: Celery applies the "*" entry AFTER the per-task one, so a "*" in
|
||||
# task_annotations would silently overwrite every specific limit defined below.
|
||||
_TASK_HARD_LIMIT = env.int("DJANGO_CELERY_TASK_TIME_LIMIT", default=6 * 60 * 60)
|
||||
_TASK_SOFT_LIMIT = env.int(
|
||||
"DJANGO_CELERY_TASK_SOFT_TIME_LIMIT", default=_TASK_HARD_LIMIT - 600
|
||||
)
|
||||
_LONG_TASK_HARD_LIMIT = env.int(
|
||||
"DJANGO_CELERY_LONG_TASK_TIME_LIMIT", default=48 * 60 * 60
|
||||
)
|
||||
_LONG_TASK_SOFT_LIMIT = env.int(
|
||||
"DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT", default=_LONG_TASK_HARD_LIMIT - 600
|
||||
)
|
||||
celery_app.conf.task_time_limit = _TASK_HARD_LIMIT
|
||||
celery_app.conf.task_soft_time_limit = _TASK_SOFT_LIMIT
|
||||
celery_app.conf.task_annotations = {
|
||||
**{
|
||||
name: {"soft_time_limit": 60, "time_limit": 120}
|
||||
for name in (
|
||||
"provider-connection-check",
|
||||
"integration-connection-check",
|
||||
"lighthouse-connection-check",
|
||||
"lighthouse-provider-connection-check",
|
||||
)
|
||||
},
|
||||
**{
|
||||
name: {
|
||||
"soft_time_limit": _LONG_TASK_SOFT_LIMIT,
|
||||
"time_limit": _LONG_TASK_HARD_LIMIT,
|
||||
}
|
||||
for name in (
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"provider-deletion",
|
||||
"tenant-deletion",
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
celery_app.autodiscover_tasks(["api"])
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from datetime import timedelta
|
||||
from config.custom_logging import LOGGING # noqa
|
||||
from config.env import BASE_DIR, env # noqa
|
||||
from config.settings.celery import * # noqa
|
||||
from config.settings.eventstream import * # noqa
|
||||
from config.settings.partitions import * # noqa
|
||||
from config.settings.sentry import * # noqa
|
||||
from config.settings.social_login import * # noqa
|
||||
@@ -44,6 +45,7 @@ INSTALLED_APPS = [
|
||||
"dj_rest_auth.registration",
|
||||
"rest_framework.authtoken",
|
||||
"drf_simple_apikey",
|
||||
"django_eventstream",
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
@@ -136,6 +138,7 @@ SPECTACULAR_SETTINGS = {
|
||||
}
|
||||
|
||||
WSGI_APPLICATION = "config.wsgi.application"
|
||||
ASGI_APPLICATION = "config.asgi.application"
|
||||
|
||||
DJANGO_GUID = {
|
||||
"GUID_HEADER_NAME": "Transaction-ID",
|
||||
@@ -306,3 +309,32 @@ SESSION_COOKIE_SECURE = True
|
||||
ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
|
||||
) # 48h
|
||||
|
||||
# Orphan task recovery feature flags. The master switch is OFF by default, so task
|
||||
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
|
||||
# toggles default to enabled, so once the master is on every group recovers unless a
|
||||
# group is explicitly turned off.
|
||||
TASK_RECOVERY_ENABLED = env.bool("DJANGO_TASK_RECOVERY_ENABLED", False)
|
||||
TASK_RECOVERY_SUMMARIES_ENABLED = env.bool(
|
||||
"DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED", True
|
||||
)
|
||||
TASK_RECOVERY_DELETIONS_ENABLED = env.bool(
|
||||
"DJANGO_TASK_RECOVERY_DELETIONS_ENABLED", True
|
||||
)
|
||||
|
||||
|
||||
def label_postgres_connections(databases):
|
||||
"""Tag each Postgres connection with ``application_name="<component>:<alias>"``
|
||||
so connections are attributable by component in ``pg_stat_activity`` (and any
|
||||
tooling that surfaces ``application_name``). The component (api / worker /
|
||||
scan / ...) is injected per process by the container entrypoint via
|
||||
``DJANGO_APP_COMPONENT``; the alias distinguishes which pool inside the
|
||||
process owns the connection. The neo4j entry is skipped (not a Postgres
|
||||
backend). Postgres truncates ``application_name`` at 63 bytes.
|
||||
"""
|
||||
component = env.str("DJANGO_APP_COMPONENT", default="api")
|
||||
for alias, config in databases.items():
|
||||
engine = config.get("ENGINE", "")
|
||||
if engine.startswith("psqlextra") or "postgresql" in engine:
|
||||
name = f"{component}:{alias}"[:63]
|
||||
config.setdefault("OPTIONS", {})["application_name"] = name
|
||||
|
||||
@@ -54,6 +54,8 @@ DATABASES = {
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
label_postgres_connections(DATABASES) # noqa: F405
|
||||
|
||||
REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = tuple( # noqa: F405
|
||||
render_class
|
||||
for render_class in REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] # noqa: F405
|
||||
|
||||
@@ -58,3 +58,5 @@ DATABASES = {
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
label_postgres_connections(DATABASES) # noqa: F405
|
||||
|
||||
@@ -34,3 +34,8 @@ DRF_API_KEY = {
|
||||
# JWT
|
||||
|
||||
SIMPLE_JWT["ALGORITHM"] = "HS256" # noqa: F405
|
||||
# pyjwt >= 2.13.0 rejects an empty HMAC signing key, so HS256 tests need a real
|
||||
# key (>= 32 bytes also avoids the InsecureKeyLengthWarning). Production uses RS256.
|
||||
SIMPLE_JWT["SIGNING_KEY"] = env.str( # noqa: F405
|
||||
"DJANGO_TOKEN_SIGNING_KEY", "insecure-testing-jwt-signing-key-do-not-use-in-prod"
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import threading
|
||||
|
||||
from config.env import env
|
||||
|
||||
@@ -11,6 +12,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
|
||||
import django # noqa: E402
|
||||
|
||||
django.setup()
|
||||
from api.compliance import warm_compliance_caches # noqa: E402
|
||||
from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG # noqa: E402
|
||||
from config.custom_logging import BackendLogger # noqa: E402
|
||||
|
||||
@@ -23,6 +25,15 @@ bind = f"{BIND_ADDRESS}:{PORT}"
|
||||
workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1)
|
||||
reload = DEBUG
|
||||
|
||||
# Native ASGI worker (gunicorn 24+). Required so SSE endpoints can keep the
|
||||
# event loop alive while waiting for events.
|
||||
worker_class = env("DJANGO_WORKER_CLASS", default="asgi")
|
||||
|
||||
# Preload the application before forking workers in production: the app is
|
||||
# imported once in the master and workers fork from it. In development, disable
|
||||
# preload so the server restarts on code changes.
|
||||
preload_app = not DEBUG
|
||||
|
||||
# Logging
|
||||
logconfig_dict = DJANGO_LOGGERS
|
||||
gunicorn_logger = logging.getLogger(BackendLogger.GUNICORN)
|
||||
@@ -41,3 +52,26 @@ def on_reload(_):
|
||||
|
||||
def when_ready(_):
|
||||
gunicorn_logger.info("Gunicorn server is ready")
|
||||
|
||||
|
||||
def _warm_compliance_caches_in_background():
|
||||
"""Warm compliance caches off the request path and log the outcome."""
|
||||
failed = warm_compliance_caches()
|
||||
if failed:
|
||||
gunicorn_logger.warning("Compliance caches warmed (skipped: %s)", failed)
|
||||
else:
|
||||
gunicorn_logger.info("Compliance caches warmed")
|
||||
|
||||
|
||||
def post_fork(_server, worker):
|
||||
"""Warm compliance caches after each worker fork.
|
||||
|
||||
Warm compliance caches in a background thread so the worker becomes ready
|
||||
immediately. A request for a not-yet-warmed provider lazily loads just that
|
||||
provider, which stays well under the worker timeout.
|
||||
"""
|
||||
threading.Thread(
|
||||
target=_warm_compliance_caches_in_background,
|
||||
name="warm-compliance-caches",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Server-Sent Events (SSE) configuration.
|
||||
|
||||
Wires django-eventstream into the platform: Valkey Pub/Sub backend on a
|
||||
dedicated DB (separate from the Celery broker), the platform channel
|
||||
manager, and headers that match the existing CORS allowlist.
|
||||
"""
|
||||
|
||||
from config.env import env
|
||||
from config.settings.celery import (
|
||||
VALKEY_HOST,
|
||||
VALKEY_PASSWORD,
|
||||
VALKEY_PORT,
|
||||
VALKEY_SCHEME,
|
||||
VALKEY_USERNAME,
|
||||
)
|
||||
|
||||
# Dedicated Valkey DB for the SSE Pub/Sub bus. Kept distinct from the
|
||||
# Celery broker DB so a noisy broker can't shoulder out streaming
|
||||
# traffic on the same keyspace.
|
||||
EVENTSTREAM_VALKEY_DB = env.int("EVENTSTREAM_VALKEY_DB", default=2)
|
||||
|
||||
EVENTSTREAM_REDIS: dict = {
|
||||
"host": VALKEY_HOST,
|
||||
"port": int(VALKEY_PORT),
|
||||
"db": EVENTSTREAM_VALKEY_DB,
|
||||
}
|
||||
if VALKEY_PASSWORD:
|
||||
EVENTSTREAM_REDIS["password"] = VALKEY_PASSWORD
|
||||
if VALKEY_USERNAME:
|
||||
EVENTSTREAM_REDIS["username"] = VALKEY_USERNAME
|
||||
if VALKEY_SCHEME == "rediss":
|
||||
EVENTSTREAM_REDIS["ssl"] = True
|
||||
|
||||
# Platform channel manager — performs the per-feature authorization and
|
||||
# rewrites the placeholder channel from the URL into the canonical
|
||||
# tenant-scoped channel name. See ``api.sse.channelmanager``.
|
||||
EVENTSTREAM_CHANNELMANAGER_CLASS = "api.sse.channelmanager.SSEChannelManager"
|
||||
|
||||
# Headers a browser EventSource may legitimately send. Keep tight; the
|
||||
# stream itself reads no body, so no permissive defaults.
|
||||
EVENTSTREAM_ALLOW_HEADERS = "Cache-Control, Last-Event-ID"
|
||||
@@ -1,12 +1,14 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from celery import current_app, states
|
||||
from celery import states
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES
|
||||
from tasks.jobs.attack_paths.db_utils import (
|
||||
_mark_scan_finished,
|
||||
recover_graph_data_ready,
|
||||
)
|
||||
from tasks.jobs.orphan_recovery import is_worker_alive as _is_worker_alive
|
||||
from tasks.jobs.orphan_recovery import revoke_task as _revoke_task
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
@@ -150,32 +152,6 @@ def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _is_worker_alive(worker: str) -> bool:
|
||||
"""Ping a specific Celery worker. Returns `True` if it responds or on error."""
|
||||
try:
|
||||
response = current_app.control.inspect(destination=[worker], timeout=1.0).ping()
|
||||
return response is not None and worker in response
|
||||
except Exception:
|
||||
logger.exception(f"Failed to ping worker {worker}, treating as alive")
|
||||
return True
|
||||
|
||||
|
||||
def _revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task. Non-fatal on failure.
|
||||
|
||||
`terminate=True` SIGTERMs the worker if the task is mid-execution; use
|
||||
for EXECUTING cleanup. `terminate=False` only marks the task id revoked
|
||||
across workers, so any worker pulling the queued message discards it;
|
||||
use for SCHEDULED cleanup where the task hasn't run yet.
|
||||
"""
|
||||
try:
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
|
||||
|
||||
def _cleanup_scan(scan, task_result, reason: str) -> bool:
|
||||
"""
|
||||
Clean up a single stale `AttackPathsScan`:
|
||||
|
||||
@@ -39,11 +39,6 @@ from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS
|
||||
from prowler.lib.outputs.compliance.cisa_scuba.cisa_scuba_googleworkspace import (
|
||||
GoogleWorkspaceCISASCuBA,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.csa.csa_alibabacloud import AlibabaCloudCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_aws import AWSCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_azure import AzureCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_gcp import GCPCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_oraclecloud import OracleCloudCSA
|
||||
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
|
||||
@@ -63,6 +58,9 @@ from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import (
|
||||
AzureMitreAttack,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack
|
||||
from prowler.lib.outputs.compliance.okta_idaas_stig.okta_idaas_stig_okta import (
|
||||
OktaIDaaSSTIG,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_alibaba import (
|
||||
ProwlerThreatScoreAlibaba,
|
||||
)
|
||||
@@ -102,7 +100,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "prowler_threatscore_aws", ProwlerThreatScoreAWS),
|
||||
(lambda name: name.startswith("ccc_"), CCC_AWS),
|
||||
(lambda name: name.startswith("c5_"), AWSC5),
|
||||
(lambda name: name.startswith("csa_"), AWSCSA),
|
||||
(lambda name: name == "asd_essential_eight_aws", ASDEssentialEightAWS),
|
||||
],
|
||||
"azure": [
|
||||
@@ -113,7 +110,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("ccc_"), CCC_Azure),
|
||||
(lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure),
|
||||
(lambda name: name == "c5_azure", AzureC5),
|
||||
(lambda name: name.startswith("csa_"), AzureCSA),
|
||||
],
|
||||
"gcp": [
|
||||
(lambda name: name.startswith("cis_"), GCPCIS),
|
||||
@@ -123,7 +119,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP),
|
||||
(lambda name: name.startswith("ccc_"), CCC_GCP),
|
||||
(lambda name: name == "c5_gcp", GCPC5),
|
||||
(lambda name: name.startswith("csa_"), GCPCSA),
|
||||
],
|
||||
"kubernetes": [
|
||||
(lambda name: name.startswith("cis_"), KubernetesCIS),
|
||||
@@ -152,16 +147,17 @@ COMPLIANCE_CLASS_MAP = {
|
||||
"image": [],
|
||||
"oraclecloud": [
|
||||
(lambda name: name.startswith("cis_"), OracleCloudCIS),
|
||||
(lambda name: name.startswith("csa_"), OracleCloudCSA),
|
||||
],
|
||||
"alibabacloud": [
|
||||
(lambda name: name.startswith("cis_"), AlibabaCloudCIS),
|
||||
(lambda name: name.startswith("csa_"), AlibabaCloudCSA),
|
||||
(
|
||||
lambda name: name == "prowler_threatscore_alibabacloud",
|
||||
ProwlerThreatScoreAlibaba,
|
||||
),
|
||||
],
|
||||
"okta": [
|
||||
(lambda name: name.startswith("okta_idaas_stig"), OktaIDaaSSTIG),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Detect and recover orphaned Celery tasks.
|
||||
|
||||
A task is "orphaned" when its result row is non-terminal (STARTED/RECEIVED) but the
|
||||
worker that was running it is gone (deploy, OOM, eviction). We tell a real orphan
|
||||
from a still-running task by pinging the worker recorded on its `TaskResult`:
|
||||
|
||||
- worker responds -> the task is in flight, leave it alone (never double-run);
|
||||
- worker is gone -> real orphan: mark the stale result terminal (so pending/started
|
||||
alerts clear), then re-enqueue the task from its stored name + kwargs.
|
||||
|
||||
This recovers only allowlisted tasks with local, proven idempotency. Celery's
|
||||
`result_extended=True` gives us the stored `task_name`/`task_kwargs`/`worker` once
|
||||
the task starts, but external side-effect tasks are failed instead of blindly
|
||||
re-run. A small recovery cap stops a task that repeatedly kills its worker from
|
||||
looping forever.
|
||||
|
||||
This is the shared engine behind both the periodic Beat watchdog and the
|
||||
`reconcile_orphan_tasks` management command.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from celery import current_app, states
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db import connections
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Arbitrary constant key for pg_try_advisory_lock so only one reconciliation
|
||||
# runs at a time across replicas / the watchdog / the command.
|
||||
ORPHAN_RECOVERY_LOCK_KEY = 0x70726F77 # "prow"
|
||||
|
||||
# Non-terminal states that mean "a worker had this and may have died with it".
|
||||
IN_FLIGHT_STATES = (states.STARTED, states.RECEIVED)
|
||||
|
||||
# Tasks with proven idempotency are eligible for auto re-enqueue, grouped so each
|
||||
# group can be toggled independently by a feature flag (see config.django.base).
|
||||
# Summaries clear and rewrite their own rows and deletions are idempotent. Tasks with
|
||||
# external side effects are never eligible: integration-jira would create duplicate
|
||||
# issues, integration-s3 rebuilds its upload from worker-local files that do not
|
||||
# survive a crash, and report/Security Hub recovery is out of scope.
|
||||
RECOVERY_TASK_GROUPS = {
|
||||
"summaries": {
|
||||
"scan-summary",
|
||||
"scan-compliance-overviews",
|
||||
"scan-provider-compliance-scores",
|
||||
"scan-daily-severity",
|
||||
"scan-finding-group-summaries",
|
||||
"scan-reset-ephemeral-resources",
|
||||
},
|
||||
"deletions": {"provider-deletion", "tenant-deletion"},
|
||||
}
|
||||
|
||||
|
||||
def reenqueueable_tasks() -> set[str]:
|
||||
"""Task names eligible for auto re-enqueue, honoring the per-group feature flags.
|
||||
|
||||
A group whose flag is disabled is dropped, so its orphaned tasks are marked
|
||||
terminal instead of re-enqueued.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
group_enabled = {
|
||||
"summaries": settings.TASK_RECOVERY_SUMMARIES_ENABLED,
|
||||
"deletions": settings.TASK_RECOVERY_DELETIONS_ENABLED,
|
||||
}
|
||||
return {
|
||||
task
|
||||
for group, tasks in RECOVERY_TASK_GROUPS.items()
|
||||
if group_enabled[group]
|
||||
for task in tasks
|
||||
}
|
||||
|
||||
|
||||
# Tasks the watchdog ignores entirely (not even marked terminal): scan tasks are not
|
||||
# auto-recovered, since re-running a scan is not safe to do automatically; attack-paths
|
||||
# scans are handled by their own stale-cleanup (which also drops the temp Neo4j db);
|
||||
# and the maintenance tasks must not self-recover (they run again on their own schedule).
|
||||
_SKIP_RECOVERY = {
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"attack-paths-scan-perform",
|
||||
"attack-paths-cleanup-stale-scans",
|
||||
"reconcile-orphan-tasks",
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def advisory_lock(key: int = ORPHAN_RECOVERY_LOCK_KEY, using: str = "default"):
|
||||
"""Yield True if this session won a Postgres advisory lock, else False.
|
||||
|
||||
Non-blocking: losers get False and should no-op. The lock is released on
|
||||
exit (and implicitly if the session dies).
|
||||
"""
|
||||
with connections[using].cursor() as cursor:
|
||||
cursor.execute("SELECT pg_try_advisory_lock(%s)", [key])
|
||||
acquired = bool(cursor.fetchone()[0])
|
||||
try:
|
||||
yield acquired
|
||||
finally:
|
||||
if acquired:
|
||||
cursor.execute("SELECT pg_advisory_unlock(%s)", [key])
|
||||
|
||||
|
||||
def is_worker_alive(worker: str, timeout: float = 1.0) -> bool:
|
||||
"""Ping a specific Celery worker. Returns True if it responds, or on error.
|
||||
|
||||
Erring on the side of "alive" means an unreachable control bus never causes
|
||||
a still-running task to be re-enqueued.
|
||||
"""
|
||||
try:
|
||||
response = current_app.control.inspect(
|
||||
destination=[worker], timeout=timeout
|
||||
).ping()
|
||||
return response is not None and worker in response
|
||||
except Exception:
|
||||
logger.exception(f"Failed to ping worker {worker}, treating as alive")
|
||||
return True
|
||||
|
||||
|
||||
def revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task by its TaskResult. Non-fatal on failure.
|
||||
|
||||
terminate=True SIGTERMs the worker if the task is mid-execution; terminate=False
|
||||
only marks the id revoked so any worker pulling the queued message discards it
|
||||
(use before re-enqueuing, so a later broker redelivery of the stale message is
|
||||
dropped).
|
||||
"""
|
||||
try:
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
|
||||
|
||||
def _decode_celery_field(value, default):
|
||||
"""Decode django-celery-results' stored task_args/task_kwargs to a Python object.
|
||||
|
||||
The backend stores them as a (sometimes double-encoded) repr/JSON string. An
|
||||
empty or missing field returns ``default``; a non-empty value that cannot be
|
||||
decoded raises ``ValueError`` so the caller can avoid re-enqueuing a task with
|
||||
the wrong arguments.
|
||||
"""
|
||||
obj = value
|
||||
for _ in range(2): # values can be double-encoded (a string holding a repr)
|
||||
if not isinstance(obj, str):
|
||||
break
|
||||
text = obj.strip()
|
||||
if not text:
|
||||
return default
|
||||
parsed = None
|
||||
for parser in (ast.literal_eval, json.loads):
|
||||
try:
|
||||
parsed = parser(text)
|
||||
break
|
||||
except (ValueError, SyntaxError, TypeError):
|
||||
continue
|
||||
if parsed is None:
|
||||
raise ValueError(f"undecodable celery field: {text[:120]!r}")
|
||||
obj = parsed
|
||||
return default if obj is None else obj
|
||||
|
||||
|
||||
def reconcile_orphans(
|
||||
grace_minutes: int = 2,
|
||||
max_attempts: int = 3,
|
||||
window_hours: int = 6,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Run the full orphan sweep under a single-flight advisory lock.
|
||||
|
||||
Recovers any orphaned in-flight task and delegates attack-paths scans that
|
||||
never reached a worker to their existing stale-cleanup. Returns a summary;
|
||||
a no-op (lock not won) is reported too.
|
||||
"""
|
||||
with advisory_lock() as acquired:
|
||||
if not acquired:
|
||||
logger.info("Orphan reconcile skipped: another run holds the lock")
|
||||
return {"acquired": False}
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
if settings.TASK_RECOVERY_ENABLED:
|
||||
# Populate the task registry so we can re-enqueue any task by name.
|
||||
import tasks.tasks # noqa: F401
|
||||
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=grace_minutes,
|
||||
max_attempts=max_attempts,
|
||||
window_hours=window_hours,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
result["enabled"] = True
|
||||
else:
|
||||
logger.info("Orphan task recovery disabled by feature flag")
|
||||
result = {"recovered": [], "failed": [], "skipped": [], "enabled": False}
|
||||
|
||||
if not dry_run:
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
result["attack_paths"] = cleanup_stale_attack_paths_scans()
|
||||
|
||||
return {"acquired": True, **result}
|
||||
|
||||
|
||||
def _reconcile_task_results(
|
||||
grace_minutes: int, max_attempts: int, window_hours: int, dry_run: bool
|
||||
) -> dict:
|
||||
from django_celery_results.models import TaskResult
|
||||
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(minutes=grace_minutes)
|
||||
candidates = list(
|
||||
TaskResult.objects.filter(status__in=IN_FLIGHT_STATES, date_created__lt=cutoff)
|
||||
.exclude(worker__isnull=True)
|
||||
.exclude(worker="")
|
||||
.exclude(task_name__in=_SKIP_RECOVERY)
|
||||
)
|
||||
|
||||
# Ping each distinct worker at most once.
|
||||
worker_alive = {w: is_worker_alive(w) for w in {tr.worker for tr in candidates}}
|
||||
|
||||
recovered, failed, skipped = [], [], []
|
||||
for task_result in candidates:
|
||||
if worker_alive.get(task_result.worker, True):
|
||||
skipped.append(task_result.task_id) # in flight, do not double-run
|
||||
continue
|
||||
if dry_run:
|
||||
recovered.append(task_result.task_id)
|
||||
continue
|
||||
outcome = _recover_task(task_result, max_attempts, window_hours)
|
||||
(recovered if outcome == "recovered" else failed).append(task_result.task_id)
|
||||
|
||||
logger.info(
|
||||
"Orphan reconcile: recovered=%d failed=%d skipped(in-flight)=%d",
|
||||
len(recovered),
|
||||
len(failed),
|
||||
len(skipped),
|
||||
)
|
||||
return {"recovered": recovered, "failed": failed, "skipped": skipped}
|
||||
|
||||
|
||||
def _recovery_attempt_count(name: str, kwargs_repr, window_hours: int) -> int:
|
||||
"""Increment and return the recovery count for this (task, kwargs) within the
|
||||
window. Backed by Valkey so it survives result-row churn (a worker processing
|
||||
the revoke can blank the TaskResult fields). Fail-open if Valkey is down (the
|
||||
broker being unreachable means nothing is running anyway).
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
try:
|
||||
import redis
|
||||
|
||||
client = redis.from_url(settings.CELERY_BROKER_URL)
|
||||
signature = f"{name}|{kwargs_repr}".encode()
|
||||
key = (
|
||||
"orphan-recovery:"
|
||||
+ hashlib.sha1(signature, usedforsecurity=False).hexdigest()
|
||||
)
|
||||
count = client.incr(key)
|
||||
if count == 1:
|
||||
client.expire(key, max(1, window_hours) * 3600)
|
||||
return int(count)
|
||||
except Exception:
|
||||
logger.exception("Recovery-attempt counter unavailable; allowing recovery")
|
||||
return 1
|
||||
|
||||
|
||||
def _recover_task(task_result, max_attempts: int, window_hours: int) -> str:
|
||||
"""Recover one orphaned task. Returns 'recovered' or 'failed'."""
|
||||
# Capture name/args/kwargs now: revoking can let a worker blank the row.
|
||||
name = task_result.task_name
|
||||
args_repr = task_result.task_args
|
||||
kwargs_repr = task_result.task_kwargs
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
|
||||
# Drop any future broker redelivery of the stale message.
|
||||
revoke_task(task_result, terminate=False)
|
||||
|
||||
# Mark the stale result terminal so "pending/started forever" alerts clear.
|
||||
task_result.status = states.REVOKED
|
||||
task_result.date_done = now
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
if name not in reenqueueable_tasks():
|
||||
logger.warning(
|
||||
"Orphan %s (%s) not re-enqueued: not allowlisted for auto recovery",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
# Count the attempt only once the task is allowlisted, so a task sitting in a
|
||||
# disabled group does not burn its recovery budget while the flag is off (and is
|
||||
# not already over the cap the moment the group is re-enabled).
|
||||
attempt = _recovery_attempt_count(name, kwargs_repr, window_hours)
|
||||
if attempt > max_attempts:
|
||||
logger.warning(
|
||||
"Orphan %s (%s) not re-enqueued: recovery cap reached (%d/%d)",
|
||||
task_result.task_id,
|
||||
name,
|
||||
attempt,
|
||||
max_attempts,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
task_obj = current_app.tasks.get(name)
|
||||
if task_obj is None:
|
||||
logger.error(
|
||||
"Orphan %s: task %s not registered, cannot re-enqueue",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
try:
|
||||
args = _decode_celery_field(args_repr, [])
|
||||
kwargs = _decode_celery_field(kwargs_repr, {})
|
||||
except ValueError:
|
||||
logger.error(
|
||||
"Orphan %s (%s): could not decode stored args/kwargs, not re-enqueuing",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
new_task_id = str(uuid4())
|
||||
task_obj.apply_async(
|
||||
args=list(args) if isinstance(args, (list, tuple)) else [],
|
||||
kwargs=kwargs if isinstance(kwargs, dict) else {},
|
||||
task_id=new_task_id,
|
||||
)
|
||||
logger.info(
|
||||
"Re-enqueued orphan %s (%s) as %s", task_result.task_id, name, new_task_id
|
||||
)
|
||||
return "recovered"
|
||||
@@ -29,7 +29,10 @@ from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
@@ -571,7 +574,7 @@ def generate_csa_report(
|
||||
Args:
|
||||
tenant_id: The tenant ID for Row-Level Security context.
|
||||
scan_id: ID of the scan executed by Prowler.
|
||||
compliance_id: ID of the compliance framework (e.g., "csa_ccm_4.0_aws").
|
||||
compliance_id: ID of the compliance framework (e.g., "csa_ccm_4.0").
|
||||
output_path: Output PDF file path.
|
||||
provider_id: Provider ID for the scan.
|
||||
only_failed: If True, only include failed requirements in detailed section.
|
||||
@@ -883,9 +886,11 @@ def generate_compliance_reports(
|
||||
frameworks_bulk.get(f"nis2_{provider_type}")
|
||||
)
|
||||
if generate_csa:
|
||||
pending_checks_by_framework["csa"] = _get_compliance_check_ids(
|
||||
frameworks_bulk.get(f"csa_ccm_4.0_{provider_type}")
|
||||
)
|
||||
# csa_ccm_4.0 lives at the top level, not under compliance/{provider}/.
|
||||
csa_framework = frameworks_bulk.get(
|
||||
"csa_ccm_4.0"
|
||||
) or get_bulk_compliance_frameworks_universal(provider_type).get("csa_ccm_4.0")
|
||||
pending_checks_by_framework["csa"] = _get_compliance_check_ids(csa_framework)
|
||||
if generate_cis and latest_cis:
|
||||
pending_checks_by_framework["cis"] = _get_compliance_check_ids(
|
||||
frameworks_bulk.get(latest_cis)
|
||||
@@ -1183,7 +1188,7 @@ def generate_compliance_reports(
|
||||
if generate_csa:
|
||||
generated_report_keys.append("csa")
|
||||
csa_path = output_paths["csa"]
|
||||
compliance_id_csa = f"csa_ccm_4.0_{provider_type}"
|
||||
compliance_id_csa = "csa_ccm_4.0"
|
||||
pdf_path_csa = f"{csa_path}_csa_report.pdf"
|
||||
logger.info("Generating CSA CCM report with compliance %s", compliance_id_csa)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
@@ -26,7 +27,10 @@ from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, StatusChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
from .components import (
|
||||
@@ -222,6 +226,46 @@ def get_requirement_metadata(
|
||||
return None
|
||||
|
||||
|
||||
def _universal_attributes_to_list(attributes) -> list:
|
||||
"""Flatten a universal requirement's ``attributes`` into a list of objects
|
||||
with attribute access. MITRE wraps its list under ``_raw_attributes``."""
|
||||
if isinstance(attributes, dict) and "_raw_attributes" in attributes:
|
||||
entries = attributes.get("_raw_attributes") or []
|
||||
return [
|
||||
SimpleNamespace(**entry) for entry in entries if isinstance(entry, dict)
|
||||
]
|
||||
if isinstance(attributes, dict):
|
||||
return [SimpleNamespace(**attributes)] if attributes else []
|
||||
return list(attributes or [])
|
||||
|
||||
|
||||
def _adapt_universal_to_legacy(framework, provider_type: str) -> SimpleNamespace:
|
||||
"""Expose a universal ``ComplianceFramework`` under the legacy ``Compliance``
|
||||
attribute names used by the PDF pipeline."""
|
||||
provider_key = (provider_type or "").lower()
|
||||
requirements = []
|
||||
for requirement in framework.requirements:
|
||||
checks_by_provider = (
|
||||
requirement.checks if isinstance(requirement.checks, dict) else {}
|
||||
)
|
||||
requirements.append(
|
||||
SimpleNamespace(
|
||||
Id=requirement.id,
|
||||
Description=requirement.description or "",
|
||||
Checks=list(checks_by_provider.get(provider_key, [])),
|
||||
Attributes=_universal_attributes_to_list(requirement.attributes),
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
Framework=framework.framework,
|
||||
Name=framework.name,
|
||||
Version=framework.version or "",
|
||||
Description=framework.description or "",
|
||||
Provider=framework.provider or provider_type,
|
||||
Requirements=requirements,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PDF Styles Cache
|
||||
# =============================================================================
|
||||
@@ -869,9 +913,18 @@ class BaseComplianceReportGenerator(ABC):
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
compliance_obj = frameworks_bulk.get(compliance_id)
|
||||
# Load compliance framework — fall back to the universal loader
|
||||
# for top-level JSONs (e.g. csa_ccm_4.0) that Compliance.get_bulk
|
||||
# does not scan.
|
||||
compliance_obj = Compliance.get_bulk(provider_type).get(compliance_id)
|
||||
if not compliance_obj:
|
||||
universal_framework = get_bulk_compliance_frameworks_universal(
|
||||
provider_type
|
||||
).get(compliance_id)
|
||||
if universal_framework:
|
||||
compliance_obj = _adapt_universal_to_legacy(
|
||||
universal_framework, provider_type
|
||||
)
|
||||
|
||||
if not compliance_obj:
|
||||
raise ValueError(f"Compliance framework not found: {compliance_id}")
|
||||
|
||||
+178
-138
@@ -5,6 +5,7 @@ import re
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -22,7 +23,6 @@ from django.db.models import (
|
||||
Max,
|
||||
Min,
|
||||
OuterRef,
|
||||
Prefetch,
|
||||
Q,
|
||||
Sum,
|
||||
When,
|
||||
@@ -269,6 +269,7 @@ def _store_resources(
|
||||
provider=provider_instance,
|
||||
uid=finding.resource_uid,
|
||||
defaults={
|
||||
"name": finding.resource_name,
|
||||
"region": finding.region,
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
@@ -276,6 +277,7 @@ def _store_resources(
|
||||
)
|
||||
|
||||
if not created:
|
||||
resource_instance.name = finding.resource_name
|
||||
resource_instance.region = finding.region
|
||||
resource_instance.service = finding.service_name
|
||||
resource_instance.type = finding.resource_type
|
||||
@@ -355,68 +357,71 @@ def _copy_compliance_requirement_rows(
|
||||
|
||||
|
||||
def _persist_compliance_requirement_rows(
|
||||
tenant_id: str, rows: list[dict[str, Any]], batch_size: int = 10000
|
||||
) -> None:
|
||||
tenant_id: str, rows: Iterable[dict[str, Any]], batch_size: int = 10000
|
||||
) -> int:
|
||||
"""Persist compliance requirement rows using batched COPY with ORM fallback.
|
||||
|
||||
Splits large row sets into batches to reduce lock duration and improve concurrency.
|
||||
``rows`` is consumed lazily in batches, so peak memory stays at ~``batch_size``
|
||||
rows instead of the full set. A batch that fails COPY falls back to an ORM
|
||||
``bulk_create`` of just that batch.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID.
|
||||
rows: Precomputed row dictionaries that reflect the compliance
|
||||
overview state for a scan.
|
||||
rows: Iterable of row dictionaries reflecting the compliance overview
|
||||
state for a scan.
|
||||
batch_size: Number of rows per COPY batch (default: 10000).
|
||||
|
||||
Returns:
|
||||
int: total number of rows persisted.
|
||||
"""
|
||||
if not rows:
|
||||
return
|
||||
|
||||
total_rows = len(rows)
|
||||
total_batches = (total_rows + batch_size - 1) // batch_size
|
||||
|
||||
try:
|
||||
# Process rows in batches to reduce lock duration
|
||||
for batch_num in range(total_batches):
|
||||
start_idx = batch_num * batch_size
|
||||
end_idx = min(start_idx + batch_size, total_rows)
|
||||
batch = rows[start_idx:end_idx]
|
||||
total_rows = 0
|
||||
batch_num = 0
|
||||
|
||||
for batch, _is_last in batched(rows, batch_size):
|
||||
if not batch:
|
||||
continue
|
||||
batch_num += 1
|
||||
try:
|
||||
_copy_compliance_requirement_rows(tenant_id, batch)
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
f"COPY bulk insert for compliance requirements batch {batch_num} "
|
||||
"failed; falling back to ORM bulk_create for this batch",
|
||||
exc_info=error,
|
||||
)
|
||||
fallback_objects = [
|
||||
ComplianceRequirementOverview(
|
||||
id=row["id"],
|
||||
tenant_id=row["tenant_id"],
|
||||
inserted_at=row["inserted_at"],
|
||||
compliance_id=row["compliance_id"],
|
||||
framework=row["framework"],
|
||||
version=row["version"],
|
||||
description=row["description"],
|
||||
region=row["region"],
|
||||
requirement_id=row["requirement_id"],
|
||||
requirement_status=row["requirement_status"],
|
||||
passed_checks=row["passed_checks"],
|
||||
failed_checks=row["failed_checks"],
|
||||
total_checks=row["total_checks"],
|
||||
passed_findings=row.get("passed_findings", 0),
|
||||
total_findings=row.get("total_findings", 0),
|
||||
scan_id=row["scan_id"],
|
||||
)
|
||||
for row in batch
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.bulk_create(
|
||||
fallback_objects, batch_size=500
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Compliance COPY batch {batch_num + 1}/{total_batches}: "
|
||||
f"inserted {len(batch)} rows ({start_idx + len(batch)}/{total_rows} total)"
|
||||
)
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"COPY bulk insert for compliance requirements failed; falling back to ORM bulk_create",
|
||||
exc_info=error,
|
||||
total_rows += len(batch)
|
||||
logger.info(
|
||||
f"Compliance COPY batch {batch_num}: inserted {len(batch)} rows "
|
||||
f"({total_rows} total)"
|
||||
)
|
||||
# Fallback: use ORM bulk_create for all remaining rows
|
||||
fallback_objects = [
|
||||
ComplianceRequirementOverview(
|
||||
id=row["id"],
|
||||
tenant_id=row["tenant_id"],
|
||||
inserted_at=row["inserted_at"],
|
||||
compliance_id=row["compliance_id"],
|
||||
framework=row["framework"],
|
||||
version=row["version"],
|
||||
description=row["description"],
|
||||
region=row["region"],
|
||||
requirement_id=row["requirement_id"],
|
||||
requirement_status=row["requirement_status"],
|
||||
passed_checks=row["passed_checks"],
|
||||
failed_checks=row["failed_checks"],
|
||||
total_checks=row["total_checks"],
|
||||
passed_findings=row.get("passed_findings", 0),
|
||||
total_findings=row.get("total_findings", 0),
|
||||
scan_id=row["scan_id"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.bulk_create(
|
||||
fallback_objects, batch_size=500
|
||||
)
|
||||
|
||||
return total_rows
|
||||
|
||||
|
||||
def _create_compliance_summaries(
|
||||
@@ -476,9 +481,12 @@ def _create_compliance_summaries(
|
||||
)
|
||||
)
|
||||
|
||||
# Bulk insert summaries
|
||||
if summary_objects:
|
||||
with rls_transaction(tenant_id):
|
||||
# Idempotent re-run: clear this scan's prior summaries before re-inserting, so a
|
||||
# recovered scan-compliance-overviews run reflects its own re-derived rows instead
|
||||
# of keeping a stale one (bulk_create ignore_conflicts alone would keep the old).
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceOverviewSummary.objects.filter(scan_id=scan_id).delete()
|
||||
if summary_objects:
|
||||
ComplianceOverviewSummary.objects.bulk_create(
|
||||
summary_objects, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
@@ -701,6 +709,12 @@ def _process_finding_micro_batch(
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
if (
|
||||
finding.resource_name
|
||||
and resource_instance.name != finding.resource_name
|
||||
):
|
||||
resource_instance.name = finding.resource_name
|
||||
updated = True
|
||||
if resource_instance.service != finding.service_name:
|
||||
resource_instance.service = finding.service_name
|
||||
updated = True
|
||||
@@ -942,6 +956,7 @@ def _process_finding_micro_batch(
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_bulk_update,
|
||||
[
|
||||
"name",
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
@@ -1433,9 +1448,13 @@ def _aggregate_findings_by_region(
|
||||
tenant_id: str, scan_id: str, modeled_threatscore_compliance_id: str
|
||||
) -> tuple[dict, dict]:
|
||||
"""
|
||||
Aggregate findings by region using optimized ORM queries.
|
||||
Aggregate findings by region using streaming, column-scoped ORM reads.
|
||||
|
||||
Replaces nested Python loops with efficient queries and aggregation.
|
||||
Reads only the consumed columns as tuples via ``values_list`` and streams
|
||||
them with ``.iterator()``, using the denormalized ``resource_regions`` array
|
||||
instead of ``prefetch_related("resources")``. ``resource_regions`` mirrors the
|
||||
regions of a finding's related resources, so it yields the same per-region
|
||||
tally without joining the resource table.
|
||||
|
||||
Args:
|
||||
tenant_id: Tenant UUID
|
||||
@@ -1447,12 +1466,12 @@ def _aggregate_findings_by_region(
|
||||
- check_status_by_region: {region: {check_id: status}}
|
||||
- findings_count_by_compliance: {region: {normalized_id: {requirement_id: {total, pass}}}}
|
||||
"""
|
||||
check_status_by_region = {}
|
||||
findings_count_by_compliance = {}
|
||||
check_status_by_region: dict = {}
|
||||
findings_count_by_compliance: dict = {}
|
||||
|
||||
normalized_id = re.sub(r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower())
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Fetch only PASS/FAIL findings (optimized query reduces data transfer)
|
||||
# Other statuses are not needed for check_status or ThreatScore calculation
|
||||
findings = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
@@ -1460,42 +1479,28 @@ def _aggregate_findings_by_region(
|
||||
muted=False,
|
||||
status__in=["PASS", "FAIL"],
|
||||
)
|
||||
.only("id", "check_id", "status", "compliance")
|
||||
.prefetch_related(
|
||||
Prefetch(
|
||||
"resources",
|
||||
queryset=Resource.objects.only("id", "region"),
|
||||
to_attr="small_resources",
|
||||
)
|
||||
.values_list("check_id", "status", "resource_regions", "compliance")
|
||||
.iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE)
|
||||
)
|
||||
|
||||
for check_id, status, resource_regions, compliance in findings:
|
||||
threatscore_requirements = (compliance or {}).get(
|
||||
modeled_threatscore_compliance_id
|
||||
)
|
||||
)
|
||||
|
||||
# Process findings in a single pass (more efficient than original nested loops)
|
||||
normalized_id = re.sub(
|
||||
r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower()
|
||||
)
|
||||
|
||||
for finding in findings:
|
||||
status = finding.status
|
||||
|
||||
for resource in finding.small_resources:
|
||||
region = resource.region
|
||||
|
||||
# Aggregate check status by region
|
||||
current_status = check_status_by_region.setdefault(region, {})
|
||||
for region in resource_regions or ():
|
||||
# Priority: FAIL > any other status
|
||||
if current_status.get(finding.check_id) != "FAIL":
|
||||
current_status[finding.check_id] = status
|
||||
current_status = check_status_by_region.setdefault(region, {})
|
||||
if current_status.get(check_id) != "FAIL":
|
||||
current_status[check_id] = status
|
||||
|
||||
# Aggregate ThreatScore compliance counts
|
||||
if modeled_threatscore_compliance_id in (finding.compliance or {}):
|
||||
if threatscore_requirements:
|
||||
compliance_key = findings_count_by_compliance.setdefault(
|
||||
region, {}
|
||||
).setdefault(normalized_id, {})
|
||||
|
||||
for requirement_id in finding.compliance[
|
||||
modeled_threatscore_compliance_id
|
||||
]:
|
||||
for requirement_id in threatscore_requirements:
|
||||
requirement_stats = compliance_key.setdefault(
|
||||
requirement_id, {"total": 0, "pass": 0}
|
||||
)
|
||||
@@ -1542,8 +1547,8 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
(compliance_id, requirement_id)
|
||||
)
|
||||
|
||||
compliance_requirement_rows: list[dict[str, Any]] = []
|
||||
regions = []
|
||||
requirements_created = 0
|
||||
requirement_statuses = defaultdict(
|
||||
lambda: {"fail_count": 0, "pass_count": 0, "total_count": 0}
|
||||
)
|
||||
@@ -1583,44 +1588,93 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
else:
|
||||
requirement_stats["failed_checks"] += 1
|
||||
|
||||
# Prepare compliance requirement rows and compute summaries in single pass
|
||||
utc_datetime_now = datetime.now(tz=timezone.utc)
|
||||
|
||||
# Pre-compute shared strings (optimization: reduces string conversions)
|
||||
tenant_id_str = str(tenant_id)
|
||||
scan_id_str = str(scan_instance.id)
|
||||
|
||||
for region in regions:
|
||||
region_stats = region_requirement_stats.get(region, {})
|
||||
for compliance_id, compliance in compliance_template.items():
|
||||
modeled_compliance_id = _normalized_compliance_key(
|
||||
compliance["framework"], compliance["version"]
|
||||
# Per-framework constants that don't depend on the region.
|
||||
compliance_plan = []
|
||||
for compliance_id, compliance in compliance_template.items():
|
||||
modeled_compliance_id = _normalized_compliance_key(
|
||||
compliance["framework"], compliance["version"]
|
||||
)
|
||||
framework = compliance["framework"]
|
||||
version = compliance["version"] or ""
|
||||
requirements = [
|
||||
(
|
||||
requirement_id,
|
||||
requirement.get("description") or "",
|
||||
len(requirement["checks"]),
|
||||
)
|
||||
compliance_stats = region_stats.get(compliance_id, {})
|
||||
# Create an overview record for each requirement within each compliance framework
|
||||
for requirement_id, requirement in compliance[
|
||||
"requirements"
|
||||
].items():
|
||||
stats = compliance_stats.get(requirement_id)
|
||||
passed_checks = stats["passed_checks"] if stats else 0
|
||||
failed_checks = stats["failed_checks"] if stats else 0
|
||||
total_checks = len(requirement["checks"])
|
||||
if total_checks == 0:
|
||||
requirement_status = "MANUAL"
|
||||
elif failed_checks > 0:
|
||||
requirement_status = "FAIL"
|
||||
else:
|
||||
requirement_status = "PASS"
|
||||
].items()
|
||||
]
|
||||
compliance_plan.append(
|
||||
(
|
||||
compliance_id,
|
||||
framework,
|
||||
version,
|
||||
modeled_compliance_id,
|
||||
requirements,
|
||||
)
|
||||
)
|
||||
|
||||
compliance_requirement_rows.append(
|
||||
{
|
||||
# Yield rows lazily (consumed batch-by-batch by COPY) so peak memory
|
||||
# stays bounded; tally requirement_statuses in the same pass.
|
||||
def _iter_compliance_requirement_rows():
|
||||
for region in regions:
|
||||
region_stats = region_requirement_stats.get(region, {})
|
||||
region_findings = findings_count_by_compliance.get(region, {})
|
||||
for (
|
||||
compliance_id,
|
||||
framework,
|
||||
version,
|
||||
modeled_compliance_id,
|
||||
requirements,
|
||||
) in compliance_plan:
|
||||
compliance_stats = region_stats.get(compliance_id, {})
|
||||
compliance_findings = region_findings.get(
|
||||
modeled_compliance_id, {}
|
||||
)
|
||||
for requirement_id, description, total_checks in requirements:
|
||||
stats = compliance_stats.get(requirement_id)
|
||||
if stats:
|
||||
passed_checks = stats["passed_checks"]
|
||||
failed_checks = stats["failed_checks"]
|
||||
else:
|
||||
passed_checks = 0
|
||||
failed_checks = 0
|
||||
if total_checks == 0:
|
||||
requirement_status = "MANUAL"
|
||||
elif failed_checks > 0:
|
||||
requirement_status = "FAIL"
|
||||
else:
|
||||
requirement_status = "PASS"
|
||||
|
||||
finding_counts = compliance_findings.get(requirement_id)
|
||||
if finding_counts:
|
||||
passed_findings = finding_counts.get("pass", 0)
|
||||
total_findings = finding_counts.get("total", 0)
|
||||
else:
|
||||
passed_findings = 0
|
||||
total_findings = 0
|
||||
|
||||
key = (compliance_id, requirement_id)
|
||||
requirement_statuses[key]["total_count"] += 1
|
||||
if requirement_status == "FAIL":
|
||||
requirement_statuses[key]["fail_count"] += 1
|
||||
elif requirement_status == "PASS":
|
||||
requirement_statuses[key]["pass_count"] += 1
|
||||
|
||||
yield {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id_str,
|
||||
"inserted_at": utc_datetime_now,
|
||||
"compliance_id": compliance_id,
|
||||
"framework": compliance["framework"],
|
||||
"version": compliance["version"] or "",
|
||||
"description": requirement.get("description") or "",
|
||||
"framework": framework,
|
||||
"version": version,
|
||||
"description": description,
|
||||
"region": region,
|
||||
"requirement_id": requirement_id,
|
||||
"requirement_status": requirement_status,
|
||||
@@ -1628,37 +1682,23 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
"failed_checks": failed_checks,
|
||||
"total_checks": total_checks,
|
||||
"scan_id": scan_id_str,
|
||||
"passed_findings": findings_count_by_compliance.get(
|
||||
region, {}
|
||||
)
|
||||
.get(modeled_compliance_id, {})
|
||||
.get(requirement_id, {})
|
||||
.get("pass", 0),
|
||||
"total_findings": findings_count_by_compliance.get(
|
||||
region, {}
|
||||
)
|
||||
.get(modeled_compliance_id, {})
|
||||
.get(requirement_id, {})
|
||||
.get("total", 0),
|
||||
"passed_findings": passed_findings,
|
||||
"total_findings": total_findings,
|
||||
}
|
||||
)
|
||||
|
||||
# Update summary tracking (single-pass optimization)
|
||||
key = (compliance_id, requirement_id)
|
||||
requirement_statuses[key]["total_count"] += 1
|
||||
if requirement_status == "FAIL":
|
||||
requirement_statuses[key]["fail_count"] += 1
|
||||
elif requirement_status == "PASS":
|
||||
requirement_statuses[key]["pass_count"] += 1
|
||||
# Idempotent re-run: clear this scan's rows before re-inserting.
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete()
|
||||
|
||||
# Bulk create requirement records using PostgreSQL COPY
|
||||
_persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows)
|
||||
requirements_created = _persist_compliance_requirement_rows(
|
||||
tenant_id, _iter_compliance_requirement_rows()
|
||||
)
|
||||
|
||||
# Create pre-aggregated summaries for fast compliance overview lookups
|
||||
_create_compliance_summaries(tenant_id, scan_id, requirement_statuses)
|
||||
|
||||
return {
|
||||
"requirements_created": len(compliance_requirement_rows),
|
||||
"requirements_created": requirements_created,
|
||||
"regions_processed": list(regions),
|
||||
"compliance_frameworks": (
|
||||
list(compliance_template.keys()) if regions else []
|
||||
|
||||
@@ -359,35 +359,40 @@ def _load_findings_for_requirement_checks(
|
||||
def _get_compliance_check_ids(compliance_obj) -> set[str]:
|
||||
"""Return the union of all check_ids referenced by a compliance framework.
|
||||
|
||||
Used by the master report orchestrator to know which checks each
|
||||
framework consumes from the shared ``findings_cache``, so that once a
|
||||
framework finishes the entries no other pending framework needs can be
|
||||
evicted from the cache (PROWLER-1733).
|
||||
Used by the master report orchestrator to evict entries from
|
||||
``findings_cache`` once no pending framework needs them (PROWLER-1733).
|
||||
|
||||
Args:
|
||||
compliance_obj: A loaded Compliance framework object exposing a
|
||||
``Requirements`` iterable, each requirement carrying ``Checks``.
|
||||
``None`` is treated as "no checks" rather than raising, so the
|
||||
caller can pass ``frameworks_bulk.get(...)`` directly without
|
||||
an extra existence check.
|
||||
|
||||
Returns:
|
||||
Set of check_id strings (empty if ``compliance_obj`` is ``None``).
|
||||
Accepts the legacy ``Compliance`` shape (``Requirements`` / ``Checks``
|
||||
lists) and the universal ``ComplianceFramework`` shape (``requirements``
|
||||
/ ``checks`` dict keyed by provider). ``None`` returns an empty set so
|
||||
callers can pass ``frameworks_bulk.get(...)`` directly.
|
||||
"""
|
||||
if compliance_obj is None:
|
||||
return set()
|
||||
checks: set[str] = set()
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or []
|
||||
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or getattr(
|
||||
compliance_obj, "requirements", None
|
||||
)
|
||||
if not requirements:
|
||||
return set()
|
||||
|
||||
check_ids: set[str] = set()
|
||||
try:
|
||||
# Defensive: Mock objects (used in unit tests) return another Mock
|
||||
# for any attribute access, which is truthy but not iterable. Treat
|
||||
# any non-iterable Requirements value as "no checks".
|
||||
for req in requirements:
|
||||
req_checks = getattr(req, "Checks", None) or []
|
||||
# Mock objects in unit tests return another Mock for any attribute
|
||||
# access — truthy but not iterable. Treat that as "no checks".
|
||||
for requirement in requirements:
|
||||
requirement_checks = getattr(requirement, "Checks", None)
|
||||
if requirement_checks is None:
|
||||
checks_by_provider = getattr(requirement, "checks", None) or {}
|
||||
requirement_checks = [
|
||||
check_id
|
||||
for check_ids_list in checks_by_provider.values()
|
||||
for check_id in check_ids_list
|
||||
]
|
||||
try:
|
||||
checks.update(req_checks)
|
||||
check_ids.update(requirement_checks)
|
||||
except TypeError:
|
||||
continue
|
||||
except TypeError:
|
||||
return set()
|
||||
return checks
|
||||
return check_ids
|
||||
|
||||
@@ -46,6 +46,7 @@ from tasks.jobs.lighthouse_providers import (
|
||||
refresh_lighthouse_provider_models,
|
||||
)
|
||||
from tasks.jobs.muting import mute_historical_findings
|
||||
from tasks.jobs.orphan_recovery import reconcile_orphans
|
||||
from tasks.jobs.report import (
|
||||
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
|
||||
_cleanup_stale_tmp_output_directories,
|
||||
@@ -67,7 +68,10 @@ from tasks.utils import (
|
||||
get_next_execution_datetime,
|
||||
)
|
||||
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.compliance import (
|
||||
get_compliance_frameworks,
|
||||
get_prowler_provider_compliance,
|
||||
)
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import delete_related_daily_task, rls_transaction
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
@@ -75,6 +79,9 @@ from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateC
|
||||
from api.utils import initialize_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.compliance.compliance import (
|
||||
process_universal_compliance_frameworks,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
@@ -253,7 +260,9 @@ def delete_provider_task(provider_id: str, tenant_id: str):
|
||||
return delete_provider(tenant_id=tenant_id, pk=provider_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-perform", queue="scans")
|
||||
# acks_late=False: a re-run would duplicate findings and the task is not auto-recovered,
|
||||
# so a crashed scan is dropped rather than redelivered by the broker (as before #11416).
|
||||
@shared_task(base=RLSTask, name="scan-perform", queue="scans", acks_late=False)
|
||||
@handle_provider_deletion
|
||||
def perform_scan_task(
|
||||
tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None
|
||||
@@ -297,7 +306,14 @@ def perform_scan_task(
|
||||
return result
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, bind=True, name="scan-perform-scheduled", queue="scans")
|
||||
# acks_late=False: like scan-perform; a dropped run is re-fired by Beat on the next tick.
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
bind=True,
|
||||
name="scan-perform-scheduled",
|
||||
queue="scans",
|
||||
acks_late=False,
|
||||
)
|
||||
@handle_provider_deletion
|
||||
def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
"""
|
||||
@@ -462,13 +478,42 @@ def cleanup_stale_attack_paths_scans_task():
|
||||
return cleanup_stale_attack_paths_scans()
|
||||
|
||||
|
||||
@shared_task(name="reconcile-orphan-tasks", queue="celery")
|
||||
def reconcile_orphan_tasks_task():
|
||||
"""Periodic watchdog: recover tasks whose worker is gone (deploys, crashes)."""
|
||||
return reconcile_orphans()
|
||||
|
||||
|
||||
@shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,))
|
||||
def delete_tenant_task(tenant_id: str):
|
||||
return delete_tenant(pk=tenant_id)
|
||||
|
||||
|
||||
def _scan_tmp_output_directory(tenant_id: str, scan_id: str) -> Path:
|
||||
"""Root tmp output directory for a scan ({tmp}/{tenant_id}/{scan_id})."""
|
||||
return Path(DJANGO_TMP_OUTPUT_DIRECTORY) / str(tenant_id) / str(scan_id)
|
||||
|
||||
|
||||
class ScanReportRLSTask(RLSTask):
|
||||
"""
|
||||
RLS task that removes the scan's tmp output directory when the task fails.
|
||||
|
||||
Covers failures both inside and outside the task body (e.g. ENOSPC mid-write,
|
||||
or setup errors) so partial artifacts do not accumulate on the worker disk.
|
||||
"""
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002
|
||||
del args # Required by Celery's Task.on_failure signature; not used.
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
scan_id = kwargs.get("scan_id")
|
||||
|
||||
if tenant_id and scan_id:
|
||||
logger.error(f"Scan report task {task_id} failed: {exc}")
|
||||
rmtree(_scan_tmp_output_directory(tenant_id, scan_id), ignore_errors=True)
|
||||
|
||||
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
base=ScanReportRLSTask,
|
||||
name="scan-report",
|
||||
queue="scan-reports",
|
||||
)
|
||||
@@ -513,11 +558,23 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
provider_uid = provider_obj.uid
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Per-framework exporters in `COMPLIANCE_CLASS_MAP` consume the legacy bulk.
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
# Universal-only frameworks (top-level JSONs like `dora.json`) are emitted
|
||||
# via `process_universal_compliance_frameworks` below.
|
||||
universal_bulk = get_prowler_provider_compliance(provider_type)
|
||||
universal_only_names = {
|
||||
name
|
||||
for name in universal_bulk
|
||||
if name not in frameworks_bulk and universal_bulk[name].outputs
|
||||
}
|
||||
frameworks_avail = get_compliance_frameworks(provider_type)
|
||||
out_dir, comp_dir = _generate_output_directory(
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
|
||||
)
|
||||
# Removed on success here and on failure by ScanReportRLSTask.on_failure,
|
||||
# so partial artifacts do not accumulate and fill the disk (ENOSPC).
|
||||
scan_tmp_dir = _scan_tmp_output_directory(tenant_id, scan_id)
|
||||
|
||||
def get_writer(writer_map, name, factory, is_last):
|
||||
"""
|
||||
@@ -535,6 +592,10 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
|
||||
output_writers = {}
|
||||
compliance_writers = {}
|
||||
# Shared across batches so universal writers are created once and reused.
|
||||
universal_compliance_state: dict[str, list] = {"compliance": []}
|
||||
universal_base_dir = os.path.dirname(out_dir)
|
||||
universal_output_filename = os.path.basename(out_dir)
|
||||
|
||||
scan_summary = FindingOutput._transform_findings_stats(
|
||||
ScanSummary.objects.filter(scan_id=scan_id)
|
||||
@@ -589,8 +650,30 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
writer.batch_write_data_to_file(**extra)
|
||||
writer._data.clear()
|
||||
|
||||
# Compliance CSVs
|
||||
# Universal-only frameworks (e.g. `dora.json`).
|
||||
if universal_only_names:
|
||||
process_universal_compliance_frameworks(
|
||||
input_compliance_frameworks=universal_only_names,
|
||||
universal_frameworks=universal_bulk,
|
||||
finding_outputs=fos,
|
||||
output_directory=universal_base_dir,
|
||||
output_filename=universal_output_filename,
|
||||
provider=provider_type,
|
||||
generated_outputs=universal_compliance_state,
|
||||
from_cli=False,
|
||||
is_last=is_last,
|
||||
)
|
||||
|
||||
# Compliance CSVs (per-framework exporters).
|
||||
for name in frameworks_avail:
|
||||
if name in universal_only_names:
|
||||
continue
|
||||
if name not in frameworks_bulk:
|
||||
logger.warning(
|
||||
"Compliance framework '%s' missing from bulk; skipping CSV export",
|
||||
name,
|
||||
)
|
||||
continue
|
||||
compliance_obj = frameworks_bulk[name]
|
||||
|
||||
klass = GenericCompliance
|
||||
@@ -666,7 +749,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
# TODO: We need to create a new periodic task to delete the output files
|
||||
# This task shouldn't be responsible for deleting the output files
|
||||
try:
|
||||
rmtree(Path(compressed).parent, ignore_errors=True)
|
||||
rmtree(scan_tmp_dir, ignore_errors=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting output files: {e}")
|
||||
final_location, did_upload = upload_uri, True
|
||||
@@ -1077,10 +1160,13 @@ def security_hub_integration_task(
|
||||
return upload_security_hub_integration(tenant_id, provider_id, scan_id)
|
||||
|
||||
|
||||
# acks_late=False: Jira sends are not deduplicated and the task is not auto-recovered,
|
||||
# so a crashed send is dropped rather than redelivered (avoids duplicate Jira issues).
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
name="integration-jira",
|
||||
queue="integrations",
|
||||
acks_late=False,
|
||||
)
|
||||
def jira_integration_task(
|
||||
tenant_id: str,
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from celery import states
|
||||
from django.test import override_settings
|
||||
from django_celery_results.models import TaskResult
|
||||
|
||||
from tasks.jobs.orphan_recovery import (
|
||||
_decode_celery_field,
|
||||
_reconcile_task_results,
|
||||
_recovery_attempt_count,
|
||||
advisory_lock,
|
||||
is_worker_alive,
|
||||
reconcile_orphans,
|
||||
reenqueueable_tasks,
|
||||
)
|
||||
|
||||
|
||||
def _orphan_result(*, name, kwargs, worker, created_minutes_ago, status=states.STARTED):
|
||||
"""Create a TaskResult mimicking an in-flight task, backdated past the grace."""
|
||||
tr = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
status=status,
|
||||
task_name=name,
|
||||
worker=worker,
|
||||
task_kwargs=repr(kwargs),
|
||||
task_args=repr([]),
|
||||
)
|
||||
TaskResult.objects.filter(pk=tr.pk).update(
|
||||
date_created=datetime.now(tz=timezone.utc)
|
||||
- timedelta(minutes=created_minutes_ago)
|
||||
)
|
||||
tr.refresh_from_db()
|
||||
return tr
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestDecodeCeleryField:
|
||||
def test_decodes_single_encoded_repr(self):
|
||||
assert _decode_celery_field("{'tenant_id': 'abc'}", {}) == {"tenant_id": "abc"}
|
||||
|
||||
def test_decodes_double_encoded(self):
|
||||
import json
|
||||
|
||||
stored = json.dumps(repr({"tenant_id": "abc", "scan_id": "s1"}))
|
||||
assert _decode_celery_field(stored, {}) == {"tenant_id": "abc", "scan_id": "s1"}
|
||||
|
||||
def test_empty_returns_default(self):
|
||||
assert _decode_celery_field(None, {}) == {}
|
||||
assert _decode_celery_field("", []) == []
|
||||
|
||||
def test_unparseable_raises(self):
|
||||
with pytest.raises(ValueError):
|
||||
_decode_celery_field("<<not a literal>>", {})
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestReconcileTaskResults:
|
||||
def _patches(self, alive):
|
||||
"""Patch worker liveness, revoke, and the task registry for re-enqueue."""
|
||||
mock_app = MagicMock()
|
||||
mock_task = MagicMock()
|
||||
mock_app.tasks.get.return_value = mock_task
|
||||
return (
|
||||
patch("tasks.jobs.orphan_recovery.is_worker_alive", return_value=alive),
|
||||
patch("tasks.jobs.orphan_recovery.revoke_task"),
|
||||
patch("tasks.jobs.orphan_recovery.current_app", mock_app),
|
||||
mock_task,
|
||||
)
|
||||
|
||||
def test_recovers_non_scan_task(self, tenants_fixture):
|
||||
"""A NON-scan task (tenant-deletion) left orphaned is re-enqueued too."""
|
||||
tenant = tenants_fixture[0]
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenant.id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["recovered"]
|
||||
tr.refresh_from_db()
|
||||
assert tr.status == states.REVOKED # stale result cleared (no pending alert)
|
||||
mock_task.apply_async.assert_called_once()
|
||||
call = mock_task.apply_async.call_args.kwargs
|
||||
assert call["kwargs"] == {"tenant_id": str(tenant.id)}
|
||||
assert call["task_id"] != tr.task_id # fresh task id
|
||||
|
||||
def test_external_integration_task_is_not_reenqueued_by_default(
|
||||
self, tenants_fixture
|
||||
):
|
||||
"""External side-effect tasks without proven idempotency stay terminal.
|
||||
|
||||
integration-s3 rebuilds its upload from worker-local files that do not
|
||||
survive the crash, so re-enqueuing it would upload nothing.
|
||||
"""
|
||||
tr = _orphan_result(
|
||||
name="integration-s3",
|
||||
kwargs={
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
"provider_id": str(uuid4()),
|
||||
"output_directory": "/tmp/gone",
|
||||
},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@override_settings(TASK_RECOVERY_SUMMARIES_ENABLED=False)
|
||||
def test_disabled_group_task_is_not_reenqueued(self, tenants_fixture):
|
||||
"""A task whose group feature flag is off stays terminal, not re-enqueued."""
|
||||
tr = _orphan_result(
|
||||
name="scan-summary",
|
||||
kwargs={
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
"scan_id": str(uuid4()),
|
||||
},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@override_settings(TASK_RECOVERY_SUMMARIES_ENABLED=False)
|
||||
def test_disabled_group_task_does_not_consume_recovery_attempt(
|
||||
self, tenants_fixture
|
||||
):
|
||||
"""A disabled-group task is failed without incrementing its Valkey attempt
|
||||
counter, so re-enabling the group does not start it at the cap."""
|
||||
tr = _orphan_result(
|
||||
name="scan-summary",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id), "scan_id": str(uuid4())},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count") as mock_count,
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_count.assert_not_called()
|
||||
|
||||
def test_scan_task_is_skipped_entirely(self, tenants_fixture):
|
||||
"""Scan tasks are excluded from recovery: the watchdog never touches them."""
|
||||
tr = _orphan_result(
|
||||
name="scan-perform",
|
||||
kwargs={
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
"scan_id": str(uuid4()),
|
||||
},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with p_alive, p_revoke, p_app:
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id not in result["recovered"]
|
||||
assert tr.task_id not in result["failed"]
|
||||
assert tr.task_id not in result["skipped"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_jira_integration_task_is_not_reenqueued(self, tenants_fixture):
|
||||
"""integration-jira stays terminal: re-running it would create duplicate Jira
|
||||
issues, so an orphaned send is failed instead of re-enqueued."""
|
||||
tenant = tenants_fixture[0]
|
||||
kwargs = {
|
||||
"tenant_id": str(tenant.id),
|
||||
"integration_id": str(uuid4()),
|
||||
"project_key": "PROWLER",
|
||||
"issue_type": "Task",
|
||||
"finding_ids": [str(uuid4()), str(uuid4())],
|
||||
}
|
||||
tr = _orphan_result(
|
||||
name="integration-jira",
|
||||
kwargs=kwargs,
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
tr.refresh_from_db()
|
||||
assert tr.status == states.REVOKED # stale result cleared (no pending alert)
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_skips_live_worker(self, tenants_fixture):
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="alive@host",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=True)
|
||||
with p_alive, p_revoke, p_app:
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["skipped"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_skips_recently_created(self, tenants_fixture):
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=0,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with p_alive, p_revoke, p_app:
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
# too recent: excluded by the grace window (not even a candidate)
|
||||
assert tr.task_id not in result["recovered"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_denylisted_task_failed_not_reenqueued(self, tenants_fixture):
|
||||
"""A non-allowlisted task is failed, never blind re-run."""
|
||||
tr = _orphan_result(
|
||||
name="some-non-idempotent-task",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
tr.refresh_from_db()
|
||||
assert tr.status == states.REVOKED
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_recovery_cap_marks_failed(self, tenants_fixture):
|
||||
"""When the recovery counter exceeds the cap, the task is failed not re-run."""
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=4),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestOrphanRecoveryHelpers:
|
||||
def test_advisory_lock_acquires_and_releases(self):
|
||||
with advisory_lock() as acquired:
|
||||
assert acquired is True
|
||||
|
||||
def test_is_worker_alive_true_when_responds(self):
|
||||
inspect = MagicMock()
|
||||
inspect.ping.return_value = {"w@h": {"ok": "pong"}}
|
||||
with patch(
|
||||
"tasks.jobs.orphan_recovery.current_app.control.inspect",
|
||||
return_value=inspect,
|
||||
):
|
||||
assert is_worker_alive("w@h") is True
|
||||
|
||||
def test_is_worker_alive_false_when_silent(self):
|
||||
inspect = MagicMock()
|
||||
inspect.ping.return_value = None
|
||||
with patch(
|
||||
"tasks.jobs.orphan_recovery.current_app.control.inspect",
|
||||
return_value=inspect,
|
||||
):
|
||||
assert is_worker_alive("w@h") is False
|
||||
|
||||
def test_recovery_attempt_count_increments(self):
|
||||
# Unique signature so the Valkey counter starts fresh for this test.
|
||||
kwargs_repr = repr({"probe": str(uuid4())})
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.side_effect = [1, 2]
|
||||
with patch("redis.from_url", return_value=redis_client):
|
||||
assert _recovery_attempt_count("probe-task", kwargs_repr, 6) == 1
|
||||
assert _recovery_attempt_count("probe-task", kwargs_repr, 6) == 2
|
||||
|
||||
|
||||
class TestRecoveryFeatureFlags:
|
||||
def test_all_groups_enabled_by_default(self):
|
||||
tasks = reenqueueable_tasks()
|
||||
assert "scan-summary" in tasks
|
||||
assert {"provider-deletion", "tenant-deletion"} <= tasks
|
||||
|
||||
@override_settings(TASK_RECOVERY_SUMMARIES_ENABLED=False)
|
||||
def test_summaries_group_flag_excludes_summary_tasks(self):
|
||||
tasks = reenqueueable_tasks()
|
||||
assert "scan-summary" not in tasks
|
||||
assert "scan-compliance-overviews" not in tasks
|
||||
assert "provider-deletion" in tasks
|
||||
|
||||
@override_settings(TASK_RECOVERY_DELETIONS_ENABLED=False)
|
||||
def test_deletions_group_flag_excludes_deletion_tasks(self):
|
||||
tasks = reenqueueable_tasks()
|
||||
assert "provider-deletion" not in tasks
|
||||
assert "tenant-deletion" not in tasks
|
||||
assert "scan-summary" in tasks
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestRecoveryMasterFlag:
|
||||
@override_settings(TASK_RECOVERY_ENABLED=False)
|
||||
def test_master_flag_disables_task_recovery(self):
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.orphan_recovery._reconcile_task_results"
|
||||
) as mock_reconcile,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.cleanup.cleanup_stale_attack_paths_scans",
|
||||
return_value={},
|
||||
),
|
||||
):
|
||||
result = reconcile_orphans(grace_minutes=2, max_attempts=3, dry_run=False)
|
||||
|
||||
mock_reconcile.assert_not_called()
|
||||
assert result["acquired"] is True
|
||||
assert result["enabled"] is False
|
||||
|
||||
@override_settings(TASK_RECOVERY_ENABLED=True)
|
||||
def test_master_flag_enabled_runs_task_recovery(self):
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.orphan_recovery._reconcile_task_results",
|
||||
return_value={"recovered": [], "failed": [], "skipped": []},
|
||||
) as mock_reconcile,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.cleanup.cleanup_stale_attack_paths_scans",
|
||||
return_value={},
|
||||
),
|
||||
):
|
||||
reconcile_orphans(grace_minutes=2, max_attempts=3, dry_run=False)
|
||||
|
||||
mock_reconcile.assert_called_once()
|
||||
@@ -80,7 +80,7 @@ def basic_csa_compliance_data():
|
||||
tenant_id="tenant-123",
|
||||
scan_id="scan-456",
|
||||
provider_id="provider-789",
|
||||
compliance_id="csa_ccm_4.0_aws",
|
||||
compliance_id="csa_ccm_4.0",
|
||||
framework="CSA-CCM",
|
||||
name="CSA Cloud Controls Matrix v4.0",
|
||||
version="4.0",
|
||||
|
||||
@@ -315,6 +315,7 @@ class TestPerformScan:
|
||||
provider=provider_instance,
|
||||
uid=finding.resource_uid,
|
||||
defaults={
|
||||
"name": finding.resource_name,
|
||||
"region": finding.region,
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
@@ -348,6 +349,7 @@ class TestPerformScan:
|
||||
|
||||
resource_instance = MagicMock()
|
||||
resource_instance.uid = finding.resource_uid
|
||||
resource_instance.name = "old_name"
|
||||
resource_instance.region = "us-west-1"
|
||||
resource_instance.service = "old_service"
|
||||
resource_instance.type = "old_type"
|
||||
@@ -366,6 +368,7 @@ class TestPerformScan:
|
||||
provider=provider_instance,
|
||||
uid=finding.resource_uid,
|
||||
defaults={
|
||||
"name": finding.resource_name,
|
||||
"region": finding.region,
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
@@ -373,6 +376,7 @@ class TestPerformScan:
|
||||
)
|
||||
|
||||
# Check that resource fields were updated
|
||||
assert resource_instance.name == finding.resource_name
|
||||
assert resource_instance.region == finding.region
|
||||
assert resource_instance.service == finding.service_name
|
||||
assert resource_instance.type == finding.resource_type
|
||||
@@ -1565,6 +1569,75 @@ class TestProcessFindingMicroBatch:
|
||||
assert resource_cache[finding.resource_uid].service == finding.service_name
|
||||
assert tag_cache.keys() == {("team", "devsec")}
|
||||
|
||||
def test_process_finding_micro_batch_refreshes_empty_resource_name(
|
||||
self, tenants_fixture, scans_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
provider = scan.provider
|
||||
|
||||
# Old resource stored before names were persisted: empty name.
|
||||
existing_resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid="arn:aws:s3:::my-bucket",
|
||||
name="",
|
||||
region="us-east-1",
|
||||
service="s3",
|
||||
type="bucket",
|
||||
)
|
||||
|
||||
finding = FakeFinding(
|
||||
uid="finding-empty-name",
|
||||
status=StatusChoices.PASS,
|
||||
status_extended="passing",
|
||||
severity=Severity.low,
|
||||
check_id="s3_bucket_public_access",
|
||||
resource_uid=existing_resource.uid,
|
||||
resource_name="my-bucket",
|
||||
region="us-east-1",
|
||||
service_name="s3",
|
||||
resource_type="bucket",
|
||||
partition="aws",
|
||||
raw={"status": "PASS"},
|
||||
metadata={"source": "prowler"},
|
||||
)
|
||||
|
||||
resource_cache = {existing_resource.uid: existing_resource}
|
||||
tag_cache = {}
|
||||
last_status_cache = {}
|
||||
resource_failed_findings_cache = {existing_resource.uid: 0}
|
||||
unique_resources: set[tuple[str, str]] = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
mute_rules_cache = {}
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
group_resources_cache: dict[str, set] = {}
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
|
||||
patch("api.db_utils.rls_transaction", new=noop_rls_transaction),
|
||||
):
|
||||
_process_finding_micro_batch(
|
||||
str(tenant.id),
|
||||
[finding],
|
||||
scan,
|
||||
provider,
|
||||
resource_cache,
|
||||
tag_cache,
|
||||
last_status_cache,
|
||||
resource_failed_findings_cache,
|
||||
unique_resources,
|
||||
scan_resource_cache,
|
||||
mute_rules_cache,
|
||||
scan_categories_cache,
|
||||
scan_resource_groups_cache,
|
||||
group_resources_cache,
|
||||
)
|
||||
|
||||
existing_resource.refresh_from_db()
|
||||
assert existing_resource.name == finding.resource_name
|
||||
|
||||
def test_process_finding_micro_batch_skips_long_uid(
|
||||
self, tenants_fixture, scans_fixture
|
||||
):
|
||||
@@ -1880,6 +1953,62 @@ class TestCreateComplianceRequirements:
|
||||
|
||||
assert "requirements_created" in result
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_create_compliance_requirements_idempotent_on_rerun(
|
||||
self,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
providers_fixture,
|
||||
findings_fixture,
|
||||
):
|
||||
"""Re-running compliance materialization must not raise nor duplicate rows.
|
||||
|
||||
Uses transaction=True because the COPY path commits on its own connection,
|
||||
so the test must use real commits (mirroring production) rather than the
|
||||
default rollback wrapper.
|
||||
"""
|
||||
from api.models import ComplianceRequirementOverview
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE"
|
||||
) as mock_compliance_template:
|
||||
tenant_id = str(tenants_fixture[0].id)
|
||||
scan_id = str(scans_fixture[0].id)
|
||||
|
||||
mock_compliance_template.__getitem__.return_value = {
|
||||
"test_compliance": {
|
||||
"framework": "Test Framework",
|
||||
"version": "1.0",
|
||||
"requirements": {
|
||||
"req_1": {
|
||||
"description": "Test Requirement 1",
|
||||
"checks": {"test_check_id": None},
|
||||
"checks_status": {
|
||||
"pass": 2,
|
||||
"fail": 1,
|
||||
"manual": 0,
|
||||
"total": 3,
|
||||
},
|
||||
"status": "FAIL",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
create_compliance_requirements(tenant_id, scan_id)
|
||||
count_after_first = ComplianceRequirementOverview.objects.filter(
|
||||
scan_id=scan_id
|
||||
).count()
|
||||
|
||||
# Second run must not raise and must not duplicate rows.
|
||||
create_compliance_requirements(tenant_id, scan_id)
|
||||
count_after_second = ComplianceRequirementOverview.objects.filter(
|
||||
scan_id=scan_id
|
||||
).count()
|
||||
|
||||
assert count_after_first > 0
|
||||
assert count_after_second == count_after_first
|
||||
|
||||
def test_create_compliance_requirements_kubernetes_provider(
|
||||
self,
|
||||
tenants_fixture,
|
||||
@@ -3545,19 +3674,19 @@ class TestAggregateFindingsByRegion:
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
# Mock findings with resources
|
||||
mock_finding1 = MagicMock()
|
||||
mock_finding1.check_id = "check1"
|
||||
mock_finding1.status = "FAIL"
|
||||
mock_finding1.compliance = {modeled_threatscore_compliance_id: ["req1", "req2"]}
|
||||
|
||||
mock_resource1 = MagicMock()
|
||||
mock_resource1.region = "us-east-1"
|
||||
mock_finding1.small_resources = [mock_resource1]
|
||||
# (check_id, status, resource_regions, compliance) tuples
|
||||
finding_rows = [
|
||||
(
|
||||
"check1",
|
||||
"FAIL",
|
||||
["us-east-1"],
|
||||
{modeled_threatscore_compliance_id: ["req1", "req2"]},
|
||||
)
|
||||
]
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.only.return_value = mock_queryset
|
||||
mock_queryset.prefetch_related.return_value = [mock_finding1]
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = finding_rows
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
@@ -3571,6 +3700,12 @@ class TestAggregateFindingsByRegion:
|
||||
)
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
# Verify structure of check_status_by_region
|
||||
assert isinstance(check_status_by_region, dict)
|
||||
assert "us-east-1" in check_status_by_region
|
||||
@@ -3590,27 +3725,15 @@ class TestAggregateFindingsByRegion:
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
# First finding with PASS status
|
||||
mock_finding1 = MagicMock()
|
||||
mock_finding1.check_id = "check1"
|
||||
mock_finding1.status = "PASS"
|
||||
mock_finding1.compliance = {}
|
||||
mock_resource1 = MagicMock()
|
||||
mock_resource1.region = "us-east-1"
|
||||
mock_finding1.small_resources = [mock_resource1]
|
||||
|
||||
# Second finding with FAIL status for same check/region
|
||||
mock_finding2 = MagicMock()
|
||||
mock_finding2.check_id = "check1"
|
||||
mock_finding2.status = "FAIL"
|
||||
mock_finding2.compliance = {}
|
||||
mock_resource2 = MagicMock()
|
||||
mock_resource2.region = "us-east-1"
|
||||
mock_finding2.small_resources = [mock_resource2]
|
||||
# Same check/region: PASS first, then FAIL — FAIL must win
|
||||
finding_rows = [
|
||||
("check1", "PASS", ["us-east-1"], {}),
|
||||
("check1", "FAIL", ["us-east-1"], {}),
|
||||
]
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.only.return_value = mock_queryset
|
||||
mock_queryset.prefetch_related.return_value = [mock_finding1, mock_finding2]
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = finding_rows
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
@@ -3622,6 +3745,12 @@ class TestAggregateFindingsByRegion:
|
||||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
# FAIL should override PASS
|
||||
assert check_status_by_region["us-east-1"]["check1"] == "FAIL"
|
||||
|
||||
@@ -3636,8 +3765,8 @@ class TestAggregateFindingsByRegion:
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.only.return_value = mock_queryset
|
||||
mock_queryset.prefetch_related.return_value = []
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = []
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
@@ -3649,6 +3778,12 @@ class TestAggregateFindingsByRegion:
|
||||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
# Verify filter was called with muted=False
|
||||
mock_findings_filter.assert_called_once_with(
|
||||
tenant_id=tenant_id,
|
||||
@@ -3667,27 +3802,25 @@ class TestAggregateFindingsByRegion:
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
# Finding with PASS status
|
||||
mock_finding1 = MagicMock()
|
||||
mock_finding1.check_id = "check1"
|
||||
mock_finding1.status = "PASS"
|
||||
mock_finding1.compliance = {modeled_threatscore_compliance_id: ["req1"]}
|
||||
mock_resource1 = MagicMock()
|
||||
mock_resource1.region = "us-east-1"
|
||||
mock_finding1.small_resources = [mock_resource1]
|
||||
|
||||
# Finding with FAIL status
|
||||
mock_finding2 = MagicMock()
|
||||
mock_finding2.check_id = "check2"
|
||||
mock_finding2.status = "FAIL"
|
||||
mock_finding2.compliance = {modeled_threatscore_compliance_id: ["req1"]}
|
||||
mock_resource2 = MagicMock()
|
||||
mock_resource2.region = "us-east-1"
|
||||
mock_finding2.small_resources = [mock_resource2]
|
||||
# PASS and FAIL findings mapped to the same ThreatScore requirement
|
||||
finding_rows = [
|
||||
(
|
||||
"check1",
|
||||
"PASS",
|
||||
["us-east-1"],
|
||||
{modeled_threatscore_compliance_id: ["req1"]},
|
||||
),
|
||||
(
|
||||
"check2",
|
||||
"FAIL",
|
||||
["us-east-1"],
|
||||
{modeled_threatscore_compliance_id: ["req1"]},
|
||||
),
|
||||
]
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.only.return_value = mock_queryset
|
||||
mock_queryset.prefetch_related.return_value = [mock_finding1, mock_finding2]
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = finding_rows
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
@@ -3699,6 +3832,12 @@ class TestAggregateFindingsByRegion:
|
||||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
# Verify compliance counts
|
||||
normalized_id = re.sub(
|
||||
r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower()
|
||||
@@ -3721,27 +3860,15 @@ class TestAggregateFindingsByRegion:
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
# Finding in us-east-1
|
||||
mock_finding1 = MagicMock()
|
||||
mock_finding1.check_id = "check1"
|
||||
mock_finding1.status = "FAIL"
|
||||
mock_finding1.compliance = {}
|
||||
mock_resource1 = MagicMock()
|
||||
mock_resource1.region = "us-east-1"
|
||||
mock_finding1.small_resources = [mock_resource1]
|
||||
|
||||
# Finding in us-west-2
|
||||
mock_finding2 = MagicMock()
|
||||
mock_finding2.check_id = "check1"
|
||||
mock_finding2.status = "PASS"
|
||||
mock_finding2.compliance = {}
|
||||
mock_resource2 = MagicMock()
|
||||
mock_resource2.region = "us-west-2"
|
||||
mock_finding2.small_resources = [mock_resource2]
|
||||
# One finding per region
|
||||
finding_rows = [
|
||||
("check1", "FAIL", ["us-east-1"], {}),
|
||||
("check1", "PASS", ["us-west-2"], {}),
|
||||
]
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.only.return_value = mock_queryset
|
||||
mock_queryset.prefetch_related.return_value = [mock_finding1, mock_finding2]
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = finding_rows
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
@@ -3753,6 +3880,12 @@ class TestAggregateFindingsByRegion:
|
||||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
# Verify both regions are present with correct statuses
|
||||
assert "us-east-1" in check_status_by_region
|
||||
assert "us-west-2" in check_status_by_region
|
||||
@@ -3761,17 +3894,26 @@ class TestAggregateFindingsByRegion:
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_findings_by_region_empty_findings(
|
||||
def test_aggregate_findings_by_region_multi_region_finding(
|
||||
self, mock_rls_transaction, mock_findings_filter
|
||||
):
|
||||
"""Test with no findings - should return empty dicts."""
|
||||
"""A finding with multiple resource_regions is tallied in every region."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
finding_rows = [
|
||||
(
|
||||
"check1",
|
||||
"FAIL",
|
||||
["us-east-1", "eu-west-1"],
|
||||
{modeled_threatscore_compliance_id: ["req1"]},
|
||||
)
|
||||
]
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.only.return_value = mock_queryset
|
||||
mock_queryset.prefetch_related.return_value = []
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = finding_rows
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
@@ -3785,6 +3927,92 @@ class TestAggregateFindingsByRegion:
|
||||
)
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
normalized_id = re.sub(
|
||||
r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower()
|
||||
)
|
||||
for region in ("us-east-1", "eu-west-1"):
|
||||
assert check_status_by_region[region]["check1"] == "FAIL"
|
||||
req_stats = findings_count_by_compliance[region][normalized_id]["req1"]
|
||||
assert req_stats == {"total": 1, "pass": 0}
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_findings_by_region_skips_empty_regions(
|
||||
self, mock_rls_transaction, mock_findings_filter
|
||||
):
|
||||
"""A finding with no denormalized regions contributes nothing."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
finding_rows = [
|
||||
("check1", "FAIL", [], {modeled_threatscore_compliance_id: ["req1"]}),
|
||||
("check2", "PASS", None, {}),
|
||||
]
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = finding_rows
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
check_status_by_region, findings_count_by_compliance = (
|
||||
_aggregate_findings_by_region(
|
||||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||||
)
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
assert check_status_by_region == {}
|
||||
assert findings_count_by_compliance == {}
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_findings_by_region_empty_findings(
|
||||
self, mock_rls_transaction, mock_findings_filter
|
||||
):
|
||||
"""Test with no findings - should return empty dicts."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = str(uuid.uuid4())
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values_list.return_value = mock_queryset
|
||||
mock_queryset.iterator.return_value = []
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
check_status_by_region, findings_count_by_compliance = (
|
||||
_aggregate_findings_by_region(
|
||||
tenant_id, scan_id, modeled_threatscore_compliance_id
|
||||
)
|
||||
)
|
||||
|
||||
# Streaming query contract: column-scoped values_list + iterator
|
||||
mock_queryset.values_list.assert_called_once_with(
|
||||
"check_id", "status", "resource_regions", "compliance"
|
||||
)
|
||||
mock_queryset.iterator.assert_called_once()
|
||||
|
||||
assert check_status_by_region == {}
|
||||
assert findings_count_by_compliance == {}
|
||||
|
||||
|
||||
@@ -15,8 +15,10 @@ from tasks.jobs.lighthouse_providers import (
|
||||
from tasks.tasks import (
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY,
|
||||
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
|
||||
ScanReportRLSTask,
|
||||
_cleanup_orphan_scheduled_scans,
|
||||
_perform_scan_complete_tasks,
|
||||
_scan_tmp_output_directory,
|
||||
check_integrations_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
generate_outputs_task,
|
||||
@@ -321,6 +323,7 @@ class TestGenerateOutputs:
|
||||
|
||||
mock_transformed_stats = {"some": "stats"}
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch(
|
||||
"tasks.tasks.FindingOutput._transform_findings_stats",
|
||||
return_value=mock_transformed_stats,
|
||||
@@ -439,6 +442,7 @@ class TestGenerateOutputs:
|
||||
mock_provider.uid = "test-provider-uid"
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter,
|
||||
patch("tasks.tasks.Provider.objects.get", return_value=mock_provider),
|
||||
patch("tasks.tasks.initialize_prowler_provider"),
|
||||
@@ -594,6 +598,7 @@ class TestGenerateOutputs:
|
||||
]
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch("tasks.tasks.ScanSummary.objects.filter") as mock_summary,
|
||||
patch(
|
||||
"tasks.tasks.Provider.objects.get",
|
||||
@@ -668,6 +673,7 @@ class TestGenerateOutputs:
|
||||
mock_provider.uid = "test-provider-uid"
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter,
|
||||
patch("tasks.tasks.Provider.objects.get", return_value=mock_provider),
|
||||
patch("tasks.tasks.initialize_prowler_provider"),
|
||||
@@ -771,6 +777,38 @@ class TestGenerateOutputs:
|
||||
mock_s3_task.assert_called_once()
|
||||
|
||||
|
||||
class TestScanReportRLSTaskOnFailure:
|
||||
def test_on_failure_removes_scan_tmp_directory(self):
|
||||
task = ScanReportRLSTask()
|
||||
|
||||
with patch("tasks.tasks.rmtree") as mock_rmtree:
|
||||
task.on_failure(
|
||||
exc=OSError("No space left on device"),
|
||||
task_id="task-abc",
|
||||
args=(),
|
||||
kwargs={"tenant_id": "t-1", "scan_id": "s-1"},
|
||||
_einfo=None,
|
||||
)
|
||||
|
||||
mock_rmtree.assert_called_once_with(
|
||||
_scan_tmp_output_directory("t-1", "s-1"), ignore_errors=True
|
||||
)
|
||||
|
||||
def test_on_failure_skips_when_missing_kwargs(self):
|
||||
task = ScanReportRLSTask()
|
||||
|
||||
with patch("tasks.tasks.rmtree") as mock_rmtree:
|
||||
task.on_failure(
|
||||
exc=OSError("No space left on device"),
|
||||
task_id="task-abc",
|
||||
args=(),
|
||||
kwargs={},
|
||||
_einfo=None,
|
||||
)
|
||||
|
||||
mock_rmtree.assert_not_called()
|
||||
|
||||
|
||||
class TestScanCompleteTasks:
|
||||
@patch("tasks.tasks.aggregate_attack_surface_task.apply_async")
|
||||
@patch("tasks.tasks.chain")
|
||||
@@ -1079,6 +1117,7 @@ class TestCheckIntegrationsTask:
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
@patch("tasks.tasks.get_prowler_provider_compliance", return_value={})
|
||||
@patch("tasks.tasks.s3_integration_task")
|
||||
@patch("tasks.tasks.Integration.objects.filter")
|
||||
@patch("tasks.tasks.ScanSummary.objects.filter")
|
||||
@@ -1111,6 +1150,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_scan_summary,
|
||||
mock_integration_filter,
|
||||
mock_s3_task,
|
||||
mock_get_prowler_compliance,
|
||||
):
|
||||
"""Test that ASFF output is generated for AWS providers with SecurityHub integration."""
|
||||
# Setup
|
||||
@@ -1207,6 +1247,7 @@ class TestCheckIntegrationsTask:
|
||||
|
||||
assert result == {"upload": True}
|
||||
|
||||
@patch("tasks.tasks.get_prowler_provider_compliance", return_value={})
|
||||
@patch("tasks.tasks.s3_integration_task")
|
||||
@patch("tasks.tasks.Integration.objects.filter")
|
||||
@patch("tasks.tasks.ScanSummary.objects.filter")
|
||||
@@ -1239,6 +1280,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_scan_summary,
|
||||
mock_integration_filter,
|
||||
mock_s3_task,
|
||||
mock_get_prowler_compliance,
|
||||
):
|
||||
"""Test that ASFF output is NOT generated for AWS providers without SecurityHub integration."""
|
||||
# Setup
|
||||
@@ -1332,6 +1374,7 @@ class TestCheckIntegrationsTask:
|
||||
|
||||
assert result == {"upload": True}
|
||||
|
||||
@patch("tasks.tasks.get_prowler_provider_compliance", return_value={})
|
||||
@patch("tasks.tasks.ScanSummary.objects.filter")
|
||||
@patch("tasks.tasks.Provider.objects.get")
|
||||
@patch("tasks.tasks.initialize_prowler_provider")
|
||||
@@ -1360,6 +1403,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_initialize_provider,
|
||||
mock_provider_get,
|
||||
mock_scan_summary,
|
||||
mock_get_prowler_compliance,
|
||||
):
|
||||
"""Test that ASFF output is NOT generated for non-AWS providers (e.g., Azure, GCP)."""
|
||||
# Setup
|
||||
@@ -2672,3 +2716,36 @@ class TestReaggregateAllFindingGroupSummaries:
|
||||
assert result == {"scans_reaggregated": 0}
|
||||
mock_group.assert_not_called()
|
||||
mock_chain.assert_not_called()
|
||||
|
||||
|
||||
class TestTaskTimeLimits:
|
||||
"""The per-task limits in task_annotations must actually take effect.
|
||||
|
||||
Celery applies a "*" annotation after the per-task one, so a "*" entry would
|
||||
silently overwrite every specific limit and cap long scans at the default. The
|
||||
default is set as the global limit instead, and these per-task limits must win.
|
||||
"""
|
||||
|
||||
def test_long_running_tasks_exceed_the_default_limit(self):
|
||||
from config.celery import celery_app
|
||||
|
||||
default = celery_app.conf.task_time_limit
|
||||
for name in (
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"provider-deletion",
|
||||
"tenant-deletion",
|
||||
):
|
||||
assert celery_app.tasks[name].time_limit > default
|
||||
|
||||
def test_connection_checks_stay_below_the_default_limit(self):
|
||||
from config.celery import celery_app
|
||||
|
||||
default = celery_app.conf.task_time_limit
|
||||
for name in (
|
||||
"provider-connection-check",
|
||||
"integration-connection-check",
|
||||
"lighthouse-connection-check",
|
||||
"lighthouse-provider-connection-check",
|
||||
):
|
||||
assert celery_app.tasks[name].time_limit < default
|
||||
|
||||
Generated
+228
-115
@@ -16,7 +16,7 @@ constraints = [
|
||||
{ name = "aiobotocore", specifier = "==2.25.1" },
|
||||
{ name = "aiofiles", specifier = "==24.1.0" },
|
||||
{ name = "aiohappyeyeballs", specifier = "==2.6.1" },
|
||||
{ name = "aiohttp", specifier = "==3.13.5" },
|
||||
{ name = "aiohttp", specifier = "==3.14.0" },
|
||||
{ name = "aioitertools", specifier = "==0.13.0" },
|
||||
{ name = "aiosignal", specifier = "==1.4.0" },
|
||||
{ name = "alibabacloud-actiontrail20200706", specifier = "==2.4.1" },
|
||||
@@ -61,9 +61,8 @@ constraints = [
|
||||
{ name = "astroid", specifier = "==3.2.4" },
|
||||
{ name = "async-timeout", specifier = "==5.0.1" },
|
||||
{ name = "attrs", specifier = "==25.4.0" },
|
||||
{ name = "authlib", specifier = "==1.6.9" },
|
||||
{ name = "authlib", specifier = "==1.6.12" },
|
||||
{ name = "autopep8", specifier = "==2.3.2" },
|
||||
{ name = "awsipranges", specifier = "==0.3.3" },
|
||||
{ name = "azure-cli-core", specifier = "==2.83.0" },
|
||||
{ name = "azure-cli-telemetry", specifier = "==1.1.0" },
|
||||
{ name = "azure-common", specifier = "==1.1.28" },
|
||||
@@ -146,6 +145,7 @@ constraints = [
|
||||
{ name = "django-celery-results", specifier = "==2.6.0" },
|
||||
{ name = "django-cors-headers", specifier = "==4.4.0" },
|
||||
{ name = "django-environ", specifier = "==0.11.2" },
|
||||
{ name = "django-eventstream", specifier = "==5.3.3" },
|
||||
{ name = "django-filter", specifier = "==24.3" },
|
||||
{ name = "django-guid", specifier = "==3.5.0" },
|
||||
{ name = "django-postgres-extra", specifier = "==2.0.9" },
|
||||
@@ -163,7 +163,7 @@ constraints = [
|
||||
{ name = "drf-simple-apikey", specifier = "==2.2.1" },
|
||||
{ name = "drf-spectacular", specifier = "==0.27.2" },
|
||||
{ name = "drf-spectacular-jsonapi", specifier = "==0.5.1" },
|
||||
{ name = "dulwich", specifier = "==0.23.0" },
|
||||
{ name = "dulwich", specifier = "==1.2.5" },
|
||||
{ name = "duo-client", specifier = "==5.5.0" },
|
||||
{ name = "durationpy", specifier = "==0.10" },
|
||||
{ name = "email-validator", specifier = "==2.2.0" },
|
||||
@@ -190,7 +190,7 @@ constraints = [
|
||||
{ name = "grpc-google-iam-v1", specifier = "==0.14.3" },
|
||||
{ name = "grpcio", specifier = "==1.76.0" },
|
||||
{ name = "grpcio-status", specifier = "==1.76.0" },
|
||||
{ name = "gunicorn", specifier = "==23.0.0" },
|
||||
{ name = "gunicorn", specifier = "==26.0.0" },
|
||||
{ name = "h11", specifier = "==0.16.0" },
|
||||
{ name = "h2", specifier = "==4.3.0" },
|
||||
{ name = "hpack", specifier = "==4.1.0" },
|
||||
@@ -199,8 +199,8 @@ constraints = [
|
||||
{ name = "httpx", specifier = "==0.28.1" },
|
||||
{ name = "humanfriendly", specifier = "==10.0" },
|
||||
{ name = "hyperframe", specifier = "==6.1.0" },
|
||||
{ name = "iamdata", specifier = "==0.1.202602021" },
|
||||
{ name = "idna", specifier = "==3.11" },
|
||||
{ name = "iamdata", specifier = "==0.1.202605131" },
|
||||
{ name = "idna", specifier = "==3.15" },
|
||||
{ name = "importlib-metadata", specifier = "==8.7.1" },
|
||||
{ name = "inflection", specifier = "==0.5.1" },
|
||||
{ name = "iniconfig", specifier = "==2.3.0" },
|
||||
@@ -252,7 +252,7 @@ constraints = [
|
||||
{ name = "neo4j", specifier = "==6.1.0" },
|
||||
{ name = "nest-asyncio", specifier = "==1.6.0" },
|
||||
{ name = "nltk", specifier = "==3.9.4" },
|
||||
{ name = "numpy", specifier = "==2.0.2" },
|
||||
{ name = "numpy", specifier = "==2.2.6" },
|
||||
{ name = "oauthlib", specifier = "==3.3.1" },
|
||||
{ name = "oci", specifier = "==2.169.0" },
|
||||
{ name = "openai", specifier = "==1.109.1" },
|
||||
@@ -281,7 +281,7 @@ constraints = [
|
||||
{ name = "psutil", specifier = "==7.2.2" },
|
||||
{ name = "psycopg2-binary", specifier = "==2.9.9" },
|
||||
{ name = "py-deviceid", specifier = "==0.1.1" },
|
||||
{ name = "py-iam-expand", specifier = "==0.1.0" },
|
||||
{ name = "py-iam-expand", specifier = "==0.3.0" },
|
||||
{ name = "py-ocsf-models", specifier = "==0.8.1" },
|
||||
{ name = "pyasn1", specifier = "==0.6.3" },
|
||||
{ name = "pyasn1-modules", specifier = "==0.4.2" },
|
||||
@@ -291,7 +291,7 @@ constraints = [
|
||||
{ name = "pydantic-core", specifier = "==2.41.5" },
|
||||
{ name = "pygithub", specifier = "==2.8.0" },
|
||||
{ name = "pygments", specifier = "==2.20.0" },
|
||||
{ name = "pyjwt", specifier = "==2.12.1" },
|
||||
{ name = "pyjwt", specifier = "==2.13.0" },
|
||||
{ name = "pylint", specifier = "==3.2.5" },
|
||||
{ name = "pymsalruntime", specifier = "==0.18.1" },
|
||||
{ name = "pynacl", specifier = "==1.6.2" },
|
||||
@@ -374,8 +374,10 @@ constraints = [
|
||||
{ name = "zstd", specifier = "==1.5.7.3" },
|
||||
]
|
||||
overrides = [
|
||||
{ name = "dulwich", specifier = "==1.2.5" },
|
||||
{ name = "microsoft-kiota-abstractions", specifier = "==1.9.9" },
|
||||
{ name = "okta", specifier = "==3.4.2" },
|
||||
{ name = "pyjwt", extras = ["crypto"], specifier = "==2.13.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -393,7 +395,7 @@ version = "1.2.7"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
@@ -467,7 +469,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "aiohttp"
|
||||
version = "3.13.5"
|
||||
version = "3.14.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aiohappyeyeballs" },
|
||||
@@ -476,44 +478,47 @@ dependencies = [
|
||||
{ name = "frozenlist" },
|
||||
{ name = "multidict" },
|
||||
{ name = "propcache" },
|
||||
{ name = "typing-extensions" },
|
||||
{ name = "yarl" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/f5/a20c4ac64aeaef1679e25c9983573618ff765d7aa829fa2b84ae7573169e/aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6", size = 757513, upload-time = "2026-03-31T21:57:02.146Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/0a/39fa6c6b179b53fcb3e4b3d2b6d6cad0180854eda17060c7218540102bef/aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d", size = 506748, upload-time = "2026-03-31T21:57:04.275Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/ec/e38ce072e724fd7add6243613f8d1810da084f54175353d25ccf9f9c7e5a/aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c", size = 501673, upload-time = "2026-03-31T21:57:06.208Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/ba/3bc7525d7e2beaa11b309a70d48b0d3cfc3c2089ec6a7d0820d59c657053/aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb", size = 1763757, upload-time = "2026-03-31T21:57:07.882Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/ab/e87744cf18f1bd78263aba24924d4953b41086bd3a31d22452378e9028a0/aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6", size = 1720152, upload-time = "2026-03-31T21:57:09.946Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/f3/ed17a6f2d742af17b50bae2d152315ed1b164b07a5fd5cc1754d99e4dfa5/aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13", size = 1818010, upload-time = "2026-03-31T21:57:12.157Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/06/ecbc63dc937192e2a5cb46df4d3edb21deb8225535818802f210a6ea5816/aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174", size = 1907251, upload-time = "2026-03-31T21:57:14.023Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/a5/0521aa32c1ddf3aa1e71dcc466be0b7db2771907a13f18cddaa45967d97b/aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc", size = 1759969, upload-time = "2026-03-31T21:57:16.146Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/78/a38f8c9105199dd3b9706745865a8a59d0041b6be0ca0cc4b2ccf1bab374/aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6", size = 1616871, upload-time = "2026-03-31T21:57:17.856Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/41/27392a61ead8ab38072105c71aa44ff891e71653fe53d576a7067da2b4e8/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49", size = 1739844, upload-time = "2026-03-31T21:57:19.679Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/55/5564e7ae26d94f3214250009a0b1c65a0c6af4bf88924ccb6fdab901de28/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8", size = 1731969, upload-time = "2026-03-31T21:57:22.006Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/c5/705a3929149865fc941bcbdd1047b238e4a72bcb215a9b16b9d7a2e8d992/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d", size = 1795193, upload-time = "2026-03-31T21:57:24.256Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/19/edabed62f718d02cff7231ca0db4ef1c72504235bc467f7b67adb1679f48/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c", size = 1606477, upload-time = "2026-03-31T21:57:26.364Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/fc/76f80ef008675637d88d0b21584596dc27410a990b0918cb1e5776545b5b/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac", size = 1813198, upload-time = "2026-03-31T21:57:28.316Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/67/5b3ac26b80adb20ea541c487f73730dc8fa107d632c998f25bbbab98fcda/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3", size = 1752321, upload-time = "2026-03-31T21:57:30.549Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/06/e4a2e49255ea23fa4feeb5ab092d90240d927c15e47b5b5c48dff5a9ce29/aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06", size = 439069, upload-time = "2026-03-31T21:57:32.388Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/43/8c7163a596dab4f8be12c190cf467a1e07e4734cf90eebb39f7f5d53fc6a/aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8", size = 462859, upload-time = "2026-03-31T21:57:34.455Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/6f/353954c29e7dcce7cf00280a02c75f30e133c00793c7a2ed3776d7b2f426/aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9", size = 748876, upload-time = "2026-03-31T21:57:36.319Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/1b/428a7c64687b3b2e9cd293186695affc0e1e54a445d0361743b231f11066/aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416", size = 499557, upload-time = "2026-03-31T21:57:38.236Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/47/7be41556bfbb6917069d6a6634bb7dd5e163ba445b783a90d40f5ac7e3a7/aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2", size = 500258, upload-time = "2026-03-31T21:57:39.923Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/84/c9ecc5828cb0b3695856c07c0a6817a99d51e2473400f705275a2b3d9239/aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4", size = 1749199, upload-time = "2026-03-31T21:57:41.938Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/d3/3c6d610e66b495657622edb6ae7c7fd31b2e9086b4ec50b47897ad6042a9/aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9", size = 1721013, upload-time = "2026-03-31T21:57:43.904Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/a0/24409c12217456df0bae7babe3b014e460b0b38a8e60753d6cb339f6556d/aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5", size = 1781501, upload-time = "2026-03-31T21:57:46.285Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/98/9d/b65ec649adc5bccc008b0957a9a9c691070aeac4e41cea18559fef49958b/aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e", size = 1878981, upload-time = "2026-03-31T21:57:48.734Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/d8/8d44036d7eb7b6a8ec4c5494ea0c8c8b94fbc0ed3991c1a7adf230df03bf/aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1", size = 1767934, upload-time = "2026-03-31T21:57:51.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/04/d3f8211f273356f158e3464e9e45484d3fb8c4ce5eb2f6fe9405c3273983/aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286", size = 1566671, upload-time = "2026-03-31T21:57:53.326Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/41/db/073e4ebe00b78e2dfcacff734291651729a62953b48933d765dc513bf798/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9", size = 1705219, upload-time = "2026-03-31T21:57:55.385Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/45/7dfba71a2f9fd97b15c95c06819de7eb38113d2cdb6319669195a7d64270/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88", size = 1743049, upload-time = "2026-03-31T21:57:57.341Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/71/901db0061e0f717d226386a7f471bb59b19566f2cae5f0d93874b017271f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3", size = 1749557, upload-time = "2026-03-31T21:57:59.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/d5/41eebd16066e59cd43728fe74bce953d7402f2b4ddfdfef2c0e9f17ca274/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b", size = 1558931, upload-time = "2026-03-31T21:58:01.972Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/e6/4a799798bf05740e66c3a1161079bda7a3dd8e22ca392481d7a7f9af82a6/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe", size = 1774125, upload-time = "2026-03-31T21:58:04.007Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/63/7749337c90f92bc2cb18f9560d67aa6258c7060d1397d21529b8004fcf6f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14", size = 1732427, upload-time = "2026-03-31T21:58:06.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/98/de/cf2f44ff98d307e72fb97d5f5bbae3bfcb442f0ea9790c0bf5c5c2331404/aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3", size = 433534, upload-time = "2026-03-31T21:58:08.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/ca/eadf6f9c8fa5e31d40993e3db153fb5ed0b11008ad5d9de98a95045bed84/aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1", size = 460446, upload-time = "2026-03-31T21:58:10.945Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/47/7727bfe8db93f8835a001bd4359d8480cc68d1259b8bce334668f8be97bd/aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a", size = 759147, upload-time = "2026-06-01T19:37:12.918Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/f2/cd3fedff6fade73d71df9ec908c210cec518ef90fd00289250684b90aecf/aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803", size = 513705, upload-time = "2026-06-01T19:37:14.633Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/fe/49746b6b610144a06323bebd8e1211a390310d8c69b98dd6d52df341bc3e/aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e", size = 509627, upload-time = "2026-06-01T19:37:16.385Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/3f/28f2f6cf3d5c0e7b01b27140d0e7873fd11fb341169ad3ce78ad04aba628/aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903", size = 1769293, upload-time = "2026-06-01T19:37:18.067Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/6f/2e5f1b525d5474b12b3c60abf733a755845f3bceff21542081ada515f837/aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb", size = 1732363, upload-time = "2026-06-01T19:37:20.138Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/ce/596120faa85ca7b19cd061e3f2f3be23aa8f11a0aedf9191db9e0da1bd76/aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2", size = 1840375, upload-time = "2026-06-01T19:37:22.104Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/3c/a7ffe05a757a4a7867643da69357ec41f506879fbd1b231d2ed90af246b2/aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81", size = 1921484, upload-time = "2026-06-01T19:37:24.068Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/93/fa/2c861170bbd4a491de93a69e081db1d971092569e0d593a98ef62c384dc1/aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee", size = 1774153, upload-time = "2026-06-01T19:37:26.256Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/da/1d2f5a165f47ec9b1f69d37b8b977fdc4d501aa72ffb7930db27bb9e49ea/aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d", size = 1632569, upload-time = "2026-06-01T19:37:28.192Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/1d/7a6e295c4257252f70f69e90864fdad74b6a1293054fb3f9e65a15de6d63/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00", size = 1740325, upload-time = "2026-06-01T19:37:30.08Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/7e/e1899b1ca3ec62f1eab2a5cbde14039b97493f7f53eb88d9b668562ffa8d/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026", size = 1748691, upload-time = "2026-06-01T19:37:32.211Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/54/4e6b61c1fe7d3433f82bcc6bd7e4d7c683a742a10c9b12a025fd3695c047/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86", size = 1814477, upload-time = "2026-06-01T19:37:34.173Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/38/86fd51be2e08d8e45c83d879d255f10391903cd9fe2a16512f7591a15873/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f", size = 1623393, upload-time = "2026-06-01T19:37:36.281Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/49/466e947a42a88ee23c486d036e7e5d1b097f1bafd8084ad9c9a0a92f0f43/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93", size = 1824097, upload-time = "2026-06-01T19:37:38.421Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/89/35f3410bc284682338a1be6b6ea0c5abfa05f063942cfaa9256608440434/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996", size = 1764790, upload-time = "2026-06-01T19:37:40.755Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/80/2d4291bd5724d3d17e5951aff5a3e02281483fb47295f0788276ee66cd73/aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae", size = 454176, upload-time = "2026-06-01T19:37:42.837Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/ed/41d0ad4f6ececffc32bdf1f7b494e5498f7ca5c849ea2e3cc9bbd1668251/aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5", size = 479334, upload-time = "2026-06-01T19:37:44.776Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/86/c0b5e305c770053f8c3d069bb52b8196917ba91949d1962d52eb307fb0d2/aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43", size = 450262, upload-time = "2026-06-01T19:37:46.461Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672, upload-time = "2026-06-01T19:37:55.791Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064, upload-time = "2026-06-01T19:37:57.931Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125, upload-time = "2026-06-01T19:38:00.225Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356, upload-time = "2026-06-01T19:38:04.637Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216, upload-time = "2026-06-01T19:38:08.736Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500, upload-time = "2026-06-01T19:38:11.111Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224, upload-time = "2026-06-01T19:38:13.391Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252, upload-time = "2026-06-01T19:38:15.498Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650, upload-time = "2026-06-01T19:38:19.545Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145, upload-time = "2026-06-01T19:38:21.505Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250, upload-time = "2026-06-01T19:38:24.027Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1043,15 +1048,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/43/53afb8ba17218f19b77c7834128566c5bbb100a0ad9ba2e8e89d089d7079/autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128", size = 45807, upload-time = "2025-01-14T14:46:15.466Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "awsipranges"
|
||||
version = "0.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/19/2e/6efa95f995369da828715f41705686cd214b9259ed758266942553d40441/awsipranges-0.3.3.tar.gz", hash = "sha256:4f0b3f22a9dc1163c85b513bed812b6c92bdacd674e6a7b68252a3c25b99e2c0", size = 16739, upload-time = "2022-02-10T21:08:32.825Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/83/ce/5c9a8bf91bdc9592a409c99e58fd99f2727ab8d634719c0ad796021b76d7/awsipranges-0.3.3-py3-none-any.whl", hash = "sha256:f3d7a54aeaf7fe310beb5d377a4034a63a51b72677ae6af3e0967bc4de7eedaf", size = 18106, upload-time = "2022-02-10T21:08:31.497Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "azure-cli-core"
|
||||
version = "2.83.0"
|
||||
@@ -1074,7 +1070,7 @@ dependencies = [
|
||||
{ name = "pkginfo" },
|
||||
{ name = "psutil", marker = "sys_platform != 'cygwin'" },
|
||||
{ name = "py-deviceid" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "pyopenssl" },
|
||||
{ name = "requests", extra = ["socks"] },
|
||||
]
|
||||
@@ -2360,6 +2356,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/f1/468b49cccba3b42dda571063a14c668bb0b53a1d5712426d18e36663bd53/django_environ-0.11.2-py2.py3-none-any.whl", hash = "sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05", size = 19141, upload-time = "2023-09-01T21:02:59.88Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "django-eventstream"
|
||||
version = "5.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "django" },
|
||||
{ name = "django-grip" },
|
||||
{ name = "gripcontrol" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "six" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f2/49/ec6cbc24f3f30465370df7096cfea9722bad2b0c1f35a7ff5d45fb96cff6/django_eventstream-5.3.3.tar.gz", hash = "sha256:6880b03298eebf18c1b736b972fb862eaf631dfbb79f8b27496418a3495d08dc", size = 47622, upload-time = "2025-10-23T00:22:40.291Z" }
|
||||
|
||||
[[package]]
|
||||
name = "django-filter"
|
||||
version = "24.3"
|
||||
@@ -2372,6 +2381,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/09/b1/92f1c30b47c1ebf510c35a2ccad9448f73437e5891bbd2b4febe357cc3de/django_filter-24.3-py3-none-any.whl", hash = "sha256:c4852822928ce17fb699bcfccd644b3574f1a2d80aeb2b4ff4f16b02dd49dc64", size = 95011, upload-time = "2024-08-02T13:27:55.616Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "django-grip"
|
||||
version = "3.5.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "django" },
|
||||
{ name = "gripcontrol" },
|
||||
{ name = "pubcontrol" },
|
||||
{ name = "six" },
|
||||
{ name = "werkzeug" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cb/d0/2c7b04fa864073cd8cb324f8674672162282d97540d56732cbd3a9ae5bca/django-grip-3.5.2.tar.gz", hash = "sha256:1ee1601492cd110256bd03e4a68797a9fbefa27c15f5a838bf245df97db0450c", size = 7626, upload-time = "2025-03-24T18:53:58.677Z" }
|
||||
|
||||
[[package]]
|
||||
name = "django-guid"
|
||||
version = "3.5.0"
|
||||
@@ -2457,7 +2479,7 @@ source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "django" },
|
||||
{ name = "djangorestframework" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a8/27/2874a325c11112066139769f7794afae238a07ce6adf96259f08fd37a9d7/djangorestframework_simplejwt-5.5.1.tar.gz", hash = "sha256:e72c5572f51d7803021288e2057afcbd03f17fe11d484096f40a460abc76e87f", size = 101265, upload-time = "2025-07-21T16:52:25.026Z" }
|
||||
wheels = [
|
||||
@@ -2576,24 +2598,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "dulwich"
|
||||
version = "0.23.0"
|
||||
version = "1.2.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4b/ac/ba58cf420640c7bc77ae8e1b31e174d83c9117750c63cf9ea3b5e202e5c4/dulwich-0.23.0.tar.gz", hash = "sha256:0aa6c2489dd5e978b27e9b75983b7331a66c999f0efc54ebe37cab808ed322ae", size = 575116, upload-time = "2025-06-21T17:56:47.494Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7f/85/ceb8ecff5cdeee4ceeebb86b599476dee559041dacc6c2c50cc0d4711549/dulwich-1.2.5.tar.gz", hash = "sha256:0395b2c8924c3424bafe2d9c1edd5348cc4b21ce9c1d6655bf01f9a5c47164c8", size = 1253230, upload-time = "2026-05-28T22:27:55.17Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/11/f6bbba8583f69cf19ef4bd7f5fde1a6b5ccaf8b6951781cec8db247116f4/dulwich-0.23.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d68498fdda13ab00791b483daab3bcfe9f9721c037aa458695e6ad81640c57cc", size = 972658, upload-time = "2025-06-21T17:56:13.505Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/9d/2720e0ab58666378a33c752a61543f936cd6b06dfe5d84a2215ddc0914b0/dulwich-0.23.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cb7bb930b12471a1cfcea4b3d25a671dc0ad32573f0ad25684684298959a1527", size = 1049813, upload-time = "2025-06-21T17:56:14.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/f3/81d8075141dfcc0a0449c2093596e58d3e11444e3af54e819eca63b84dd0/dulwich-0.23.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2abbce32fd2bc7902bcc5f69b10bf22576810de21651baaa864b78fd7aec261", size = 1051639, upload-time = "2025-06-21T17:56:16.437Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/0d/c06ccb227b096aef5906142fe78b5c79f9070a0ea6152fc219941186d540/dulwich-0.23.0-cp311-cp311-win32.whl", hash = "sha256:9e3151f10ce2a9ff91bca64c74345217f53bdd947dc958032343822009832f7a", size = 642918, upload-time = "2025-06-21T17:56:18.373Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/1c/1e99aa34c9aead9e641b2d9934f0a3d00257f75027cf5cdecc8a1a6c18ae/dulwich-0.23.0-cp311-cp311-win_amd64.whl", hash = "sha256:3ae9f1d9dc92d4e9a3f89ba2c55221f7b6442c5dd93b3f6f539a3c9eb3f37bdd", size = 659010, upload-time = "2025-06-21T17:56:19.947Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/d7/1e6fba0235babe912e8467b036062e37d11672cbbeb0d8074f9d4559057b/dulwich-0.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:52cdef66a7994d29528ca79ca59452518bbba3fd56a9c61c61f6c467c1c7956e", size = 960292, upload-time = "2025-06-21T17:56:21.308Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/6a/23f0c487ec03f2752600cab4a8e0dedb38186246c475bf3fa90a8db830d5/dulwich-0.23.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d473888a6ab9ed5d4a4c3f053cbe5b77f72d54b6efdf5688fed76094316e571e", size = 1047892, upload-time = "2025-06-21T17:56:22.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/e2/8f3d216be5fd0ee1180d917b59b34b54b9896384cf139f319b5d3a8f16b4/dulwich-0.23.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:19fcf20224c641a61c774da92f098fbaae9938c7e17a52841e64092adf7e78f9", size = 1048699, upload-time = "2025-06-21T17:56:24.602Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/c4/18e6223cd4ad1ae9334eb4e6aa5952fd8f5c3d75762918eb90c209fec4ba/dulwich-0.23.0-cp312-cp312-win32.whl", hash = "sha256:7fc8b76b704ef35cd001e993e3aa4e1d666a2064bf467c07c560f12b2959dcaf", size = 641268, upload-time = "2025-06-21T17:56:26.18Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/9c/65bfbbac62d8a2967e13f6a1512371c5eb6b906a61fb6dead992669cad0e/dulwich-0.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:cb0566b888b578325350b4d67c61a0de35d417e9877560e3a6df88cae4576a59", size = 657837, upload-time = "2025-06-21T17:56:27.821Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/31/49318ee9db4b402e6d8b9b01bd4cae9298f59e1bb9bd56cf4a94e48fa069/dulwich-0.23.0-py3-none-any.whl", hash = "sha256:d8da6694ca332bb48775e35ee2215aa4673821164a91b83062f699c69f7cd135", size = 313776, upload-time = "2025-06-21T17:56:46.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/4a/654ae1671610fdf6b65a64586ad67ddd8550d4d08a632b2a4b9614754b6d/dulwich-1.2.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:556593fd11637f80f6018bee1916b1a84f5b420423b470ebb3f1a782ad6ef081", size = 1399277, upload-time = "2026-05-28T22:27:00.801Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/d8/06ee3bc8eded4bd7adf8adf0c9ea5f19bf96f7e5e626bfaf7311cde4208a/dulwich-1.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a70477c991e96cfe8fdd7c866e7251faf71b38bfeb51d6f27554c9cce1caabf3", size = 1382310, upload-time = "2026-05-28T22:27:02.216Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/17/a03adf50b9095f9f5d863393f21d585dea39bdc4fdf60788ff3a9407a512/dulwich-1.2.5-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9008ef25cabd379cda4fa86000fc38ca14b72afe17db798a8c85c0b2b7ce4d1e", size = 1470993, upload-time = "2026-05-28T22:27:04.075Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/58/1dc352d2a5e80befe4338af7208febb44bcfd7496b0dde5ac6dacb07b031/dulwich-1.2.5-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a5549f4afc973e0a15ea6b0244d57f848d3f3ee13dac557eb311024aebebf128", size = 1497820, upload-time = "2026-05-28T22:27:05.549Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/a8/e058959a87e7df7753b112ef66a43ccbc57338c1bbdc23a0edf3833396df/dulwich-1.2.5-cp311-cp311-win32.whl", hash = "sha256:5108acead814d1de8b6262d6d8fb90af7e82f5a4d83788b6b48e39d01800a92f", size = 1066549, upload-time = "2026-05-28T22:27:06.832Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/91/ff0b444f686718635348986bd73dfce42e947912417893de35de399b878b/dulwich-1.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:5e067b7feceb7034bc99e7c7143a704f1d97d4be7027d9a0aa5a83c0657ff091", size = 1079481, upload-time = "2026-05-28T22:27:08.33Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/22/4f75770bbe5521cac61c4820ef46d4fbf8c2175d3519ba3d0378d4ba798e/dulwich-1.2.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:701a9ecf7a8a44f5e2459e46befa93530cf36a8b1ae3140aefc007db1d7d0207", size = 1396522, upload-time = "2026-05-28T22:27:09.997Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/b1/c07c347681c0cf6acd4b189bf6e8d6207c71a1347b7a1e865eb40faa46b9/dulwich-1.2.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f90d68bfa97c4ca71de7507984365aefe27b6d248cb28dc99644d0f3ae8c60b", size = 1334826, upload-time = "2026-05-28T22:27:11.582Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/80/6818eb7ce492e18ab2efa92ab901d173b4b0b159e5681c1424f329600c40/dulwich-1.2.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:00b54a1d56ddbacdd8eadd6d4787a51b3a05fefa30eadbf9165fd283a00b90ed", size = 1416616, upload-time = "2026-05-28T22:27:13.195Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/a7/9790e60d19870f6554f7583722bb324c1355784316f20aeda1c0b5b1491a/dulwich-1.2.5-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d8f7ea8f47e38e5b0de3fab97e07e9c9161ffddc90b3964512cab2b7749df4e6", size = 1441354, upload-time = "2026-05-28T22:27:14.683Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/44/0ea8a69c24aa1254ff5996d682eae2eab287d471b937dcdb26d9ea9720b4/dulwich-1.2.5-cp312-cp312-win32.whl", hash = "sha256:8929134acf4ff967203df7600b38535f9b5b590462067a7e30dbce01acb97af9", size = 1017058, upload-time = "2026-05-28T22:27:16.121Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/9c/2fcddda7faec3bae52db7c64bfcb5dc756f597f33fae90e8d4e4b4d3b39b/dulwich-1.2.5-cp312-cp312-win_amd64.whl", hash = "sha256:9693d2c9e226b2ea855c1dc3a87e2f4d972f7523fc0f7924e5997e9f4c23d97f", size = 1031731, upload-time = "2026-05-28T22:27:17.633Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/4b/4a18a59ad230581cd0ef460e96001f90762e566dc2dfdba22aa358eb5a0e/dulwich-1.2.5-py3-none-any.whl", hash = "sha256:1679b376433a0fc7f36586afda1d4ed7427afa7a79d4bf17e5014474eea69fa4", size = 686745, upload-time = "2026-05-28T22:27:53.695Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2980,6 +3005,17 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/ab/717c58343cf02c5265b531384b248787e04d8160b8afe53d9eec053d7b44/greenlet-3.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bfb2d1763d777de5ee495c85309460f6fd8146e50ec9d0ae0183dbf6f0a829d1", size = 226403, upload-time = "2026-01-23T15:31:39.372Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gripcontrol"
|
||||
version = "4.4.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pubcontrol" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "six" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4f/51/1cbf88384fbe97a1454fb0adddcdca8cb90ceb99c3250274c334db844f4f/gripcontrol-4.4.0.tar.gz", hash = "sha256:44ee6fe244a02870aa4e5bc810138ccaf5070dce5eb149b8ee9e27b960a95c2d", size = 12526, upload-time = "2026-05-14T21:19:28.49Z" }
|
||||
|
||||
[[package]]
|
||||
name = "grpc-google-iam-v1"
|
||||
version = "0.14.3"
|
||||
@@ -3041,14 +3077,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "gunicorn"
|
||||
version = "23.0.0"
|
||||
version = "26.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6d/b7/a4a3f632f823e432ce6bc65f62961b7980c898c77f075a2f7118cb3846fe/gunicorn-26.0.0.tar.gz", hash = "sha256:ca9346f85e3a4aeeb64d491045c16b9a35647abd37ea15efe53080eb8b090baf", size = 727286, upload-time = "2026-05-05T06:38:25.529Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/40/9c2384fc2be4ad25dd4a49decd5ad9ea5a3639814c11bd40ab77cb9f0a14/gunicorn-26.0.0-py3-none-any.whl", hash = "sha256:40233d26a5f0d1872916188c276e21641155111c2853f0c2cd55260aec0d24fc", size = 212009, upload-time = "2026-05-05T06:38:23.007Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3150,20 +3186,20 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "iamdata"
|
||||
version = "0.1.202602021"
|
||||
version = "0.1.202605131"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/93/5e/8179963b7a528c548824a8e4088150509d9fa8571dd622b7399f6d2d5680/iamdata-0.1.202602021.tar.gz", hash = "sha256:c24265fc3694076f65da91a8aa9361b60da25f7b8cfd8ba4ddd6aa1b9bb5153e", size = 771233, upload-time = "2026-02-02T05:49:56.76Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cc/ea/d68e25aa4392e8a9f8e6523adc95a5fb86baf98d052efa2cec4d4a00e7ce/iamdata-0.1.202605131.tar.gz", hash = "sha256:ab4e8f1ea080394157848fecd0ca643633e35b2e0cb1965c9ed9bdd673afe00c", size = 793465, upload-time = "2026-05-13T05:57:10.607Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/74/9e/ae7a3019aa5a27d70412b74da4f0304695efa5d9a88f0689f37ea2602ea2/iamdata-0.1.202602021-py3-none-any.whl", hash = "sha256:48419662d75dd0e1ea22b9cc98fd70201d4c72760c6897acc46ad9ab90633d18", size = 1226614, upload-time = "2026-02-02T05:49:54.735Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/93/03396b477b0faa9f1a12142209b59aa13d0fe4f64e2be47883f607789c14/iamdata-0.1.202605131-py3-none-any.whl", hash = "sha256:350e317d96fb8c8ddf30aa6da222788d302af5f13c9e357b59f9eefe50b8af5a", size = 1259166, upload-time = "2026-05-13T05:57:09.093Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.11"
|
||||
version = "3.15"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3966,30 +4002,30 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "numpy"
|
||||
version = "2.0.2"
|
||||
version = "2.2.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137, upload-time = "2024-08-26T20:07:45.345Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552, upload-time = "2024-08-26T20:08:06.666Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957, upload-time = "2024-08-26T20:08:15.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573, upload-time = "2024-08-26T20:08:27.185Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330, upload-time = "2024-08-26T20:08:48.058Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895, upload-time = "2024-08-26T20:09:16.536Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253, upload-time = "2024-08-26T20:09:46.263Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074, upload-time = "2024-08-26T20:10:08.483Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640, upload-time = "2024-08-26T20:10:19.732Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230, upload-time = "2024-08-26T20:10:43.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803, upload-time = "2024-08-26T20:11:13.916Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835, upload-time = "2024-08-26T20:11:34.779Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499, upload-time = "2024-08-26T20:11:43.902Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497, upload-time = "2024-08-26T20:11:55.09Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158, upload-time = "2024-08-26T20:12:14.95Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173, upload-time = "2024-08-26T20:12:44.049Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174, upload-time = "2024-08-26T20:13:13.634Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701, upload-time = "2024-08-26T20:13:34.851Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313, upload-time = "2024-08-26T20:13:45.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179, upload-time = "2024-08-26T20:14:08.786Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4031,7 +4067,7 @@ dependencies = [
|
||||
{ name = "pycryptodomex" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydash" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
@@ -4410,8 +4446,8 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "5.27.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#0abbb7fc590eaf7de6ed354dd5a217bca261d2b0" }
|
||||
version = "5.31.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#b5bb85c9564f6ca6a7f66c851bb56bde719205ee" }
|
||||
dependencies = [
|
||||
{ name = "alibabacloud-actiontrail20200706" },
|
||||
{ name = "alibabacloud-credentials" },
|
||||
@@ -4427,7 +4463,6 @@ dependencies = [
|
||||
{ name = "alibabacloud-tea-openapi" },
|
||||
{ name = "alibabacloud-vpc20160428" },
|
||||
{ name = "alive-progress" },
|
||||
{ name = "awsipranges" },
|
||||
{ name = "azure-identity" },
|
||||
{ name = "azure-keyvault-keys" },
|
||||
{ name = "azure-mgmt-apimanagement" },
|
||||
@@ -4484,9 +4519,14 @@ dependencies = [
|
||||
{ name = "pygithub" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "pytz" },
|
||||
{ name = "scaleway" },
|
||||
{ name = "schema" },
|
||||
{ name = "shodan" },
|
||||
{ name = "slack-sdk" },
|
||||
{ name = "stackit-core" },
|
||||
{ name = "stackit-iaas" },
|
||||
{ name = "stackit-objectstorage" },
|
||||
{ name = "stackit-resourcemanager" },
|
||||
{ name = "tabulate" },
|
||||
{ name = "tzlocal" },
|
||||
{ name = "uuid6" },
|
||||
@@ -4494,7 +4534,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.30.0"
|
||||
version = "1.32.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
@@ -4507,6 +4547,7 @@ dependencies = [
|
||||
{ name = "django-celery-results" },
|
||||
{ name = "django-cors-headers" },
|
||||
{ name = "django-environ" },
|
||||
{ name = "django-eventstream" },
|
||||
{ name = "django-filter" },
|
||||
{ name = "django-guid" },
|
||||
{ name = "django-postgres-extra" },
|
||||
@@ -4571,6 +4612,7 @@ requires-dist = [
|
||||
{ name = "django-celery-results", specifier = "==2.6.0" },
|
||||
{ name = "django-cors-headers", specifier = "==4.4.0" },
|
||||
{ name = "django-environ", specifier = "==0.11.2" },
|
||||
{ name = "django-eventstream", specifier = "==5.3.3" },
|
||||
{ name = "django-filter", specifier = "==24.3" },
|
||||
{ name = "django-guid", specifier = "==3.5.0" },
|
||||
{ name = "django-postgres-extra", specifier = "==2.0.9" },
|
||||
@@ -4583,7 +4625,7 @@ requires-dist = [
|
||||
{ name = "drf-spectacular-jsonapi", specifier = "==0.5.1" },
|
||||
{ name = "fonttools", specifier = "==4.62.1" },
|
||||
{ name = "gevent", specifier = "==25.9.1" },
|
||||
{ name = "gunicorn", specifier = "==23.0.0" },
|
||||
{ name = "gunicorn", specifier = "==26.0.0" },
|
||||
{ name = "h2", specifier = "==4.3.0" },
|
||||
{ name = "lxml", specifier = "==6.1.0" },
|
||||
{ name = "markdown", specifier = "==3.10.2" },
|
||||
@@ -4671,6 +4713,16 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/08/9c66c269b0d417a0af9fb969535f0371b8c538633535a7a6a5ca3f9231e2/psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab", size = 1163864, upload-time = "2023-10-28T09:37:28.155Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pubcontrol"
|
||||
version = "3.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/25/6a/02202a247214a6ffd5148ab1b16aca1c334b40dca211bca0442c8b7c7447/pubcontrol-3.5.0.tar.gz", hash = "sha256:a5ec6b3f53edfd005675518e5e4cc23b34122776835ae7c6dbd1db173d1ff0cb", size = 18199, upload-time = "2023-07-05T19:11:40.477Z" }
|
||||
|
||||
[[package]]
|
||||
name = "py-deviceid"
|
||||
version = "0.1.1"
|
||||
@@ -4682,14 +4734,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "py-iam-expand"
|
||||
version = "0.1.0"
|
||||
version = "0.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "iamdata" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/22/99/8d31a30b37825577275bb3663885b55075fba80257fcd6813b85d3aaffa8/py_iam_expand-0.1.0.tar.gz", hash = "sha256:5a2884dc267ac59a02c3a80fefc0b34c309dac681baa0f87c436067c6cf53a96", size = 10228, upload-time = "2025-04-30T07:15:35.304Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a3/08/f6e11a029b81f0bec4b7b1f18704aadf509a882cc386c90ef1ac043c18cc/py_iam_expand-0.3.0.tar.gz", hash = "sha256:4ccfe25f40ba0633a152c4f86b49cde8972ee3d4b6009b017a4310cc4b9e64c7", size = 10234, upload-time = "2026-02-24T09:47:47.772Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/79/19/482c2e0768cda7afaed07918e4fbd951e2418255fb5d1d9b35b284871716/py_iam_expand-0.1.0-py3-none-any.whl", hash = "sha256:b845ce7b50ac895b02b4f338e09c62a68ea51849794f76e189b02009bd388510", size = 12522, upload-time = "2025-04-30T07:15:33.799Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/dd/4056d0bc3d6317039d2dd2ee7cd6a5389575603e270399a8f9f20e11e721/py_iam_expand-0.3.0-py3-none-any.whl", hash = "sha256:94c0a1e9dd60316ce60ddc0cdc9a046119bde335b5bb9593ee29224857860d5a", size = 12527, upload-time = "2026-02-24T09:47:45.602Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4873,11 +4925,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.12.1"
|
||||
version = "2.13.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -5526,6 +5578,67 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stackit-core"
|
||||
version = "0.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "requests" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/24/90/20f9ec7387eec4067cfd3d29055d0e2b5e1e0322c601a7f48125fd8ea35f/stackit_core-0.2.0.tar.gz", hash = "sha256:b8af91877cdb060d6969a303d8cf20bc0b33b345afd91f679c44a987381e2d47", size = 8987, upload-time = "2025-06-12T08:24:45.251Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/b4/7b53187ce68956870d864ccb9ccfb68066c9df9de1c9568fd2feb03c4504/stackit_core-0.2.0-py3-none-any.whl", hash = "sha256:04632fc6742790d08ddfcb7f2313e04d1254827397a80250f838a2f81b92645b", size = 10240, upload-time = "2025-06-12T08:24:44.214Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stackit-iaas"
|
||||
version = "1.4.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "requests" },
|
||||
{ name = "stackit-core" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/52/07/24e65278300d5c3cb19cb1660bff924c80812cf8aad3e715f826bae5aa80/stackit_iaas-1.4.0.tar.gz", hash = "sha256:93523b23442350c7ebefd9129485c4c2a539f694a9c36a0f8edfaba9862057ea", size = 116236, upload-time = "2026-05-13T09:43:15.996Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/08/51/2201164d7bfacf47539888c735f10f6320c188252384957aa1b23121a210/stackit_iaas-1.4.0-py3-none-any.whl", hash = "sha256:3f4a32321b57ac238f73e5d660c6428186b92cc0425c1f0783ba801e377149d9", size = 316588, upload-time = "2026-05-13T09:43:14.943Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stackit-objectstorage"
|
||||
version = "1.4.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "requests" },
|
||||
{ name = "stackit-core" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/90/80/b790756af40a5c6d979dd688b2557394ac54b594eb4c08edc33157ba890f/stackit_objectstorage-1.4.0.tar.gz", hash = "sha256:4a3812b4de102b199f061706a802909f9e53ae9b0858769d5bd720f814c8bdbe", size = 31814, upload-time = "2026-05-13T09:43:05.027Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/68/f1/ffa8d5e2ec9f818c72a6f045691364eb4e927ee86641993a70882d00205a/stackit_objectstorage-1.4.0-py3-none-any.whl", hash = "sha256:1a3285c6840d95cff591d84fd21803575cb0d010c398e6575ed92987b9c39866", size = 65061, upload-time = "2026-05-13T09:43:04.13Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stackit-resourcemanager"
|
||||
version = "0.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "requests" },
|
||||
{ name = "stackit-core" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/23/2d/f458f18e48ed2b1c83df52cff7dbdfd5dd904fb2980ffd9385876e47bbd9/stackit_resourcemanager-0.8.0.tar.gz", hash = "sha256:f44542beab4130857f5a7f465cf02defeef657bdf63c1beeb3102f0ba3c003fe", size = 33943, upload-time = "2026-05-13T09:43:08.667Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/9c/38a74d0f7a89b4320f6d2366fb660638bda8860daa08748b12c713d84381/stackit_resourcemanager-0.8.0-py3-none-any.whl", hash = "sha256:dd04bb8353d041a137c4dcba190beabded7acfaff1bc98b218fce20a99389ebc", size = 81288, upload-time = "2026-05-13T09:43:07.81Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "statsd"
|
||||
version = "4.0.1"
|
||||
@@ -5785,7 +5898,7 @@ source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3c/2f/99fb8718274116c5c146c745755620fd5c5943f78ca52ca9b17e94348286/workos-6.0.4.tar.gz", hash = "sha256:b0bfe8fd212b8567422c4ea3732eb33608794033eb3a69900c6b04db183c32d6", size = 172217, upload-time = "2026-04-16T03:09:28.583Z" }
|
||||
wheels = [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Build command
|
||||
# docker build --platform=linux/amd64 --no-cache -t prowler:latest .
|
||||
|
||||
ARG PROWLER_VERSION=latest
|
||||
ARG PROWLER_VERSION=latest@sha256:4b796c6df40a3350c7947747b59bdda230d0da6222287500e13b0a8e1574aad4
|
||||
|
||||
FROM toniblyx/prowler:${PROWLER_VERSION}
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ from typing import Optional
|
||||
from prowler.lib.logger import logger
|
||||
from lib.models import ConnectivityGraph
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -28,7 +28,7 @@ containers:
|
||||
image:
|
||||
repository: prowlercloud/prowler-api
|
||||
pullPolicy: IfNotPresent
|
||||
command: ["../docker-entrypoint.sh", "beat"]
|
||||
command: ["/home/prowler/docker-entrypoint.sh", "beat"]
|
||||
|
||||
secrets:
|
||||
POSTGRES_HOST:
|
||||
|
||||
@@ -440,7 +440,7 @@ worker_beat:
|
||||
tag: ""
|
||||
|
||||
command:
|
||||
- ../docker-entrypoint.sh
|
||||
- /home/prowler/docker-entrypoint.sh
|
||||
args:
|
||||
- beat
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
image: nginx:alpine@sha256:8b1e78743a03dbb2c95171cc58639fef29abc8816598e27fb910ed2e621e589a
|
||||
container_name: prowler-nginx
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
|
||||
@@ -1538,6 +1538,186 @@ def get_section_container_iso(data, section_1, section_2):
|
||||
return html.Div(section_containers, className="compliance-data-layout")
|
||||
|
||||
|
||||
def _status_bar(success, failed, classname):
|
||||
"""Build the stacked PASS/FAIL bar shown next to an accordion title."""
|
||||
fig = go.Figure(
|
||||
data=[
|
||||
go.Bar(
|
||||
name="Failed",
|
||||
x=[failed],
|
||||
y=[""],
|
||||
orientation="h",
|
||||
marker=dict(color="#e77676"),
|
||||
width=[0.8],
|
||||
),
|
||||
go.Bar(
|
||||
name="Success",
|
||||
x=[success],
|
||||
y=[""],
|
||||
orientation="h",
|
||||
marker=dict(color="#45cc6e"),
|
||||
width=[0.8],
|
||||
),
|
||||
]
|
||||
)
|
||||
fig.update_layout(
|
||||
barmode="stack",
|
||||
margin=dict(l=10, r=10, t=10, b=10),
|
||||
paper_bgcolor="rgba(0,0,0,0)",
|
||||
plot_bgcolor="rgba(0,0,0,0)",
|
||||
showlegend=False,
|
||||
width=350,
|
||||
height=30,
|
||||
xaxis=dict(showticklabels=False, showgrid=False, zeroline=False),
|
||||
yaxis=dict(showticklabels=False, showgrid=False, zeroline=False),
|
||||
annotations=[
|
||||
dict(
|
||||
x=success + failed,
|
||||
y=0,
|
||||
xref="x",
|
||||
yref="y",
|
||||
text=str(success),
|
||||
showarrow=False,
|
||||
font=dict(color="#45cc6e", size=14),
|
||||
xanchor="left",
|
||||
yanchor="middle",
|
||||
),
|
||||
dict(
|
||||
x=0,
|
||||
y=0,
|
||||
xref="x",
|
||||
yref="y",
|
||||
text=str(failed),
|
||||
showarrow=False,
|
||||
font=dict(color="#e77676", size=14),
|
||||
xanchor="right",
|
||||
yanchor="middle",
|
||||
),
|
||||
],
|
||||
)
|
||||
fig.add_annotation(
|
||||
x=failed,
|
||||
y=0.3,
|
||||
text="|",
|
||||
showarrow=False,
|
||||
xanchor="center",
|
||||
yanchor="middle",
|
||||
font=dict(size=20),
|
||||
)
|
||||
return dcc.Graph(figure=fig, config={"staticPlot": True}, className=classname)
|
||||
|
||||
|
||||
def get_section_containers_generic(data, section_col, id_col):
|
||||
"""Two-level view: section -> requirement id (+ description) -> checks.
|
||||
|
||||
Sorts lexicographically so arbitrary requirement IDs never crash the
|
||||
version-aware sort used by the CIS renderer.
|
||||
"""
|
||||
data["STATUS"] = data["STATUS"].apply(map_status_to_icon)
|
||||
data[section_col] = data[section_col].astype(str)
|
||||
data[id_col] = data[id_col].astype(str)
|
||||
data.sort_values(by=[section_col, id_col], inplace=True)
|
||||
|
||||
counts_section = data.groupby([section_col, "STATUS"]).size().unstack(fill_value=0)
|
||||
counts_id = (
|
||||
data.groupby([section_col, id_col, "STATUS"]).size().unstack(fill_value=0)
|
||||
)
|
||||
|
||||
def count(counts, key, emoji):
|
||||
return counts.loc[key, emoji] if emoji in counts.columns else 0
|
||||
|
||||
has_description = "REQUIREMENTS_DESCRIPTION" in data.columns
|
||||
table_cols = ["CHECKID", "STATUS", "REGION", "ACCOUNTID", "RESOURCEID"]
|
||||
|
||||
section_containers = []
|
||||
for section in data[section_col].unique():
|
||||
graph_div = html.Div(
|
||||
_status_bar(
|
||||
count(counts_section, section, pass_emoji),
|
||||
count(counts_section, section, fail_emoji),
|
||||
"info-bar",
|
||||
),
|
||||
className="graph-section",
|
||||
)
|
||||
|
||||
internal_items = []
|
||||
for req_id in data[data[section_col] == section][id_col].unique():
|
||||
specific_data = data[
|
||||
(data[section_col] == section) & (data[id_col] == req_id)
|
||||
]
|
||||
data_table = dash_table.DataTable(
|
||||
data=specific_data.to_dict("records"),
|
||||
columns=[
|
||||
{"name": i, "id": i}
|
||||
for i in table_cols
|
||||
if i in specific_data.columns
|
||||
],
|
||||
style_table={"overflowX": "auto"},
|
||||
style_as_list_view=True,
|
||||
style_cell={"textAlign": "left", "padding": "5px"},
|
||||
)
|
||||
graph_div_req = html.Div(
|
||||
_status_bar(
|
||||
count(counts_id, (section, req_id), pass_emoji),
|
||||
count(counts_id, (section, req_id), fail_emoji),
|
||||
"info-bar-child",
|
||||
),
|
||||
className="graph-section-req",
|
||||
)
|
||||
|
||||
title = req_id
|
||||
if has_description:
|
||||
title = (
|
||||
f"{req_id} - {specific_data['REQUIREMENTS_DESCRIPTION'].iloc[0]}"
|
||||
)
|
||||
if len(title) > 130:
|
||||
title = title[:130] + " ..."
|
||||
|
||||
internal_items.append(
|
||||
html.Div(
|
||||
[
|
||||
graph_div_req,
|
||||
dbc.Accordion(
|
||||
[
|
||||
dbc.AccordionItem(
|
||||
title=title,
|
||||
children=[
|
||||
html.Div(
|
||||
[data_table],
|
||||
className="inner-accordion-content",
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
start_collapsed=True,
|
||||
flush=True,
|
||||
),
|
||||
],
|
||||
className="accordion-inner--child",
|
||||
)
|
||||
)
|
||||
|
||||
section_containers.append(
|
||||
html.Div(
|
||||
[
|
||||
graph_div,
|
||||
dbc.Accordion(
|
||||
[
|
||||
dbc.AccordionItem(
|
||||
title=f"{section}", children=internal_items
|
||||
)
|
||||
],
|
||||
start_collapsed=True,
|
||||
flush=True,
|
||||
),
|
||||
],
|
||||
className="accordion-inner",
|
||||
)
|
||||
)
|
||||
|
||||
return html.Div(section_containers, className="compliance-data-layout")
|
||||
|
||||
|
||||
def get_section_containers_format4(data, section_1):
|
||||
|
||||
data["STATUS"] = data["STATUS"].apply(map_status_to_icon)
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import warnings
|
||||
|
||||
from dashboard.common_methods import get_section_containers_3_levels
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def get_table(data):
|
||||
aux = data[
|
||||
[
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"NAME",
|
||||
"CHECKID",
|
||||
"STATUS",
|
||||
"REGION",
|
||||
"ACCOUNTID",
|
||||
"RESOURCEID",
|
||||
]
|
||||
]
|
||||
|
||||
return get_section_containers_3_levels(
|
||||
aux,
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"NAME",
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
import warnings
|
||||
|
||||
from dashboard.common_methods import (
|
||||
get_section_containers_format4,
|
||||
get_section_containers_generic,
|
||||
)
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def get_table(data):
|
||||
# Discover REQUIREMENTS_ATTRIBUTES_* columns at runtime.
|
||||
attr_cols = [c for c in data.columns if c.startswith("REQUIREMENTS_ATTRIBUTES_")]
|
||||
|
||||
# Section column (in priority order):
|
||||
# 1. REQUIREMENTS_ATTRIBUTES_SECTION — most common convention
|
||||
# 2. First discovered attribute column — covers novel schemas
|
||||
# 3. None — no section, group flat by requirement id
|
||||
if "REQUIREMENTS_ATTRIBUTES_SECTION" in attr_cols:
|
||||
section_col = "REQUIREMENTS_ATTRIBUTES_SECTION"
|
||||
elif attr_cols:
|
||||
section_col = attr_cols[0]
|
||||
else:
|
||||
section_col = None
|
||||
|
||||
base_cols = [
|
||||
"REQUIREMENTS_ID",
|
||||
"REQUIREMENTS_DESCRIPTION",
|
||||
"STATUS",
|
||||
"CHECKID",
|
||||
"REGION",
|
||||
"ACCOUNTID",
|
||||
"RESOURCEID",
|
||||
]
|
||||
|
||||
# Two levels (section -> requirement id) when a section distinct from the
|
||||
# id exists; otherwise group flat by requirement id.
|
||||
if section_col and section_col != "REQUIREMENTS_ID":
|
||||
needed = [section_col] + base_cols
|
||||
aux = data[[c for c in needed if c in data.columns]].copy()
|
||||
return get_section_containers_generic(aux, section_col, "REQUIREMENTS_ID")
|
||||
|
||||
aux = data[[c for c in base_cols if c in data.columns]].copy()
|
||||
return get_section_containers_format4(aux, "REQUIREMENTS_ID")
|
||||
@@ -156,7 +156,7 @@ def create_layout_compliance(
|
||||
html.Img(src="assets/favicon.ico", className="w-5 mr-3"),
|
||||
html.Span("Subscribe to Prowler Cloud"),
|
||||
],
|
||||
href="https://prowler.pro/",
|
||||
href="https://cloud.prowler.com/",
|
||||
target="_blank",
|
||||
className="text-prowler-stone-900 inline-flex px-4 py-2 text-xs font-bold uppercase transition-all rounded-lg text-gray-900 hover:bg-prowler-stone-900/10 border-solid border-1 hover:border-prowler-stone-900/10 hover:border-solid hover:border-1 border-prowler-stone-900/10",
|
||||
),
|
||||
|
||||
@@ -215,6 +215,58 @@ else:
|
||||
)
|
||||
|
||||
|
||||
def _ensure_scope_columns(data):
|
||||
"""Guarantee ACCOUNTID and REGION exist.
|
||||
|
||||
Scope columns always sit between DESCRIPTION and ASSESSMENTDATE, so derive
|
||||
them positionally for any provider (e.g. Okta's ORGANIZATIONDOMAIN) and
|
||||
fall back to "-" to avoid a KeyError.
|
||||
"""
|
||||
cols = list(data.columns)
|
||||
scope = []
|
||||
if "DESCRIPTION" in cols and "ASSESSMENTDATE" in cols:
|
||||
start, end = cols.index("DESCRIPTION") + 1, cols.index("ASSESSMENTDATE")
|
||||
scope = [c for c in cols[start:end] if c not in ("ACCOUNTID", "REGION")]
|
||||
|
||||
if "ACCOUNTID" not in data.columns:
|
||||
if scope:
|
||||
data.rename(columns={scope.pop(0): "ACCOUNTID"}, inplace=True)
|
||||
else:
|
||||
data["ACCOUNTID"] = "-"
|
||||
if "REGION" not in data.columns:
|
||||
if scope:
|
||||
data.rename(columns={scope.pop(0): "REGION"}, inplace=True)
|
||||
else:
|
||||
data["REGION"] = "-"
|
||||
return data
|
||||
|
||||
|
||||
def _dispatch_compliance_renderer(data, analytics_input):
|
||||
"""Resolve the compliance renderer module and return (table, deduped_data).
|
||||
|
||||
Tries to import the framework-specific builtin module. On
|
||||
ModuleNotFoundError (dynamic/external provider with no dedicated module),
|
||||
falls back to the generic renderer. Any other ImportError is re-raised.
|
||||
get_table() is called OUTSIDE the try block so errors inside the renderer
|
||||
surface as real exceptions rather than being swallowed.
|
||||
"""
|
||||
current = analytics_input.replace(".", "_")
|
||||
target = f"dashboard.compliance.{current}"
|
||||
try:
|
||||
module = importlib.import_module(target)
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name != target:
|
||||
raise
|
||||
from dashboard.compliance import generic as module
|
||||
dedup_columns = ["CHECKID", "STATUS", "RESOURCEID", "STATUSEXTENDED"]
|
||||
if "MUTED" in data.columns:
|
||||
dedup_columns.insert(2, "MUTED")
|
||||
data = data.drop_duplicates(subset=dedup_columns)
|
||||
if "threatscore" in analytics_input:
|
||||
data = get_threatscore_mean_by_pillar(data)
|
||||
return module.get_table(data), data
|
||||
|
||||
|
||||
@callback(
|
||||
[
|
||||
Output("output", "children"),
|
||||
@@ -292,7 +344,7 @@ def display_data(
|
||||
data.rename(columns={"TENANCYID": "ACCOUNTID"}, inplace=True)
|
||||
|
||||
# Filter the chosen level of the CIS
|
||||
if is_level_1:
|
||||
if is_level_1 and "REQUIREMENTS_ATTRIBUTES_PROFILE" in data.columns:
|
||||
data = data[data["REQUIREMENTS_ATTRIBUTES_PROFILE"].str.contains("Level 1")]
|
||||
|
||||
# Rename the column PROJECTID to ACCOUNTID for GCP
|
||||
@@ -314,6 +366,9 @@ def display_data(
|
||||
data.rename(columns={"SUBSCRIPTION": "ACCOUNTID"}, inplace=True)
|
||||
data["REGION"] = "-"
|
||||
|
||||
# Normalize scope columns for any remaining (e.g. dynamic) provider.
|
||||
data = _ensure_scope_columns(data)
|
||||
|
||||
# Filter ACCOUNT
|
||||
if account_filter == ["All"]:
|
||||
updated_cloud_account_values = data["ACCOUNTID"].unique()
|
||||
@@ -409,36 +464,7 @@ def display_data(
|
||||
# Check cases where the compliance start with AWS_
|
||||
if "aws_" in analytics_input:
|
||||
analytics_input = analytics_input + "_aws"
|
||||
try:
|
||||
current = analytics_input.replace(".", "_")
|
||||
compliance_module = importlib.import_module(
|
||||
f"dashboard.compliance.{current}"
|
||||
)
|
||||
# Build subset list based on available columns
|
||||
dedup_columns = ["CHECKID", "STATUS", "RESOURCEID", "STATUSEXTENDED"]
|
||||
if "MUTED" in data.columns:
|
||||
dedup_columns.insert(2, "MUTED")
|
||||
data = data.drop_duplicates(subset=dedup_columns)
|
||||
|
||||
if "threatscore" in analytics_input:
|
||||
data = get_threatscore_mean_by_pillar(data)
|
||||
|
||||
table = compliance_module.get_table(data)
|
||||
except ModuleNotFoundError:
|
||||
table = html.Div(
|
||||
[
|
||||
html.H5(
|
||||
"No data found for this compliance",
|
||||
className="card-title",
|
||||
style={"text-align": "left", "color": "black"},
|
||||
)
|
||||
],
|
||||
style={
|
||||
"width": "99%",
|
||||
"margin-right": "0.8%",
|
||||
"margin-bottom": "10px",
|
||||
},
|
||||
)
|
||||
table, data = _dispatch_compliance_renderer(data, analytics_input)
|
||||
|
||||
df = data.copy()
|
||||
# Remove Muted rows
|
||||
|
||||
@@ -1538,7 +1538,7 @@ def filter_data(
|
||||
html.Img(src="assets/favicon.ico", className="w-5 mr-3"),
|
||||
html.Span("Subscribe to Prowler Cloud"),
|
||||
],
|
||||
href="https://prowler.pro/",
|
||||
href="https://cloud.prowler.com/",
|
||||
target="_blank",
|
||||
className="text-prowler-stone-900 inline-flex px-4 py-2 text-xs font-bold uppercase transition-all rounded-lg text-gray-900 hover:bg-prowler-stone-900/10 border-solid border-1 hover:border-prowler-stone-900/10 hover:border-solid hover:border-1 border-prowler-stone-900/10",
|
||||
),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
api-dev-init:
|
||||
image: busybox:1.37.0
|
||||
image: busybox:1.37.0@sha256:9532d8c39891ca2ecde4d30d7710e01fb739c87a8b9299685c63704296b16028
|
||||
volumes:
|
||||
- ./_data/api:/data
|
||||
command: ["sh", "-c", "chown -R 1000:1000 /data"]
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
condition: service_healthy
|
||||
|
||||
postgres:
|
||||
image: postgres:16.3-alpine3.20
|
||||
image: postgres:16.3-alpine3.20@sha256:36ed71227ae36305d26382657c0b96cbaf298427b3f1eaeb10d77a6dea3eec41
|
||||
hostname: "postgres-db"
|
||||
volumes:
|
||||
- ./_data/postgres:/var/lib/postgresql/data
|
||||
@@ -88,7 +88,7 @@ services:
|
||||
retries: 5
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine3.19
|
||||
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
|
||||
hostname: "valkey"
|
||||
volumes:
|
||||
- ./_data/valkey:/data
|
||||
@@ -104,7 +104,7 @@ services:
|
||||
retries: 3
|
||||
|
||||
neo4j:
|
||||
image: graphstack/dozerdb:5.26.3.0
|
||||
image: graphstack/dozerdb:5.26.3.0@sha256:a77526ea3918fdc46d1fff70c4aea7d71d3874a26ecec059179d6775845b1247
|
||||
hostname: "neo4j"
|
||||
volumes:
|
||||
- ./_data/neo4j:/data
|
||||
@@ -139,6 +139,8 @@ services:
|
||||
|
||||
worker-dev:
|
||||
image: prowler-api-dev
|
||||
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
|
||||
stop_grace_period: 120s
|
||||
build:
|
||||
context: ./api
|
||||
dockerfile: Dockerfile
|
||||
@@ -183,7 +185,7 @@ services:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
entrypoint:
|
||||
- "../docker-entrypoint.sh"
|
||||
- "/home/prowler/docker-entrypoint.sh"
|
||||
- "beat"
|
||||
|
||||
mcp-server:
|
||||
|
||||
+7
-5
@@ -6,7 +6,7 @@
|
||||
#
|
||||
services:
|
||||
api-init:
|
||||
image: busybox:1.37.0
|
||||
image: busybox:1.37.0@sha256:9532d8c39891ca2ecde4d30d7710e01fb739c87a8b9299685c63704296b16028
|
||||
volumes:
|
||||
- ./_data/api:/data
|
||||
command: ["sh", "-c", "chown -R 1000:1000 /data"]
|
||||
@@ -60,7 +60,7 @@ services:
|
||||
start_period: 60s
|
||||
|
||||
postgres:
|
||||
image: postgres:16.3-alpine3.20
|
||||
image: postgres:16.3-alpine3.20@sha256:36ed71227ae36305d26382657c0b96cbaf298427b3f1eaeb10d77a6dea3eec41
|
||||
hostname: "postgres-db"
|
||||
volumes:
|
||||
- ./_data/postgres:/var/lib/postgresql/data
|
||||
@@ -80,7 +80,7 @@ services:
|
||||
retries: 5
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine3.19
|
||||
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
|
||||
hostname: "valkey"
|
||||
volumes:
|
||||
- ./_data/valkey:/data
|
||||
@@ -96,7 +96,7 @@ services:
|
||||
retries: 3
|
||||
|
||||
neo4j:
|
||||
image: graphstack/dozerdb:5.26.3.0
|
||||
image: graphstack/dozerdb:5.26.3.0@sha256:a77526ea3918fdc46d1fff70c4aea7d71d3874a26ecec059179d6775845b1247
|
||||
hostname: "neo4j"
|
||||
volumes:
|
||||
- ./_data/neo4j:/data
|
||||
@@ -129,6 +129,8 @@ services:
|
||||
|
||||
worker:
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
|
||||
stop_grace_period: 120s
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
@@ -158,7 +160,7 @@ services:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
entrypoint:
|
||||
- "../docker-entrypoint.sh"
|
||||
- "/home/prowler/docker-entrypoint.sh"
|
||||
- "beat"
|
||||
|
||||
mcp-server:
|
||||
|
||||
@@ -8,7 +8,77 @@ This guide explains the AI Skills system that provides on-demand context and pat
|
||||
**What are AI Skills?** Skills are structured instructions that help AI agents (Claude Code, Cursor, Copilot, etc.) understand Prowler's conventions, patterns, and best practices.
|
||||
</Info>
|
||||
|
||||
## Architecture Overview
|
||||
Skills live in the [`skills/`](https://github.com/prowler-cloud/prowler/tree/master/skills) directory of the Prowler OSS repository. Each skill is a folder containing a `SKILL.md` file with its patterns and metadata.
|
||||
|
||||
## Installation
|
||||
|
||||
To enable skills for the supported AI coding assistants, run the setup script from the repository root:
|
||||
|
||||
```bash
|
||||
./skills/setup.sh
|
||||
```
|
||||
|
||||
The script creates symlinks so each tool finds the skills in its expected location:
|
||||
|
||||
| Tool | Created by setup |
|
||||
|------|------------------|
|
||||
| Claude Code | `.claude/skills/` symlink and `CLAUDE.md` |
|
||||
| Gemini CLI | `.gemini/skills/` symlink and `GEMINI.md` |
|
||||
| Codex (OpenAI) | `.codex/skills/` symlink (uses `AGENTS.md` natively) |
|
||||
| GitHub Copilot | `.github/copilot-instructions.md` symlink to `AGENTS.md` |
|
||||
|
||||
After running the setup, restart the AI coding assistant to load the skills.
|
||||
|
||||
## Using Skills
|
||||
|
||||
AI agents discover skills automatically and load them when a request matches a skill trigger. To load a skill manually during a session, point the agent to the skill's `SKILL.md` file:
|
||||
|
||||
```text
|
||||
Read skills/{skill-name}/SKILL.md
|
||||
```
|
||||
|
||||
For the full list of available skills, their triggers, and the Auto-invoke mappings, see the [`skills/README.md`](https://github.com/prowler-cloud/prowler/blob/master/skills/README.md) and [`AGENTS.md`](https://github.com/prowler-cloud/prowler/blob/master/AGENTS.md) in the repository.
|
||||
|
||||
## Available Skills
|
||||
|
||||
| Type | Skills |
|
||||
|------|--------|
|
||||
| **Generic** | typescript, react-19, nextjs-16, tailwind-4, pytest, playwright, django-drf, zod-4, zustand-5, ai-sdk-5, vitest, tdd |
|
||||
| **Prowler** | prowler, prowler-sdk-check, prowler-api, prowler-ui, prowler-mcp, prowler-provider, prowler-compliance, prowler-compliance-review, prowler-docs, prowler-pr, prowler-ci, prowler-attack-paths-query |
|
||||
| **Testing** | prowler-test-sdk, prowler-test-api, prowler-test-ui |
|
||||
| **Meta** | skill-creator, skill-sync |
|
||||
|
||||
<Note>
|
||||
This table is a snapshot. The repository is the source of truth: see [`skills/README.md`](https://github.com/prowler-cloud/prowler/blob/master/skills/README.md) for the current, complete list.
|
||||
</Note>
|
||||
|
||||
## Skill Structure
|
||||
|
||||
Each skill follows the [Agent Skills spec](https://agentskills.io):
|
||||
|
||||
```text
|
||||
skills/{skill-name}/
|
||||
├── SKILL.md # Patterns, rules, decision trees
|
||||
├── assets/ # Code templates, schemas
|
||||
└── references/ # Links to local docs (single source of truth)
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Self-contained skills** - Critical patterns inline for fast loading
|
||||
2. **Local doc references** - No web URLs, points to `docs/developer-guide/*.mdx`
|
||||
3. **Single source of truth** - Skills reference docs, no duplication
|
||||
4. **On-demand loading** - AI loads only what's needed for the task
|
||||
|
||||
## Creating New Skills
|
||||
|
||||
Use the `skill-creator` meta-skill to create new skills that follow the Agent Skills spec. See [`AGENTS.md`](https://github.com/prowler-cloud/prowler/blob/master/AGENTS.md) for the full list of available skills and their triggers.
|
||||
|
||||
## How Skills Work
|
||||
|
||||
The diagrams below explain the internals of the skill system. They are useful for understanding the design, but are not required to install or use skills.
|
||||
|
||||
### Architecture Overview
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
@@ -28,7 +98,7 @@ graph LR
|
||||
style F fill:#1a4d2e,stroke:#66bb6a,color:#fff
|
||||
```
|
||||
|
||||
## How It Works
|
||||
### Request Lifecycle
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
@@ -68,7 +138,7 @@ sequenceDiagram
|
||||
A->>U: Creates check with correct patterns
|
||||
```
|
||||
|
||||
## Before vs After
|
||||
### With and Without Skills
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
@@ -96,7 +166,7 @@ graph TD
|
||||
style AFTER fill:#1a4d1a,stroke:#66bb6a,color:#fff
|
||||
```
|
||||
|
||||
## Complete Architecture
|
||||
### Full Component Map
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
@@ -110,7 +180,7 @@ flowchart TB
|
||||
subgraph GENERIC["Generic Skills"]
|
||||
G1["typescript"]
|
||||
G2["react-19"]
|
||||
G3["nextjs-15"]
|
||||
G3["nextjs-16"]
|
||||
G4["tailwind-4"]
|
||||
G5["pytest"]
|
||||
G6["playwright"]
|
||||
@@ -186,34 +256,3 @@ flowchart TB
|
||||
style STRUCTURE fill:#5c3d1a,stroke:#ffb74d,color:#fff
|
||||
style DOCS fill:#1a3d4d,stroke:#4dd0e1,color:#fff
|
||||
```
|
||||
|
||||
## Skills Included
|
||||
|
||||
| Type | Skills |
|
||||
|------|--------|
|
||||
| **Generic** | typescript, react-19, nextjs-15, tailwind-4, pytest, playwright, django-drf, zod-4, zustand-5, ai-sdk-5 |
|
||||
| **Prowler** | prowler, prowler-sdk-check, prowler-api, prowler-ui, prowler-mcp, prowler-provider, prowler-compliance, prowler-compliance-review, prowler-docs, prowler-pr, prowler-ci |
|
||||
| **Testing** | prowler-test-sdk, prowler-test-api, prowler-test-ui |
|
||||
| **Meta** | skill-creator, skill-sync |
|
||||
|
||||
## Skill Structure
|
||||
|
||||
Each skill follows the [Agent Skills spec](https://agentskills.io):
|
||||
|
||||
```
|
||||
skills/{skill-name}/
|
||||
├── SKILL.md # Patterns, rules, decision trees
|
||||
├── assets/ # Code templates, schemas
|
||||
└── references/ # Links to local docs (single source of truth)
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Self-contained skills** - Critical patterns inline for fast loading
|
||||
2. **Local doc references** - No web URLs, points to `docs/developer-guide/*.mdx`
|
||||
3. **Single source of truth** - Skills reference docs, no duplication
|
||||
4. **On-demand loading** - AI loads only what's needed for the task
|
||||
|
||||
## Creating New Skills
|
||||
|
||||
Use the `skill-creator` meta-skill to create new skills that follow the Agent Skills spec. See `AGENTS.md` for the full list of available skills and their triggers.
|
||||
|
||||
@@ -2,40 +2,228 @@
|
||||
title: 'Creating a New Security Compliance Framework in Prowler'
|
||||
---
|
||||
|
||||
This guide explains how to add a new security compliance framework to Prowler, end to end. It covers directory layout, the JSON schema, check mapping conventions, the Pydantic models that validate each framework, the CSV output formatter, local validation, testing, and the pull request process.
|
||||
This guide explains how to add a new security compliance framework to Prowler, end to end. It covers directory layout, the two supported JSON schemas (universal and legacy), the Pydantic models that validate each framework, check mapping conventions, output formatting, local validation, testing, and the pull request process.
|
||||
|
||||
## Introduction
|
||||
|
||||
A compliance framework in Prowler maps a public or custom control catalog (for example CIS, NIST 800-53, PCI DSS, HIPAA, ENS, CCC) to the security checks that Prowler already runs. Each requirement links to zero, one or more Prowler checks. When a scan executes, findings are aggregated per requirement to produce the compliance report rendered by Prowler CLI and Prowler Cloud.
|
||||
A compliance framework in Prowler maps a public or custom control catalog (for example CIS, NIST 800-53, PCI DSS, HIPAA, ENS, CCC, DORA) to the security checks that Prowler already runs. Each requirement links to zero, one or more Prowler checks. When a scan executes, findings are aggregated per requirement to produce the compliance report rendered by Prowler CLI and Prowler Cloud.
|
||||
|
||||
Prowler ships with 85+ compliance frameworks across All Providers. The catalog lives under `prowler/compliance/<provider>/` (or `prowler/compliance/` for universal compliance frameworks)
|
||||
Prowler ships 85+ compliance frameworks across all providers. The catalog lives under `prowler/compliance/<provider>/` (legacy, per-provider) or `prowler/compliance/` (universal, multi-provider).
|
||||
|
||||
<Warning>
|
||||
A compliance framework must represent the **complete state** of the source catalog. Every requirement defined by the framework has to be present in the JSON file, even when none of the existing Prowler checks can automate it. In that case, leave `Checks` as an empty array, but do not omit the requirement.
|
||||
A compliance framework must represent the **complete state** of the source catalog. Every requirement defined by the framework has to be present in the JSON file, even when no Prowler check can automate it. In that case, leave the requirement's check list empty, but do not omit the requirement.
|
||||
|
||||
Requirement coverage feeds the compliance percentage calculations and the metadata surfaces (dashboards, widgets, exports). Missing requirements skew those metrics and break the report as a faithful snapshot of the framework.
|
||||
</Warning>
|
||||
|
||||
### Two supported schemas
|
||||
|
||||
| Schema | When to use | File location | Discovered as |
|
||||
| --- | --- | --- | --- |
|
||||
| **Universal (recommended for new frameworks)** | Multi-provider frameworks, or single-provider frameworks that benefit from declarative table/PDF rendering | `prowler/compliance/<framework>.json` (top-level) | Available for **every** provider whose key appears in any `requirement.checks` dict |
|
||||
| **Legacy provider-specific** | Single-provider frameworks with framework-specific attribute classes already declared in the codebase (CIS, ENS, ISO 27001, etc.) | `prowler/compliance/<provider>/<framework>_<version>_<provider>.json` | Available only under that provider |
|
||||
|
||||
Auto-discovery happens in `get_bulk_compliance_frameworks_universal(provider)` (`prowler/lib/check/compliance_models.py:915`), which scans **both** the top-level `prowler/compliance/` directory and every per-provider sub-directory. Legacy frameworks are transparently converted to the universal `ComplianceFramework` model via `adapt_legacy_to_universal()` before being returned, so the rest of Prowler — CLI table rendering, CSV/OCSF outputs, PDF generation — works the same regardless of the source schema.
|
||||
|
||||
> The legacy entry-point `Compliance.get_bulk(provider)` (used by older code paths) only scans per-provider sub-directories. Universal top-level files are picked up exclusively via the universal loader; this matters if you are wiring a new code path against the legacy API.
|
||||
|
||||
For **new** frameworks, prefer the universal schema: it requires no Python code changes, supports multiple providers in a single file, and table/PDF rendering is driven entirely from declarative configuration inside the JSON.
|
||||
|
||||
> All Pydantic models in `compliance_models.py` are imported from `pydantic.v1`. Subclasses you add for the legacy schema must use `from pydantic.v1 import BaseModel`.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Before adding a new framework, complete the following checks:
|
||||
|
||||
- **Verify the framework is not already supported.** Inspect `prowler/compliance/<provider>/` for an existing JSON file matching the name and version.
|
||||
- **Verify the framework is not already supported.** Inspect `prowler/compliance/` and every `prowler/compliance/<provider>/` for an existing JSON file matching the name and version.
|
||||
- **Confirm the required checks exist.** Every requirement that can be automated must point to one or more existing Prowler checks. For each missing check, implement it first by following the [Prowler Checks](/developer-guide/checks) guide.
|
||||
- **Review a reference framework.** Use an existing framework with a similar structure as your template. `cis_2.0_aws.json` is the canonical reference for CIS-style frameworks. `ccc_aws.json`, `ens_rd2022_aws.json`, and `nist_800_53_revision_5_aws.json` illustrate other attribute shapes.
|
||||
- **Review a reference framework.** Use an existing framework with a similar structure as your template:
|
||||
- Universal: `prowler/compliance/dora.json`, `prowler/compliance/csa_ccm_4.0.json`.
|
||||
- Legacy: `prowler/compliance/aws/cis_2.0_aws.json` (canonical CIS shape), `prowler/compliance/aws/ccc_aws.json`, `prowler/compliance/aws/ens_rd2022_aws.json`, `prowler/compliance/aws/nist_800_53_revision_5_aws.json`.
|
||||
|
||||
## Four-Layer Architecture
|
||||
## Universal Compliance Framework
|
||||
|
||||
A compliance framework spans four layers. A complete contribution must touch each layer that applies.
|
||||
### Where the file lives
|
||||
|
||||
- **Layer 1 – Schema validation:** The Pydantic models in `prowler/lib/check/compliance_models.py` define the canonical schema for each attribute shape (CIS, ENS, Mitre, CCC, C5, CSA CCM, ISO 27001, KISA ISMS-P, AWS Well-Architected, Prowler ThreatScore, and a generic fallback).
|
||||
- **Layer 2 – JSON catalog:** The framework JSON file in `prowler/compliance/<provider>/` lists every requirement and maps it to checks.
|
||||
- **Layer 3 – Output formatter:** The Python module in `prowler/lib/outputs/compliance/<framework>/` builds the CSV row model, the per-provider transformer, and the CLI summary table.
|
||||
- **Layer 4 – Output dispatchers:** The dispatchers in `prowler/lib/outputs/compliance/compliance.py` and `prowler/lib/outputs/compliance/compliance_output.py` route findings to the right formatter based on the framework identifier.
|
||||
Place the file at the top level of the compliance directory:
|
||||
|
||||
The rest of this guide walks each layer in order.
|
||||
```
|
||||
prowler/compliance/<framework_name>.json
|
||||
```
|
||||
|
||||
## Directory Structure and File Naming
|
||||
Examples in the repository: `prowler/compliance/csa_ccm_4.0.json`, `prowler/compliance/dora.json`.
|
||||
|
||||
The file is auto-discovered — there is **no** need to register it in any `__init__.py`, modify `prowler/lib/outputs/`, or update any other Python module. The framework key Prowler CLI accepts via `--compliance` is the basename of the JSON file without `.json` (`dora.json` → `dora`).
|
||||
|
||||
### Top-level structure
|
||||
|
||||
```json
|
||||
{
|
||||
"framework": "<short identifier, e.g. \"DORA\" or \"CSA-CCM\">",
|
||||
"name": "<human-readable full name>",
|
||||
"version": "<framework version>",
|
||||
"description": "<one-paragraph description shown in --list-compliance and PDF reports>",
|
||||
"icon": "<short icon slug, optional>",
|
||||
"attributes_metadata": [ /* see below */ ],
|
||||
"outputs": { /* see below — optional */ },
|
||||
"requirements": [ /* see below */ ]
|
||||
}
|
||||
```
|
||||
|
||||
A `provider` field at the top level is **optional**. The framework's effective provider list is derived by `ComplianceFramework.get_providers()` (`compliance_models.py:739`) from the union of all keys appearing in `requirement.checks` across all requirements; the explicit `provider` field is used **only as a fallback** when no requirement carries any `checks` key. This is what enables a single file (e.g. `dora.json`) to cover AWS today and add Azure / GCP / etc. tomorrow without restructuring.
|
||||
|
||||
Provider keys inside `requirement.checks` must match the directory names under `prowler/providers/`. The valid keys at present are: `aws`, `azure`, `gcp`, `m365`, `kubernetes`, `iac`, `github`, `googleworkspace`, `alibabacloud`, `cloudflare`, `mongodbatlas`, `nhn`, `openstack`, `oraclecloud`, `llm`. Comparison in `supports_provider()` is case-insensitive, but lowercase is the convention used everywhere in the repository.
|
||||
|
||||
### `attributes_metadata`
|
||||
|
||||
Declares the shape of the per-requirement `attributes` dict. When this field is present, the root validator `validate_attributes_against_metadata` (`compliance_models.py:669`) enforces the schema at load time and rejects:
|
||||
|
||||
- Missing keys marked `required: true`.
|
||||
- Keys present in `attributes` but not declared in `attributes_metadata` (typo / drift guard).
|
||||
- Values that violate a declared `enum`.
|
||||
- Values whose Python type does not match a declared `int`, `float` or `bool`.
|
||||
|
||||
The runtime type check **only** covers `int`, `float` and `bool`. For `str`, `list_str` and `list_dict` the type is documentation-only — non-conforming values won't fail validation. If `attributes_metadata` is omitted, **no per-requirement validation runs at all**.
|
||||
|
||||
```json
|
||||
"attributes_metadata": [
|
||||
{
|
||||
"key": "Pillar",
|
||||
"label": "Pillar",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"enum": [
|
||||
"ICT Risk Management",
|
||||
"ICT-Related Incident Reporting",
|
||||
"Digital Operational Resilience Testing",
|
||||
"ICT Third-Party Risk Management",
|
||||
"Information Sharing"
|
||||
],
|
||||
"output_formats": { "csv": true, "ocsf": true }
|
||||
},
|
||||
{
|
||||
"key": "Article",
|
||||
"label": "Article",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"output_formats": { "csv": true, "ocsf": true }
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Per attribute:
|
||||
|
||||
- `key` (required): attribute name as it will appear in `requirement.attributes`.
|
||||
- `label`: human-readable label used in CSV headers and PDF.
|
||||
- `type`: one of `str`, `int`, `float`, `bool`, `list_str`, `list_dict`. Defaults to `str`.
|
||||
- `enum`: optional list of allowed values; non-conforming values are rejected at load time.
|
||||
- `required`: if `true`, every requirement must include this key with a non-null value.
|
||||
- `enum_display` / `enum_order`: optional per-enum-value visual metadata (label, abbreviation, color, icon) and explicit ordering for PDF rendering.
|
||||
- `output_formats`: `{ "csv": <bool>, "ocsf": <bool> }` — toggles inclusion in each output format. Both default to `true`.
|
||||
|
||||
### `outputs`
|
||||
|
||||
Optional. Controls how the framework is rendered in the console table and in the generated PDF report. Skipping it falls back to sensible defaults.
|
||||
|
||||
```json
|
||||
"outputs": {
|
||||
"table_config": {
|
||||
"group_by": "Pillar"
|
||||
},
|
||||
"pdf_config": {
|
||||
"language": "en",
|
||||
"primary_color": "#003399",
|
||||
"secondary_color": "#0055A5",
|
||||
"bg_color": "#F0F4FA",
|
||||
"group_by_field": "Pillar",
|
||||
"sections": [ "ICT Risk Management", "ICT-Related Incident Reporting", "..." ],
|
||||
"section_short_names": { "ICT Risk Management": "ICT Risk Mgmt" },
|
||||
"charts": [
|
||||
{
|
||||
"id": "pillar_compliance",
|
||||
"type": "horizontal_bar",
|
||||
"group_by": "Pillar",
|
||||
"title": "Compliance Score by Pillar",
|
||||
"y_label": "Pillar",
|
||||
"x_label": "Compliance %",
|
||||
"value_source": "compliance_percent",
|
||||
"color_mode": "by_value"
|
||||
}
|
||||
],
|
||||
"filter": { "only_failed": true, "include_manual": false }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`table_config.group_by` must reference an attribute key declared in `attributes_metadata`. The same applies to `pdf_config.group_by_field` and to every `charts[].group_by`.
|
||||
|
||||
For frameworks with weighted scoring (e.g. ThreatScore) declare `pdf_config.scoring` with `risk_field` / `weight_field` / `risk_boost_factor`. For column splitting (e.g. CIS Level 1 vs Level 2) use `table_config.split_by`.
|
||||
|
||||
### `requirements`
|
||||
|
||||
```json
|
||||
"requirements": [
|
||||
{
|
||||
"id": "DORA-Art5",
|
||||
"name": "Governance and organisation",
|
||||
"description": "Financial entities shall have a sound, comprehensive and well-documented ICT internal governance and control framework. ...",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 5",
|
||||
"ArticleTitle": "Governance and organisation"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"iam_avoid_root_usage",
|
||||
"iam_no_root_access_key",
|
||||
"iam_root_mfa_enabled"
|
||||
],
|
||||
"azure": [],
|
||||
"gcp": []
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Per requirement:
|
||||
|
||||
- `id` (required): unique identifier within the framework.
|
||||
- `description` (required): the requirement text as authored by the framework.
|
||||
- `name`: short title shown alongside the id.
|
||||
- `attributes`: flat dict; keys must conform to `attributes_metadata`.
|
||||
- `checks`: dict keyed by provider name (the same lowercase keys listed in the previous section). Each value is a list of Prowler check names that evidence this requirement for that provider. The list **may be empty** and the dict itself defaults to `{}` if omitted; either way the requirement is still loaded and listed by `--list-compliance-requirements`, it just has zero checks to execute. Note: there is **no automatic check-existence validation** at load time — referencing a non-existent check name will silently produce a requirement with no findings. Validate this yourself (see "Validating Your Framework" below).
|
||||
|
||||
For MITRE-style frameworks, additional optional fields are available on the requirement: `tactics`, `sub_techniques`, `platforms`, `technique_url` (these are populated automatically when adapting a legacy MITRE JSON to the universal model).
|
||||
|
||||
### Multi-provider frameworks
|
||||
|
||||
A single universal file can cover any number of providers. The framework appears under each provider's `--list-compliance` output as long as **at least one** requirement has that provider key in its `checks` dict.
|
||||
|
||||
When extending an existing universal framework with a new provider, the only change required is editing `requirement.checks`:
|
||||
|
||||
```diff
|
||||
"checks": {
|
||||
"aws": ["iam_avoid_root_usage", "iam_no_root_access_key"],
|
||||
+ "azure": ["entra_policy_ensure_mfa_for_admin_roles"]
|
||||
}
|
||||
```
|
||||
|
||||
No code changes, no new file, no registration step.
|
||||
|
||||
## Legacy Provider-Specific Compliance Framework
|
||||
|
||||
The legacy schema is still fully supported and remains the format used by most frameworks shipped today (CIS, NIST, ISO 27001, FedRAMP, PCI DSS, GDPR, HIPAA, ENS, etc.). It binds a framework to a single provider and validates each requirement against a framework-specific Pydantic attribute class.
|
||||
|
||||
The legacy schema spans **four layers** — a complete contribution must touch every layer that applies:
|
||||
|
||||
- **Layer 1 — Schema validation:** the Pydantic models in `prowler/lib/check/compliance_models.py` define the canonical schema for each attribute shape.
|
||||
- **Layer 2 — JSON catalog:** the framework JSON file in `prowler/compliance/<provider>/` lists every requirement and maps it to checks.
|
||||
- **Layer 3 — Output formatter:** the Python module in `prowler/lib/outputs/compliance/<framework>/` builds the CSV row model, the per-provider transformer, and the CLI summary table.
|
||||
- **Layer 4 — Output dispatchers:** the dispatchers in `prowler/lib/outputs/compliance/compliance.py` and `prowler/lib/outputs/compliance/compliance_output.py` route findings to the right formatter based on the framework identifier.
|
||||
|
||||
The universal schema collapses Layers 3 and 4 into declarative configuration inside the JSON — that is the main reason it is preferred for new contributions.
|
||||
|
||||
### Directory structure and file naming
|
||||
|
||||
Compliance frameworks live at:
|
||||
|
||||
@@ -46,8 +234,8 @@ prowler/compliance/<provider>/<framework>_<version>_<provider>.json
|
||||
The filename conventions are:
|
||||
|
||||
- All lowercase, words separated with underscores.
|
||||
- `<provider>` is a supported provider identifier: `aws`, `azure`, `gcp`, `kubernetes`, `m365`, `github`, `googleworkspace`, `alibabacloud`, `oraclecloud`, `cloudflare`, `mongodbatlas`, `nhn`, `openstack`, `iac`, `llm`.
|
||||
- `<version>` is optional. Omit it when the framework has no versioning, as in `ccc_aws.json`.
|
||||
- `<provider>` is a supported provider identifier (same lowercase list as the universal section above).
|
||||
- `<version>` is optional but recommended. Omit only when the framework has no versioning (e.g. `ccc_aws.json`).
|
||||
- The file basename (without `.json`) is the framework key that Prowler CLI accepts via `--compliance`.
|
||||
|
||||
Examples:
|
||||
@@ -62,48 +250,50 @@ The output formatter directory mirrors the framework name:
|
||||
|
||||
```
|
||||
prowler/lib/outputs/compliance/<framework>/
|
||||
├── <framework>.py # CLI summary-table dispatcher
|
||||
├── <framework>.py # CLI summary-table dispatcher
|
||||
├── <framework>_<provider>.py # Per-provider transformer class
|
||||
├── models.py # Pydantic CSV row model
|
||||
└── __init__.py
|
||||
```
|
||||
|
||||
## JSON Schema Reference
|
||||
### JSON schema reference
|
||||
|
||||
Every compliance file is a JSON document with the following top-level keys.
|
||||
Every legacy compliance file is a JSON document with the following top-level keys. `Framework`, `Name` and `Provider` are validated non-empty by the root validator `framework_and_provider_must_not_be_empty` (`compliance_models.py:329`).
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `Framework` | string | Yes | Canonical framework identifier, for example `CIS`, `NIST-800-53-Revision-5`, `ENS`, `CCC`. |
|
||||
| `Name` | string | Yes | Human-readable framework name displayed by Prowler App. |
|
||||
| `Version` | string | Yes | Framework version, for example `2.0`. Use an empty string only for frameworks without versioning. See [Version Handling](#version-handling). |
|
||||
| `Version` | string | Yes (recommended) | Framework version, e.g. `2.0`. See [Version Handling](#version-handling). |
|
||||
| `Provider` | string | Yes | Upper-cased provider identifier: `AWS`, `AZURE`, `GCP`, `KUBERNETES`, `M365`, `GITHUB`, `GOOGLEWORKSPACE`, and so on. |
|
||||
| `Description` | string | Yes | Short description of the framework's scope and purpose. |
|
||||
| `Requirements` | array | Yes | List of [requirement objects](#requirement-object). |
|
||||
|
||||
### Requirement Object
|
||||
#### Requirement Object
|
||||
|
||||
Each entry in `Requirements` describes one control or requirement.
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `Id` | string | Yes | Unique identifier within the framework, for example `1.10` or `CCC.Core.CN01.AR01`. |
|
||||
| `Name` | string | No | Optional human-readable name used by frameworks that distinguish control name from description, such as NIST. |
|
||||
| `Name` | string | No | Optional human-readable name (frameworks like NIST distinguish control name from description). |
|
||||
| `Description` | string | Yes | Verbatim description from the source framework. |
|
||||
| `Attributes` | array | Yes | List of [attribute objects](#attribute-objects). The shape depends on the framework. |
|
||||
| `Checks` | array of strings | Yes | Prowler check identifiers that automate the requirement. Leave the list empty when the control cannot be automated. |
|
||||
|
||||
### Attribute Objects
|
||||
#### Attribute Objects
|
||||
|
||||
Attributes carry the metadata that Prowler App and the CSV output display for each requirement. The object shape is framework-specific and is validated by a dedicated Pydantic model in `prowler/lib/check/compliance_models.py`. The most common shapes are summarized below.
|
||||
`Attributes` is parsed against the union declared in `Compliance_Requirement.Attributes` (`compliance_models.py:293`). Pydantic v1 tries each member of the union in declaration order and falls back to `Generic_Compliance_Requirement_Attribute` (the last entry) when nothing else matches — so a brand-new shape that doesn't match any existing class will silently be accepted as Generic, losing its specific fields.
|
||||
|
||||
#### CIS_Requirement_Attribute
|
||||
As of today, the registered attribute classes are: `CIS_Requirement_Attribute`, `ENS_Requirement_Attribute`, `ASDEssentialEight_Requirement_Attribute`, `ISO27001_2013_Requirement_Attribute`, `AWS_Well_Architected_Requirement_Attribute`, `KISA_ISMSP_Requirement_Attribute`, `Prowler_ThreatScore_Requirement_Attribute`, `CCC_Requirement_Attribute`, `C5Germany_Requirement_Attribute`, `CSA_CCM_Requirement_Attribute`, and `Generic_Compliance_Requirement_Attribute` (fallback). MITRE-style frameworks use the separate `Mitre_Requirement` model with `Tactics` / `SubTechniques` / `Platforms` / `TechniqueURL` at the requirement top level. The most common shapes are summarized below.
|
||||
|
||||
##### CIS_Requirement_Attribute
|
||||
|
||||
Used by every CIS benchmark.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|---|---|
|
||||
| `Section` | string | Yes | Top-level section, for example `1 Identity and Access Management`. |
|
||||
| `Section` | string | Yes | Top-level section, e.g. `1 Identity and Access Management`. |
|
||||
| `SubSection` | string | No | Optional second-level grouping. |
|
||||
| `Profile` | enum | Yes | One of `Level 1`, `Level 2`, `E3 Level 1`, `E3 Level 2`, `E5 Level 1`, `E5 Level 2`. |
|
||||
| `AssessmentStatus` | enum | Yes | `Manual` or `Automated`. |
|
||||
@@ -116,7 +306,7 @@ Used by every CIS benchmark.
|
||||
| `DefaultValue` | string | No | Default configuration value, when relevant. |
|
||||
| `References` | string | Yes | Colon-separated list of reference URLs. |
|
||||
|
||||
#### ENS_Requirement_Attribute
|
||||
##### ENS_Requirement_Attribute
|
||||
|
||||
Used by the Spanish ENS (Esquema Nacional de Seguridad) frameworks.
|
||||
|
||||
@@ -132,13 +322,13 @@ Used by the Spanish ENS (Esquema Nacional de Seguridad) frameworks.
|
||||
| `ModoEjecucion` | string | Yes | Execution mode (`manual`, `automático`, `híbrido`). |
|
||||
| `Dependencias` | array of strings | Yes | Ids of prerequisite controls. Empty list when none. |
|
||||
|
||||
#### CCC_Requirement_Attribute
|
||||
##### CCC_Requirement_Attribute
|
||||
|
||||
Used by the Common Cloud Controls Catalog.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|---|---|
|
||||
| `FamilyName` | string | Yes | Control family, for example `Data`. |
|
||||
| `FamilyName` | string | Yes | Control family, e.g. `Data`. |
|
||||
| `FamilyDescription` | string | Yes | Description of the family. |
|
||||
| `Section` | string | Yes | Section title. |
|
||||
| `SubSection` | string | Yes | Subsection title, or empty string. |
|
||||
@@ -148,9 +338,9 @@ Used by the Common Cloud Controls Catalog.
|
||||
| `SectionThreatMappings` | array of objects | Yes | Each entry has `ReferenceId` and `Identifiers`. |
|
||||
| `SectionGuidelineMappings` | array of objects | Yes | Each entry has `ReferenceId` and `Identifiers`. |
|
||||
|
||||
#### Generic_Compliance_Requirement_Attribute
|
||||
##### Generic_Compliance_Requirement_Attribute
|
||||
|
||||
The fallback attribute model used when no framework-specific schema applies (for example NIST 800-53, PCI DSS, GDPR, HIPAA).
|
||||
The fallback attribute model used when no framework-specific schema applies (e.g. NIST 800-53, PCI DSS, GDPR, HIPAA). It is **always the last** element of the `Compliance_Requirement.Attributes` Union; that ordering is load-bearing.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|---|---|
|
||||
@@ -158,17 +348,17 @@ The fallback attribute model used when no framework-specific schema applies (for
|
||||
| `Section` | string | No | Section name. |
|
||||
| `SubSection` | string | No | Subsection name. |
|
||||
| `SubGroup` | string | No | Subgroup name. |
|
||||
| `Service` | string | No | Affected service, for example `aws`, `iam`. |
|
||||
| `Service` | string | No | Affected service, e.g. `iam`. |
|
||||
| `Type` | string | No | Control type. |
|
||||
| `Comment` | string | No | Free-form comment. |
|
||||
|
||||
Additional per-framework attribute models exist for `AWS_Well_Architected_Requirement_Attribute`, `ISO27001_2013_Requirement_Attribute`, `Mitre_Requirement_Attribute_<Provider>`, `KISA_ISMSP_Requirement_Attribute`, `Prowler_ThreatScore_Requirement_Attribute`, `C5Germany_Requirement_Attribute`, and `CSA_CCM_Requirement_Attribute`. Consult `prowler/lib/check/compliance_models.py` for their full field sets.
|
||||
For the remaining attribute classes (`AWS_Well_Architected_Requirement_Attribute`, `ISO27001_2013_Requirement_Attribute`, `Mitre_Requirement_Attribute_<Provider>`, `KISA_ISMSP_Requirement_Attribute`, `Prowler_ThreatScore_Requirement_Attribute`, `C5Germany_Requirement_Attribute`, `CSA_CCM_Requirement_Attribute`) consult `prowler/lib/check/compliance_models.py` for the full field sets.
|
||||
|
||||
<Note>
|
||||
The `Attributes` field is a Pydantic `Union`. The generic attribute model must remain the last element of that Union, otherwise Pydantic v1 silently coerces every framework into the generic shape and your specialized fields are dropped.
|
||||
The `Attributes` field is a Pydantic `Union`. The generic attribute model **must** remain the last element of that Union — otherwise Pydantic v1 silently coerces every framework into the generic shape and your specialized fields are dropped. Adding a brand-new attribute shape requires inserting the Pydantic class **before** `Generic_Compliance_Requirement_Attribute`.
|
||||
</Note>
|
||||
|
||||
## Minimal Working Example
|
||||
#### Minimal working example
|
||||
|
||||
The following snippet is a complete, valid framework file named `my_framework_1.0_aws.json`, saved at `prowler/compliance/aws/my_framework_1.0_aws.json`. It uses the generic attribute shape for simplicity.
|
||||
|
||||
@@ -214,26 +404,26 @@ The following snippet is a complete, valid framework file named `my_framework_1.
|
||||
}
|
||||
```
|
||||
|
||||
## Mapping Checks to Requirements
|
||||
### Mapping checks to requirements
|
||||
|
||||
Each requirement links to the Prowler checks that, together, produce a PASS or FAIL verdict for that control.
|
||||
|
||||
- **Include every requirement from the source catalog.** The framework file must mirror the full control list, one-to-one. Compliance percentages, dashboards, and exported metadata are computed against the total requirement count, so omitting an unmappable control inflates coverage and misrepresents the framework.
|
||||
- List every check by its canonical identifier, the value of `CheckID` inside the check's `.metadata.json` file.
|
||||
- **Include every requirement from the source catalog.** The framework file must mirror the full control list, one-to-one. Compliance percentages, dashboards, and exported metadata are computed against the total requirement count.
|
||||
- List every check by its canonical identifier — the value of `CheckID` inside the check's `.metadata.json` file.
|
||||
- One requirement can reference multiple checks. The requirement is evaluated as FAIL when any referenced check produces a FAIL finding for a resource in scope.
|
||||
- Leave `Checks` as an empty array when the requirement cannot be automated. The requirement still appears in the report, contributes to the total, and resolves to `MANUAL`. An empty mapping is valid; a missing requirement is not.
|
||||
- Leave `Checks` (legacy) or `checks.<provider>` (universal) as an empty array when the requirement cannot be automated. The requirement still appears in the report and contributes to the total.
|
||||
- Reuse checks across requirements when the same control applies in multiple places. Do not duplicate check logic to match framework structure.
|
||||
- Avoid referencing checks from a different provider. A compliance file is bound to one provider, and cross-provider checks will never match findings in the scan.
|
||||
- Avoid referencing checks from a different provider. A legacy compliance file is bound to one provider, and cross-provider checks will never match findings in the scan.
|
||||
|
||||
To discover available checks, run:
|
||||
To discover available checks:
|
||||
|
||||
```bash
|
||||
uv run python prowler-cli.py <provider> --list-checks
|
||||
```
|
||||
|
||||
## Supporting Multiple Providers
|
||||
### Supporting multiple providers (legacy)
|
||||
|
||||
Each compliance file targets a single provider. To cover several providers with the same framework (for example CIS across AWS, Azure, and GCP), ship one JSON file per provider:
|
||||
The legacy schema binds each file to a single provider. To cover several providers with the same framework, ship one JSON file per provider:
|
||||
|
||||
```
|
||||
prowler/compliance/aws/cis_2.0_aws.json
|
||||
@@ -241,15 +431,15 @@ prowler/compliance/azure/cis_2.0_azure.json
|
||||
prowler/compliance/gcp/cis_2.0_gcp.json
|
||||
```
|
||||
|
||||
Keep the `Framework` and `Version` values identical across the files so the dispatcher matches them, and change only the `Provider`, `Checks`, and provider-specific metadata.
|
||||
Keep the `Framework` and `Version` values identical across the files so the dispatcher matches them; change only the `Provider`, `Checks`, and provider-specific metadata. The CIS output formatter already supports every provider listed above.
|
||||
|
||||
The CIS output formatter already supports every provider listed above. For a brand-new framework that spans several providers, add one transformer per provider in `prowler/lib/outputs/compliance/<framework>/` and extend the summary-table dispatcher accordingly. See [Output Formatter](#output-formatter).
|
||||
For a brand-new framework that spans several providers, **prefer the universal schema** — it covers every provider from a single file. If you must use the legacy schema, add one transformer per provider in `prowler/lib/outputs/compliance/<framework>/` and extend the summary-table dispatcher accordingly. See [Output Formatter](#output-formatter).
|
||||
|
||||
## Output Formatter
|
||||
### Output formatter
|
||||
|
||||
Prowler renders every compliance framework in two forms: a detailed CSV report written to disk, and a summary table printed in the CLI. Both are produced by the output formatter package for the framework.
|
||||
Legacy frameworks render in two forms: a detailed CSV report written to disk, and a summary table printed in the CLI. Both are produced by the output formatter package for the framework. Universal frameworks do **not** need a Python output formatter — the `outputs` config inside the JSON drives rendering — so this section applies only to the legacy schema.
|
||||
|
||||
For a new framework named `my_framework`, create:
|
||||
For a new legacy framework named `my_framework`, create:
|
||||
|
||||
```
|
||||
prowler/lib/outputs/compliance/my_framework/
|
||||
@@ -259,19 +449,19 @@ prowler/lib/outputs/compliance/my_framework/
|
||||
└── models.py # CSV row Pydantic model
|
||||
```
|
||||
|
||||
### Step 1 – Define the CSV Row Model
|
||||
#### Step 1 — Define the CSV row model
|
||||
|
||||
In `models.py`, declare a Pydantic v1 model with one field per CSV column. Use existing models such as `AWSCISModel` in `prowler/lib/outputs/compliance/cis/models.py` as the reference. Fields typically include `Provider`, `Description`, `AccountId`, `Region`, `AssessmentDate`, `Requirements_Id`, `Requirements_Description`, one `Requirements_Attributes_*` field per attribute key, plus the finding fields `Status`, `StatusExtended`, `ResourceId`, `ResourceName`, `CheckId`, `Muted`, `Framework`, `Name`.
|
||||
|
||||
### Step 2 – Implement the Transformer Class
|
||||
#### Step 2 — Implement the transformer
|
||||
|
||||
In `my_framework_aws.py`, subclass `ComplianceOutput` from `prowler.lib.outputs.compliance.compliance_output` and implement `transform(findings, compliance, compliance_name)`. Iterate over `findings`, match each finding to the requirements it satisfies through `finding.compliance.get(compliance_name, [])`, and append one row per attribute to `self._data`.
|
||||
|
||||
### Step 3 – Add the Summary-Table Dispatcher
|
||||
#### Step 3 — Add the summary-table dispatcher
|
||||
|
||||
In `my_framework.py`, implement `get_my_framework_table(findings, bulk_checks_metadata, compliance_framework, output_filename, output_directory, compliance_overview)` following the pattern in `prowler/lib/outputs/compliance/cis/cis.py`.
|
||||
|
||||
### Step 4 – Register the Framework in the Dispatchers
|
||||
#### Step 4 — Register the framework in the dispatchers
|
||||
|
||||
- Add the dispatcher call in `prowler/lib/outputs/compliance/compliance.py`, inside `display_compliance_table`, with a branch such as `elif "my_framework" in compliance_framework:`.
|
||||
- Register the CSV model and transformer in `prowler/lib/outputs/compliance/compliance_output.py` so the CSV file is emitted during the scan.
|
||||
@@ -280,49 +470,94 @@ In `my_framework.py`, implement `get_my_framework_table(findings, bulk_checks_me
|
||||
For NIST-style catalogs that use `Generic_Compliance_Requirement_Attribute`, no custom formatter is needed. The generic formatter in `prowler/lib/outputs/compliance/generic/` handles them automatically, provided the JSON validates against the generic attribute schema.
|
||||
</Note>
|
||||
|
||||
## Version Handling
|
||||
### Legacy-to-universal adapter
|
||||
|
||||
At load time, every legacy file is transparently adapted to a `ComplianceFramework` via `adapt_legacy_to_universal()` (`compliance_models.py:819`), which: (a) flattens the first element of `Attributes` into a flat `attributes` dict, (b) wraps `Checks` as `{provider_lower: [...]}`, (c) infers `attributes_metadata` from the matched Pydantic class via `_infer_attribute_metadata()`. The rest of Prowler (CSV/OCSF/PDF output, CLI table) then treats both formats identically.
|
||||
|
||||
Loader-error behaviour differs between the two entry points:
|
||||
|
||||
- `load_compliance_framework()` (legacy) is **fail-fast**: it calls `sys.exit(1)` on any `ValidationError` (`compliance_models.py:464`).
|
||||
- `load_compliance_framework_universal()` is more lenient — it logs the error and returns `None`, so `get_bulk_compliance_frameworks_universal()` simply skips the broken file and keeps loading the rest.
|
||||
|
||||
## Version handling
|
||||
|
||||
Prowler matches frameworks by concatenating `Framework` and `Version`. A missing or empty `Version` collapses several frameworks to the same key and breaks CLI filtering with `--compliance`.
|
||||
|
||||
- Always set `Version` to a non-empty string, even for frameworks that rename editions rather than version them. Use the edition identifier (for example `RD2022`, `v2025.10`, `4.0`).
|
||||
- Always set `Version` (or `version` for universal frameworks) to a non-empty string, even for frameworks that rename editions rather than version them. Use the edition identifier (for example `RD2022`, `v2025.10`, `4.0`, `2022/2554`).
|
||||
- When the source catalog has no version, use the first year of adoption or the release date.
|
||||
- Make sure the version substring embedded in the filename matches `Version`, because the CLI dispatcher reads `compliance_framework.split("_")[1]` to select the correct version.
|
||||
- For **legacy** files, make sure the version substring embedded in the filename matches `Version`, because the CLI dispatcher reads `compliance_framework.split("_")[1]` to select the correct version.
|
||||
|
||||
## Validating the Framework Locally
|
||||
## Validating Your Framework
|
||||
|
||||
Follow the steps below before opening a pull request.
|
||||
Before opening a PR, validate the JSON loads cleanly against the model and that every referenced check actually exists.
|
||||
|
||||
### 1. Run the Compliance Model Validator
|
||||
### 1. Schema validation
|
||||
|
||||
For **universal** frameworks, load the file and inspect what was parsed. The framework key inside `bulk` is the **basename of the JSON file** (without `.json`); for `prowler/compliance/dora.json` that key is `dora`, for `prowler/compliance/aws/cis_5.0_aws.json` it is `cis_5.0_aws`.
|
||||
|
||||
```python
|
||||
from prowler.lib.check.compliance_models import (
|
||||
load_compliance_framework_universal,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
|
||||
fw = load_compliance_framework_universal("prowler/compliance/<your_framework>.json")
|
||||
assert fw is not None, "load returned None — check the logs for the validation error"
|
||||
print(fw.framework, len(fw.requirements), fw.get_providers())
|
||||
|
||||
bulk = get_bulk_compliance_frameworks_universal("aws")
|
||||
assert "<your_framework_filename_without_json>" in bulk
|
||||
```
|
||||
|
||||
### 2. Check existence cross-check
|
||||
|
||||
There is **no automatic check-existence validation** at load time. Cross-check that every check name in your framework maps to a real check directory:
|
||||
|
||||
```python
|
||||
import os
|
||||
real = set()
|
||||
for svc in os.listdir("prowler/providers/aws/services"):
|
||||
svc_path = f"prowler/providers/aws/services/{svc}"
|
||||
if not os.path.isdir(svc_path):
|
||||
continue
|
||||
for entry in os.listdir(svc_path):
|
||||
if os.path.isfile(f"{svc_path}/{entry}/{entry}.metadata.json"):
|
||||
real.add(entry)
|
||||
|
||||
referenced = {c for r in fw.requirements for c in r.checks.get("aws", [])}
|
||||
missing = referenced - real
|
||||
assert not missing, f"checks referenced in framework but not found in repo: {sorted(missing)}"
|
||||
```
|
||||
|
||||
### 3. CLI smoke test
|
||||
|
||||
```bash
|
||||
uv run python prowler-cli.py <provider> --list-compliance
|
||||
```
|
||||
|
||||
The framework must appear in the output. A validation error indicates a schema mismatch between the JSON file and the attribute model.
|
||||
|
||||
### 2. Run a Scan Filtered by the Framework
|
||||
The framework must appear in the output. A validation error indicates a schema mismatch.
|
||||
|
||||
```bash
|
||||
uv run python prowler-cli.py <provider> \
|
||||
--compliance <framework>_<version>_<provider> \
|
||||
--compliance <framework_key> \
|
||||
--log-level ERROR
|
||||
```
|
||||
|
||||
Verify that:
|
||||
|
||||
- Prowler produces a CSV file under `output/compliance/` with the expected name.
|
||||
- The CLI summary table lists every section in the framework.
|
||||
- The CLI summary table lists every section / pillar of the framework.
|
||||
- Findings roll up under the expected requirements.
|
||||
|
||||
### 3. Inspect the CSV Output
|
||||
### 4. Inspect the CSV output
|
||||
|
||||
Open the generated CSV and confirm:
|
||||
|
||||
- All columns defined in `models.py` appear.
|
||||
- Every requirement has at least one row per scanned resource.
|
||||
- Values such as `Requirements_Attributes_Section` reflect the JSON content.
|
||||
- All columns defined in `models.py` (legacy) or in `attributes_metadata` (universal) appear.
|
||||
- Every requirement has at least one row per scanned resource (when there are findings).
|
||||
- Attribute values such as `Requirements_Attributes_Section` reflect the JSON content.
|
||||
|
||||
### 4. Verify the Framework in Prowler App
|
||||
### 5. Verify the framework in Prowler App
|
||||
|
||||
Launch Prowler App locally (`docker compose up` from the repository root) and run a scan with the new compliance framework. Confirm the compliance page renders the requirements, sections, and status widgets correctly.
|
||||
|
||||
@@ -331,7 +566,7 @@ Launch Prowler App locally (`docker compose up` from the repository root) and ru
|
||||
Compliance contributions require two layers of tests.
|
||||
|
||||
- **Schema tests** exercise the Pydantic models. Extend `tests/lib/check/universal_compliance_models_test.py` with a case that loads the new JSON file and asserts the attribute type matches the expected model.
|
||||
- **Output tests** exercise the transformer. Mirror the structure under `tests/lib/outputs/compliance/<framework>/` with fixtures that feed synthetic findings through the transformer and assert the resulting CSV rows.
|
||||
- **Output tests** (legacy frameworks only) exercise the transformer. Mirror the structure under `tests/lib/outputs/compliance/<framework>/` with fixtures that feed synthetic findings through the transformer and assert the resulting CSV rows.
|
||||
|
||||
Run the suite with:
|
||||
|
||||
@@ -342,7 +577,20 @@ uv run pytest -n auto tests/lib/check/universal_compliance_models_test.py \
|
||||
|
||||
For guidance on writing Prowler SDK tests, refer to [Unit Testing](/developer-guide/unit-testing).
|
||||
|
||||
## Submitting the Pull Request
|
||||
## Running and listing your framework
|
||||
|
||||
Once the file is in place, the CLI auto-discovers it:
|
||||
|
||||
```sh
|
||||
prowler <provider> --list-compliance # framework appears in the list
|
||||
prowler <provider> --compliance <framework_key> --list-checks
|
||||
prowler <provider> --compliance <framework_key> # full scan + compliance report
|
||||
prowler <provider> --compliance <framework_key> --list-compliance-requirements <framework_key>
|
||||
```
|
||||
|
||||
For end-user-facing tutorials (recommended for high-profile frameworks), add a dedicated page under `docs/user-guide/compliance/tutorials/` and register it in the `"Compliance"` group of `docs/docs.json`. See `docs/user-guide/compliance/tutorials/threatscore.mdx` as a reference.
|
||||
|
||||
## Submitting the pull request
|
||||
|
||||
Before opening the pull request:
|
||||
|
||||
@@ -352,28 +600,31 @@ Before opening the pull request:
|
||||
uv run pytest -n auto
|
||||
```
|
||||
2. Add a changelog entry under the `### 🚀 Added` section of `prowler/CHANGELOG.md`, describing the new framework and the providers it covers.
|
||||
3. Follow the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and set the PR title using Conventional Commits, for example `feat(compliance): add My Framework 1.0 for AWS`.
|
||||
3. Follow the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and set the PR title using Conventional Commits, e.g. `feat(compliance): add My Framework 1.0 for AWS`.
|
||||
4. Request review from the compliance codeowners listed in `.github/CODEOWNERS`.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
The following issues are the most common when contributing a compliance framework.
|
||||
|
||||
- **`ValidationError: field required` during scan.** The JSON is missing a required attribute field. Re-check the matching Pydantic model in `prowler/lib/check/compliance_models.py`.
|
||||
- **All attributes collapse to `Generic_Compliance_Requirement_Attribute` values.** The Pydantic `Union` is ordered incorrectly, or the JSON matches only the generic shape. Move the generic model to the last Union position and ensure every required field is present in the JSON.
|
||||
- **`--compliance` filter does not find the framework.** The filename does not match the expected pattern `<framework>_<version>_<provider>.json`, the version is empty, or the file lives outside `prowler/compliance/<provider>/`.
|
||||
- **CLI summary table is empty but the CSV is populated.** The dispatcher branch in `prowler/lib/outputs/compliance/compliance.py` is missing or its substring match does not catch the framework key.
|
||||
- **CSV file is missing after the scan.** The transformer class is not registered in `prowler/lib/outputs/compliance/compliance_output.py`, or `transform()` raises silently. Run the scan with `--log-level DEBUG`.
|
||||
- **Findings do not roll up under a requirement.** A check listed in `Checks` either does not exist for that provider or is spelled incorrectly. Run `--list-checks | grep <check_name>` to confirm.
|
||||
- **`ValidationError: field required` during scan (legacy).** The JSON is missing a required attribute field. Re-check the matching Pydantic model in `prowler/lib/check/compliance_models.py`.
|
||||
- **All attributes collapse to `Generic_Compliance_Requirement_Attribute` values (legacy).** The Pydantic `Union` is ordered incorrectly, or the JSON matches only the generic shape. Keep the generic model in the last Union position and ensure every required field is present in the JSON.
|
||||
- **`attributes_metadata validation failed` (universal).** The root validator in `compliance_models.py:669` rejected the file. The error message lists each offending requirement; common causes are unknown attribute keys (typo or missing entry in `attributes_metadata`), enum violations, or missing required keys.
|
||||
- **`--compliance` filter does not find the framework.** For legacy: the filename does not match `<framework>_<version>_<provider>.json`, the version is empty, or the file lives outside `prowler/compliance/<provider>/`. For universal: the file is not at the top level of `prowler/compliance/` or it loaded as `None` (check logs for the validation error).
|
||||
- **CLI summary table is empty but the CSV is populated (legacy).** The dispatcher branch in `prowler/lib/outputs/compliance/compliance.py` is missing or its substring match does not catch the framework key.
|
||||
- **CSV file is missing after the scan (legacy).** The transformer class is not registered in `prowler/lib/outputs/compliance/compliance_output.py`, or `transform()` raises silently. Run the scan with `--log-level DEBUG`.
|
||||
- **Findings do not roll up under a requirement.** A check listed in `Checks` either does not exist for that provider or is spelled incorrectly. Run `--list-checks | grep <check_name>` to confirm, or run the check-existence cross-check from "Validating Your Framework".
|
||||
|
||||
## Reference Examples
|
||||
## Reference examples
|
||||
|
||||
Use the following files as templates when modeling a new contribution.
|
||||
|
||||
- `prowler/compliance/aws/cis_2.0_aws.json` – CIS attribute shape.
|
||||
- `prowler/compliance/aws/nist_800_53_revision_5_aws.json` – Generic attribute shape.
|
||||
- `prowler/compliance/aws/ccc_aws.json` – CCC attribute shape.
|
||||
- `prowler/compliance/azure/ens_rd2022_azure.json` – ENS attribute shape.
|
||||
- `prowler/lib/check/compliance_models.py` – Canonical Pydantic schemas.
|
||||
- `prowler/lib/outputs/compliance/cis/` – Reference implementation of a multi-provider output formatter.
|
||||
- `prowler/lib/outputs/compliance/generic/` – Reference implementation of a generic output formatter.
|
||||
- `prowler/compliance/dora.json` — universal schema, single-provider populated (AWS), ready to extend with more providers.
|
||||
- `prowler/compliance/csa_ccm_4.0.json` — universal schema, multi-provider populated (AWS, Azure, GCP, AlibabaCloud, OracleCloud).
|
||||
- `prowler/compliance/aws/cis_2.0_aws.json` — legacy CIS attribute shape.
|
||||
- `prowler/compliance/aws/nist_800_53_revision_5_aws.json` — legacy generic attribute shape.
|
||||
- `prowler/compliance/aws/ccc_aws.json` — legacy CCC attribute shape.
|
||||
- `prowler/compliance/azure/ens_rd2022_azure.json` — legacy ENS attribute shape.
|
||||
- `prowler/lib/check/compliance_models.py` — canonical Pydantic schemas for both formats.
|
||||
- `prowler/lib/outputs/compliance/cis/` — reference implementation of a multi-provider legacy output formatter.
|
||||
- `prowler/lib/outputs/compliance/generic/` — reference implementation of a legacy generic output formatter.
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
---
|
||||
title: 'Server-Sent Events (SSE)'
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
<VersionBadge version="1.32.0" />
|
||||
|
||||
This guide explains how to add a **Server-Sent Events (SSE)** endpoint to the Prowler API. SSE lets the backend push a one-way stream of events to a client over a single long-lived HTTP connection — ideal for live progress, token-by-token LLM output, or any "the server has news for you" use case where the client should not poll.
|
||||
|
||||
<Info>
|
||||
The platform ships the SSE **infrastructure** (`api.sse`) and wiring. No feature endpoint streams over SSE out of the box — this guide shows how to build one on top of the shared base.
|
||||
</Info>
|
||||
|
||||
## When to use SSE
|
||||
|
||||
| Need | Use |
|
||||
|------|-----|
|
||||
| Server pushes incremental updates, client only reads | **SSE** |
|
||||
| Bidirectional, low-latency messaging (chat both ways, games) | WebSocket |
|
||||
| Client asks, server answers once | Plain REST |
|
||||
|
||||
SSE is the right tool when the **client only consumes**: scan progress, long-running job checkpoints, streamed LLM tokens, cross-client resource-sync notifications. It rides on plain HTTP, reconnects automatically in the browser via the native [`EventSource`](https://developer.mozilla.org/en-US/docs/Web/API/EventSource) API, and needs no extra protocol.
|
||||
|
||||
## How it works
|
||||
|
||||
SSE is wired through [`django-eventstream`](https://github.com/fanout/django_eventstream) and a small platform layer in `api/src/backend/api/sse/`:
|
||||
|
||||
| Piece | File | Responsibility |
|
||||
|-------|------|----------------|
|
||||
| `BaseSSEViewSet` | `api/sse/base_views.py` | Base DRF viewset a feature subclasses. The feature implements `get_channels`; the base handles auth, the tenant transaction, and delegates streaming to `django-eventstream`. |
|
||||
| `SSEChannelManager` | `api/sse/channelmanager.py` | Registered in `settings.EVENTSTREAM_CHANNELMANAGER_CLASS`. Reads the channel set off the request and enforces the platform-wide tenant gate. |
|
||||
| `SSEAuthentication` | `api/authentication.py` | Same JWT/API-key stack as the rest of the API, plus an `?access_token=<jwt>` fallback for browser `EventSource` clients. Lives with the other authentication classes, not in the `sse` package. |
|
||||
| `make_channel_name` / `tenant_id_from_channel` | `api/sse/utils.py` | Single source of truth for the channel-name format, so publishers and the channel manager agree byte-for-byte. |
|
||||
| Settings | `config/settings/eventstream.py` | Valkey Pub/Sub backend (dedicated DB), channel manager, allowed headers. |
|
||||
|
||||
### Transport: the server runs on ASGI
|
||||
|
||||
SSE connections are long-lived. Holding one open per synchronous worker would exhaust the worker pool, so the API runs under Gunicorn's native **`asgi` worker** (`config.asgi:application`). Streams are parked on the event loop while ordinary CRUD endpoints keep their synchronous execution (Django runs sync views in a thread-sensitive executor under ASGI). This is configured in `config/guniconf.py` and used by both the dev and production entrypoints — no separate server process is needed.
|
||||
|
||||
### The data flow
|
||||
|
||||
```
|
||||
publisher (Celery task / view) subscriber (browser, CLI)
|
||||
│ │
|
||||
│ send_event(channel, "scan.progress", …) │ GET …/event-stream
|
||||
▼ ▼
|
||||
Valkey Pub/Sub ◄────────────────────► BaseSSEViewSet.list
|
||||
(EVENTSTREAM_VALKEY_DB) → get_channels() (RLS-scoped)
|
||||
→ SSEChannelManager (tenant gate)
|
||||
→ StreamingHttpResponse (text/event-stream)
|
||||
```
|
||||
|
||||
A publisher anywhere in the system (most often a Celery task) calls `send_event(channel, event_type, payload)`. `django-eventstream` fans it out over Valkey Pub/Sub to every connection subscribed to that channel.
|
||||
|
||||
## Adding an SSE endpoint to your feature
|
||||
|
||||
The example below streams progress for a long-running **scan**. Adapt the resource, prefix, and event names to your feature.
|
||||
|
||||
<Steps>
|
||||
|
||||
<Step title="Pick a channel prefix">
|
||||
|
||||
Channels follow the format `<prefix>:<tenant_id>:<resource_id>`, built only through `make_channel_name`. The prefix is owned by your feature and may contain hyphens but **never colons** (the parser splits on `:`).
|
||||
|
||||
```python
|
||||
CHANNEL_PREFIX = "scan-progress"
|
||||
```
|
||||
|
||||
The tenant id is baked into every channel name. That is what lets the platform enforce cross-tenant isolation without knowing anything about your feature.
|
||||
|
||||
</Step>
|
||||
|
||||
<Step title="Subclass BaseSSEViewSet">
|
||||
|
||||
Create the viewset for the SSE sub-resource. The only required method is `get_channels`; it runs inside the tenant transaction set up by the base class, so any database lookup inside it is automatically RLS-scoped.
|
||||
|
||||
```python
|
||||
# scans/event_streams.py
|
||||
from api.sse import BaseSSEViewSet, make_channel_name
|
||||
from django.shortcuts import get_object_or_404
|
||||
from scans.models import Scan
|
||||
|
||||
CHANNEL_PREFIX = "scan-progress"
|
||||
|
||||
|
||||
class ScanEventStreamViewSet(BaseSSEViewSet):
|
||||
def get_queryset(self):
|
||||
# RLS already scopes to the tenant; narrow further as needed
|
||||
# (e.g. only scans the requesting user may see).
|
||||
return Scan.objects.filter(tenant_id=self.request.tenant_id)
|
||||
|
||||
def get_channels(self) -> set[str]:
|
||||
scan = get_object_or_404(self.get_queryset(), pk=self.kwargs["scan_pk"])
|
||||
return {make_channel_name(CHANNEL_PREFIX, scan.tenant_id, scan.id)}
|
||||
```
|
||||
|
||||
<Warning>
|
||||
`get_channels` **must raise** the relevant DRF exception (`NotFound`, `PermissionDenied`, `NotAuthenticated`) when authorization fails — `get_object_or_404` does this for you. Returning an empty set surfaces as django-eventstream's confusing "No channels specified" error instead of the real cause.
|
||||
</Warning>
|
||||
|
||||
</Step>
|
||||
|
||||
<Step title="Wire the URL as a sub-resource">
|
||||
|
||||
Mount the endpoint as an `event-stream` sub-resource. Keep it **outside the DRF router**, which would force the URL into a list/detail convention. Route the `get` method to the viewset's `list` action.
|
||||
|
||||
```python
|
||||
# scans/urls.py
|
||||
path(
|
||||
"scans/<uuid:scan_pk>/event-stream",
|
||||
ScanEventStreamViewSet.as_view({"get": "list"}),
|
||||
name="scan-event-stream",
|
||||
),
|
||||
```
|
||||
|
||||
</Step>
|
||||
|
||||
<Step title="Define your event vocabulary">
|
||||
|
||||
A feature owns its event types in `<app>/<domain>/events.py`: one `publish_<event>` function per event type, each body a **single** `send_event` call so the wire-level string lives in exactly one place.
|
||||
|
||||
```python
|
||||
# scans/events.py
|
||||
from django_eventstream import send_event
|
||||
|
||||
|
||||
def publish_progress(channel: str, checked: int, total: int) -> None:
|
||||
send_event(channel, "scan.progress", {"checked": checked, "total": total})
|
||||
|
||||
|
||||
def publish_end(channel: str, scan_id: str) -> None:
|
||||
# Terminal event carries the canonical id so reconnecting clients
|
||||
# can refetch the persisted resource over REST.
|
||||
send_event(channel, "scan.end", {"scan_id": scan_id})
|
||||
|
||||
|
||||
def publish_error(channel: str, code: str, detail: str) -> None:
|
||||
send_event(channel, "scan.error", {"code": code, "detail": detail})
|
||||
```
|
||||
|
||||
There is no platform-side enum, registry, or dispatch table — **the naming convention is the contract** (see below).
|
||||
|
||||
</Step>
|
||||
|
||||
<Step title="Publish from the producer">
|
||||
|
||||
Wherever the work happens — usually a Celery task — build the channel the same way and publish:
|
||||
|
||||
```python
|
||||
from api.sse import make_channel_name
|
||||
from scans.events import publish_progress, publish_end
|
||||
|
||||
channel = make_channel_name("scan-progress", scan.tenant_id, scan.id)
|
||||
publish_progress(channel, checked=42, total=100)
|
||||
...
|
||||
publish_end(channel, scan_id=str(scan.id))
|
||||
```
|
||||
|
||||
</Step>
|
||||
|
||||
</Steps>
|
||||
|
||||
## Event naming convention
|
||||
|
||||
Every event uses an event type of the form **`<resource>.<verb>`** (lowercased, dot-separated). The verb comes from this platform-wide vocabulary — if you need a verb that is not listed, document the addition in this guide so the catalog stays discoverable.
|
||||
|
||||
| Verb | When to use |
|
||||
|------|-------------|
|
||||
| `delta` | An incremental piece of a stream the client concatenates (LLM text tokens, audio chunks). Standard term across OpenAI / Anthropic / LiteLLM / Vercel AI SDK. |
|
||||
| `start` | Begin marker for a compound operation (e.g. a tool call whose execution will be reported by a matching `end`). |
|
||||
| `end` | Terminal marker. Carries the canonical resource id so reconnecting clients can refetch persisted state via REST. |
|
||||
| `progress` | Periodic checkpoint with quantifiable completion, e.g. `{"checked": 42, "total": 100}`. |
|
||||
| `created` / `updated` / `deleted` | Resource-lifecycle events for cross-client sync streams. |
|
||||
| `error` | Terminal failure. Carries a stable `code` for client switching and a human-readable `detail`. |
|
||||
|
||||
<Note>
|
||||
Payloads are **flat JSON**. The wire-level `event:` field already names the event type, so do **not** wrap the payload in `{"type": ..., "data": ...}`. Include the canonical resource UUID on terminal events so reconnecting clients can reconcile via REST.
|
||||
</Note>
|
||||
|
||||
## Authentication
|
||||
|
||||
SSE endpoints use the same authentication stack as the rest of the API. Non-browser clients (CLI, programmatic) send the standard `Authorization` header — JWT or API key.
|
||||
|
||||
Browser `EventSource` is the only widely available SSE client API and it **cannot set custom headers**. For that case only, the endpoint accepts a JWT via the `?access_token=<jwt>` query parameter. The header always wins when present — a header is intentional, while a query parameter can leak into referers and logs, so it is consulted only as a fallback.
|
||||
|
||||
```javascript
|
||||
// Browser
|
||||
const es = new EventSource(
|
||||
`/api/v1/scans/${scanId}/event-stream?access_token=${jwt}`
|
||||
);
|
||||
```
|
||||
|
||||
```bash
|
||||
# CLI / programmatic — header, exactly like every other endpoint
|
||||
curl -N -H "Authorization: Bearer $JWT" \
|
||||
https://<host>/api/v1/scans/$SCAN_ID/event-stream
|
||||
```
|
||||
|
||||
## Tenant isolation & security model
|
||||
|
||||
Authorization is enforced at two layers:
|
||||
|
||||
1. **At connect**, `get_channels` runs under the regular DRF stack inside the tenant transaction (`rls_transaction`). Resource lookups are RLS-scoped, so a user cannot even resolve a channel for a resource they cannot see. Narrow the queryset further (e.g. `created_by=request.user`) when a resource is per-user within a tenant.
|
||||
2. **After connect**, `SSEChannelManager.can_read_channel` re-verifies tenant membership by parsing the tenant id embedded in the channel name. Cross-tenant subscription is rejected even if a URL-level check ever has a bug. A malformed channel name is treated as "not authorized".
|
||||
|
||||
Because the tenant id lives inside the channel name, this gate works for any feature without the platform knowing anything about it.
|
||||
|
||||
## Reconnect & state recovery
|
||||
|
||||
The platform deliberately ships **without server-side replay** (`is_channel_reliable` returns `False`). When a client reconnects, it does **not** receive missed events. Instead:
|
||||
|
||||
- Terminal events (`*.end`) carry the canonical resource **UUID**.
|
||||
- On reconnect, the client refetches the authoritative state from the normal REST endpoint using that id.
|
||||
|
||||
Design your event payloads accordingly: deltas are ephemeral and concatenated in-flight; the durable truth always lives behind a REST resource.
|
||||
|
||||
## Local development
|
||||
|
||||
- The dev and production entrypoints both launch Gunicorn with the `asgi` worker (`config.asgi:application`). In dev, `DJANGO_DEBUG=True` enables hot reload; `preload_app` is automatically disabled under debug so edited code is picked up.
|
||||
- SSE uses a **dedicated Valkey database** (`EVENTSTREAM_VALKEY_DB`, default `2`) kept separate from the Celery broker so a noisy broker cannot crowd out streaming traffic. It reuses the same `VALKEY_*` connection settings as the rest of the platform.
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
|---------|---------|---------|
|
||||
| `EVENTSTREAM_VALKEY_DB` | `2` | Valkey DB index for the SSE Pub/Sub bus |
|
||||
| `DJANGO_WORKER_CLASS` | `asgi` | Gunicorn worker class |
|
||||
|
||||
Test the stream end to end with `curl -N` (disable buffering) and an auth header:
|
||||
|
||||
```bash
|
||||
curl -N -H "Authorization: Bearer $JWT" \
|
||||
http://localhost:8080/api/v1/scans/$SCAN_ID/event-stream
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
The platform basis is covered by `api/tests/test_sse.py` (channel parsing, the tenant gate, and auth precedence). For a feature endpoint, test:
|
||||
|
||||
- `get_channels` returns the expected channel for an authorized resource and raises `NotFound`/`PermissionDenied` otherwise.
|
||||
- Each `publish_<event>` helper emits the correct event type and flat payload (mock `send_event`).
|
||||
- The producer builds the channel with `make_channel_name` using the resource's own `tenant_id`.
|
||||
+2
-1
@@ -395,7 +395,8 @@
|
||||
"developer-guide/lighthouse-architecture",
|
||||
"developer-guide/mcp-server",
|
||||
"developer-guide/ai-skills",
|
||||
"developer-guide/prowler-studio"
|
||||
"developer-guide/prowler-studio",
|
||||
"developer-guide/server-sent-events"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -20,7 +20,8 @@ Refer to the [Prowler App Tutorial](/user-guide/tutorials/prowler-app) for detai
|
||||
|
||||
_Commands_:
|
||||
|
||||
```bash
|
||||
<CodeGroup>
|
||||
```bash macOS/Linux
|
||||
VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name)
|
||||
curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
@@ -28,6 +29,15 @@ Refer to the [Prowler App Tutorial](/user-guide/tutorials/prowler-app) for detai
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
```powershell Windows PowerShell
|
||||
$VERSION = (Invoke-RestMethod -Uri "https://api.github.com/repos/prowler-cloud/prowler/releases/latest").tag_name
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/docker-compose.yml" -OutFile "docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/.env" -OutFile ".env"
|
||||
docker compose up -d
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
<Callout icon="lock" iconType="regular" color="#e74c3c">
|
||||
For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
</Callout>
|
||||
@@ -118,8 +128,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.28.0"
|
||||
PROWLER_API_VERSION="5.28.0"
|
||||
PROWLER_UI_VERSION="5.30.0"
|
||||
PROWLER_API_VERSION="5.30.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -4,7 +4,7 @@ title: 'Installation'
|
||||
|
||||
## Installation
|
||||
|
||||
To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/):
|
||||
To install Prowler as a Python package, use `Python >= 3.10, <= 3.13`. Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/):
|
||||
|
||||
<Tabs>
|
||||
<Tab title="pipx">
|
||||
@@ -12,7 +12,7 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
|
||||
_Requirements_:
|
||||
|
||||
* `Python >= 3.10, <= 3.12`
|
||||
* `Python >= 3.10, <= 3.13`
|
||||
* `pipx` installed: [pipx installation](https://pipx.pypa.io/stable/installation/).
|
||||
* AWS, GCP, Azure and/or Kubernetes credentials
|
||||
|
||||
@@ -30,7 +30,7 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
|
||||
_Requirements_:
|
||||
|
||||
* `Python >= 3.10, <= 3.12`
|
||||
* `Python >= 3.10, <= 3.13`
|
||||
* `Python pip >= 21.0.0`
|
||||
* AWS, GCP, Azure, M365 and/or Kubernetes credentials
|
||||
|
||||
@@ -40,12 +40,6 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
pip install prowler
|
||||
prowler -v
|
||||
```
|
||||
|
||||
To upgrade Prowler to the latest version:
|
||||
|
||||
``` bash
|
||||
pip install --upgrade prowler
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="Docker">
|
||||
_Requirements_:
|
||||
@@ -87,7 +81,7 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
<Tab title="Amazon Linux 2">
|
||||
_Requirements_:
|
||||
|
||||
* `Python >= 3.10, <= 3.12`
|
||||
* `Python >= 3.10, <= 3.13`
|
||||
* AWS, GCP, Azure and/or Kubernetes credentials
|
||||
|
||||
_Commands_:
|
||||
@@ -102,8 +96,8 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
<Tab title="Ubuntu">
|
||||
_Requirements_:
|
||||
|
||||
* `Ubuntu 23.04` or above. For older Ubuntu versions, check [pipx installation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#__tabbed_1_1) and ensure `Python >= 3.10, <= 3.12` is installed.
|
||||
* `Python >= 3.10, <= 3.12`
|
||||
* `Ubuntu 23.04` or above. For older Ubuntu versions, check [pipx installation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#__tabbed_1_1) and ensure `Python >= 3.10, <= 3.13` is installed.
|
||||
* `Python >= 3.10, <= 3.13`
|
||||
* AWS, GCP, Azure and/or Kubernetes credentials
|
||||
|
||||
_Commands_:
|
||||
@@ -170,6 +164,68 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Updating Prowler CLI
|
||||
|
||||
Upgrade Prowler CLI to the latest release using the same method chosen for installation:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="pipx">
|
||||
```bash
|
||||
pipx upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="pip">
|
||||
```bash
|
||||
pip install --upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="Docker">
|
||||
Pull the desired image tag to fetch the latest version:
|
||||
|
||||
```bash
|
||||
docker pull toniblyx/prowler:latest
|
||||
```
|
||||
|
||||
<Note>
|
||||
Replace `latest` with a specific release tag (for example, `stable` or `<x.y.z>`) to pin a version. Refer to the [Container Versions](#container-versions) section for the full list of available tags.
|
||||
</Note>
|
||||
</Tab>
|
||||
<Tab title="GitHub">
|
||||
Pull the latest changes and sync the environment:
|
||||
|
||||
```bash
|
||||
cd prowler
|
||||
git pull
|
||||
uv sync
|
||||
uv run python prowler-cli.py -v
|
||||
```
|
||||
|
||||
<Note>
|
||||
To upgrade to a specific release, check out the corresponding tag before syncing: `git checkout <x.y.z>`.
|
||||
</Note>
|
||||
</Tab>
|
||||
<Tab title="Brew">
|
||||
```bash
|
||||
brew upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="CloudShell">
|
||||
Both AWS CloudShell and Azure CloudShell install Prowler with `pipx`, so the upgrade command is the same:
|
||||
|
||||
```bash
|
||||
pipx upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
<Note>
|
||||
To install a specific version instead of the latest release, pin it explicitly. For example, with `pipx`: `pipx install prowler==<x.y.z>`, or with `pip`: `pip install prowler==<x.y.z>`. The available releases are listed in the [Releases GitHub section](https://github.com/prowler-cloud/prowler/releases).
|
||||
</Note>
|
||||
|
||||
## Container Versions
|
||||
|
||||
The available versions of Prowler CLI are the following:
|
||||
|
||||
@@ -141,6 +141,45 @@ Choose one of the following installation methods:
|
||||
|
||||
---
|
||||
|
||||
## Updating Prowler MCP Server
|
||||
|
||||
When running Prowler MCP Server locally ("Option 2: Run Locally"), upgrade to the latest version using the same method chosen for installation. The hosted server (`https://mcp.prowler.com/mcp`) is always kept up to date by Prowler and requires no action.
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Docker">
|
||||
Pull the latest image and restart the container:
|
||||
|
||||
```bash
|
||||
docker pull prowlercloud/prowler-mcp
|
||||
```
|
||||
|
||||
<Note>
|
||||
Recreate any running container after pulling the new image so the updated version takes effect.
|
||||
</Note>
|
||||
</Tab>
|
||||
<Tab title="From Source">
|
||||
Pull the latest changes and sync the dependencies:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
git pull
|
||||
uv sync
|
||||
uv run prowler-mcp --help
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="Build Docker Image">
|
||||
Pull the latest source and rebuild the image:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
git pull
|
||||
docker build -t prowler-mcp .
|
||||
```
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Command Line Options
|
||||
|
||||
The Prowler MCP Server supports the following command-line arguments:
|
||||
|
||||
@@ -127,7 +127,7 @@ Add the following to `.gitlab-ci.yml`:
|
||||
|
||||
```yaml
|
||||
prowler-scan:
|
||||
image: python:3.12-slim
|
||||
image: python:3.13-slim
|
||||
stage: test
|
||||
script:
|
||||
- pip install prowler
|
||||
@@ -154,7 +154,7 @@ stages:
|
||||
- security
|
||||
|
||||
.prowler-base:
|
||||
image: python:3.12-slim
|
||||
image: python:3.13-slim
|
||||
stage: security
|
||||
before_script:
|
||||
- pip install prowler
|
||||
|
||||
@@ -138,6 +138,10 @@ To keep permissions focused:
|
||||
|
||||
4. Continue through the wizard and finish. No principals need to be granted access in step 3 unless you want other identities to impersonate this account.
|
||||
|
||||
<Note>
|
||||
To use this service account with `--organization-id`, additionally grant `roles/cloudasset.viewer` at the organization node and enable the Cloud Asset API in the service account's host project. See [Scanning a Specific GCP Organization](./organization). Without these, organization-wide scans silently fall back to listing only the projects accessible to the service account.
|
||||
</Note>
|
||||
|
||||
### Step 3: Generate a JSON Key
|
||||
|
||||
1. Open the newly created service account, move to the **Keys** tab, and choose **Add key > Create new key**.
|
||||
|
||||
@@ -11,8 +11,19 @@ prowler gcp --organization-id organization-id
|
||||
```
|
||||
|
||||
<Warning>
|
||||
Ensure the credentials used have one of the following roles at the organization level:
|
||||
Cloud Asset Viewer (`roles/cloudasset.viewer`), or Cloud Asset Owner (`roles/cloudasset.owner`).
|
||||
Ensure the credentials used have one of the following roles bound **at the organization node** (not at a project): Cloud Asset Viewer (`roles/cloudasset.viewer`) or Cloud Asset Owner (`roles/cloudasset.owner`). The role must be bound directly on the organization so the Cloud Asset API can enumerate projects across the whole hierarchy.
|
||||
|
||||
```bash
|
||||
gcloud organizations add-iam-policy-binding <organization-id> \
|
||||
--member="serviceAccount:<service-account-email>" \
|
||||
--role="roles/cloudasset.viewer"
|
||||
```
|
||||
|
||||
The Cloud Asset API (`cloudasset.googleapis.com`) must also be enabled in the project that owns the credentials (the service account's host project, or the quota project for user credentials):
|
||||
|
||||
```bash
|
||||
gcloud services enable cloudasset.googleapis.com --project <credentials-project-id>
|
||||
```
|
||||
|
||||
</Warning>
|
||||
<Note>
|
||||
|
||||
@@ -35,14 +35,28 @@ The bundled checks require the following read-only scopes:
|
||||
- `okta.policies.read`
|
||||
- `okta.brands.read`
|
||||
- `okta.apps.read`
|
||||
- `okta.authenticators.read`
|
||||
- `okta.networkZones.read`
|
||||
- `okta.apiTokens.read`
|
||||
- `okta.roles.read`
|
||||
- `okta.groups.read`
|
||||
- `okta.logStreams.read`
|
||||
- `okta.idps.read`
|
||||
|
||||
Additional scopes will be needed as more services and checks are added. These are the current ones needed:
|
||||
|
||||
| Scope | Used by |
|
||||
|---|---|
|
||||
| `okta.policies.read` | Sign-on, password, and authentication policies |
|
||||
| `okta.policies.read` | Sign-on, password, authentication, and `USER_LIFECYCLE` (Workflow > Automations) policies |
|
||||
| `okta.brands.read` | Sign-in page customizations (DOD Notice and Consent Banner check) |
|
||||
| `okta.apps.read` | First-party app settings (Okta Admin Console session), integrated app inventory, and the Authentication Policies bound to Okta applications |
|
||||
| `okta.authenticators.read` | Okta authenticator configuration, including Okta Verify and Smart Card IdP |
|
||||
| `okta.networkZones.read` | Network Zone inventory, anonymized-proxy blocklist checks, and API token Network Zone validation |
|
||||
| `okta.apiTokens.read` | API token metadata and token network conditions |
|
||||
| `okta.roles.read` | Admin role assignments for API token owners (both direct and group-inherited) |
|
||||
| `okta.groups.read` | Group memberships of API token owners, used to resolve admin roles inherited via group assignment (e.g. Super Admin granted through the default admin group) |
|
||||
| `okta.logStreams.read` | Log Stream configuration (`/api/v1/logStreams`) |
|
||||
| `okta.idps.read` | Identity Providers, including Smart Card (X509) IdPs (`/api/v1/idps`) |
|
||||
|
||||
### Required Admin Role
|
||||
|
||||
@@ -68,7 +82,9 @@ Okta filters the first-party apps (`saasure`, `okta_enduser`) out of `/api/v1/ap
|
||||
|
||||
A fifth check — `application_admin_console_session_idle_timeout_15min` (STIG V-273187) — also requires Super Administrator: it calls `GET /api/v1/first-party-app-settings/admin-console`, which returns `403 E0000006` for every role below Super Administrator.
|
||||
|
||||
When the service app runs with Read-Only Administrator, the five checks listed in this section return **MANUAL** instead of PASS/FAIL — the rest of the scan keeps running.
|
||||
`user_inactivity_automation_35d_enabled` (STIG V-273188) reads `USER_LIFECYCLE` policies (`list_policies(type='USER_LIFECYCLE')`) using the `okta.policies.read` scope. The Read-Only Administrator role is enough to list them; no Super Administrator requirement.
|
||||
|
||||
When the service app runs with Read-Only Administrator, the checks listed in this section return **MANUAL** instead of PASS/FAIL — the rest of the scan keeps running.
|
||||
|
||||
<Note>
|
||||
Read-Only Administrator stays the recommended default for the least-privilege framing that aligns with DISA STIG. Assign Super Administrator on a separate run when full coverage of the first-party app checks is needed.
|
||||
@@ -122,7 +138,7 @@ Okta displays the private key **only once**. If you close the modal without copy
|
||||
|
||||
### 5. Grant the required OAuth scopes
|
||||
|
||||
On the app, open the **Okta API Scopes** tab and click **Grant** on every scope Prowler needs. The bundled checks require `okta.policies.read`, `okta.brands.read`, and `okta.apps.read`.
|
||||
On the app, open the **Okta API Scopes** tab and click **Grant** on every scope Prowler needs. The bundled checks require `okta.policies.read`, `okta.brands.read`, `okta.apps.read`, `okta.authenticators.read`, `okta.networkZones.read`, `okta.apiTokens.read`, `okta.roles.read`, `okta.groups.read`, `okta.logStreams.read`, and `okta.idps.read`.
|
||||
|
||||

|
||||
|
||||
@@ -158,8 +174,8 @@ export OKTA_PRIVATE_KEY_FILE="/secure/path/to/prowler-okta.pem"
|
||||
# or
|
||||
export OKTA_PRIVATE_KEY="$(cat /secure/path/to/prowler-okta.pem)"
|
||||
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read,okta.apps.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read"
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
|
||||
uv run python prowler-cli.py okta
|
||||
```
|
||||
@@ -200,7 +216,7 @@ Prowler validates credentials at startup by listing one sign-on policy. This err
|
||||
|
||||
Raised when the credential probe succeeds at the OAuth layer but the request is rejected because the service app lacks the required scope or admin role:
|
||||
|
||||
- **`invalid_scope`** — one of the requested scopes (`okta.policies.read`, `okta.brands.read`, or `okta.apps.read`) is not granted on the service app. Grant the missing scope from **Okta API Scopes**.
|
||||
- **`invalid_scope`** — one of the requested scopes (`okta.policies.read`, `okta.brands.read`, `okta.apps.read`, `okta.authenticators.read`, `okta.networkZones.read`, `okta.apiTokens.read`, `okta.roles.read`, `okta.groups.read`, `okta.logStreams.read`, and `okta.idps.read`) is not granted on the service app. Grant the missing scope from **Okta API Scopes**.
|
||||
- **`Forbidden` / `not authorized`** — no admin role is assigned to the service app. Assign **Read-Only Administrator** (or **Super Administrator** for the first-party application checks) from **Admin roles**.
|
||||
|
||||
### Application-service checks return MANUAL on first-party apps
|
||||
|
||||
@@ -12,7 +12,7 @@ Set up authentication for Okta with the [Okta Authentication](/user-guide/provid
|
||||
|
||||
- An Okta organization. The UI examples below use **Identity Engine** terminology such as **Global Session Policy**; Classic Engine exposes the equivalent sign-on policy concepts under older names.
|
||||
- A **Super Administrator** account on that organization for the one-time service-app setup.
|
||||
- An **API Services** app integration in the Okta Admin Console with the `okta.policies.read`, `okta.brands.read`, and `okta.apps.read` scopes granted and an admin role assigned. **Read-Only Administrator** covers every `signon` check and runs the per-app network-zone check against the apps the service app can see (under Read-Only Administrator that is typically only the service app's own row — the rest of the org's app inventory stays invisible). **Super Administrator** is required additionally to evaluate the five first-party application checks (Okta Admin Console / Okta Dashboard idle timeout, MFA, phishing-resistant authentication) and to widen the network-zone check to the full app inventory — see [Okta Authentication](/user-guide/providers/okta/authentication#required-admin-role) for the full breakdown.
|
||||
- An **API Services** app integration in the Okta Admin Console with the `okta.policies.read`, `okta.brands.read`, `okta.apps.read`, `okta.authenticators.read`, `okta.networkZones.read`, `okta.apiTokens.read`, `okta.roles.read`, `okta.groups.read`, `okta.logStreams.read`, and `okta.idps.read` scopes granted and an admin role assigned. **Read-Only Administrator** covers the Sign-On, Network, API Token, User, System Log, and Identity Provider checks, and runs the per-app application network-zone check against the apps the service app can see (under Read-Only Administrator that is typically only the service app's own row — the rest of the org's app inventory stays invisible). **Super Administrator** is required additionally to evaluate the five first-party application checks (Okta Admin Console / Okta Dashboard idle timeout, MFA, phishing-resistant authentication) and to widen the application network-zone check to the full app inventory — see [Okta Authentication](/user-guide/providers/okta/authentication#required-admin-role) for the full breakdown.
|
||||
- Python 3.10+ and Prowler 5.27.0 or later installed locally.
|
||||
|
||||
<CardGroup cols={2}>
|
||||
@@ -85,8 +85,8 @@ Follow the [Okta Authentication](/user-guide/providers/okta/authentication) guid
|
||||
export OKTA_ORG_DOMAIN="acme.okta.com"
|
||||
export OKTA_CLIENT_ID="0oa1234567890abcdef"
|
||||
export OKTA_PRIVATE_KEY_FILE="/secure/path/to/prowler-okta.pem"
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read,okta.apps.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read"
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
```
|
||||
|
||||
The private key file may contain either a PEM-encoded RSA key or a JWK JSON document.
|
||||
@@ -143,10 +143,16 @@ prowler okta --config-file /path/to/config.yaml
|
||||
|
||||
Prowler for Okta includes security checks across the following services:
|
||||
|
||||
| Service | Description |
|
||||
| --------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Sign-On** | Global session policy controls (idle timeout, lifetime, rule priority and ordering) |
|
||||
| **Application** | Okta Admin Console sign-on settings plus Authentication Policy controls for Okta applications (session idle, MFA, phishing resistance, network zones) |
|
||||
| Service | Description |
|
||||
| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Sign-On** | Global session policy controls (idle timeout, lifetime, rule priority and ordering) |
|
||||
| **Application** | Okta Admin Console sign-on settings plus Authentication Policy controls for Okta applications (session idle, MFA, phishing resistance, network zones) |
|
||||
| **Authenticator** | Password Policy controls plus Okta Verify FIPS and Smart Card IdP authenticator status |
|
||||
| **Network** | Network Zone blocklists for anonymized proxy sources |
|
||||
| **API Token** | API token owner-role validation and Network Zone restrictions |
|
||||
| **User** | User lifecycle automations (inactivity-based deprovisioning) |
|
||||
| **System Log** | Log Stream configuration that off-loads audit records to a central SIEM |
|
||||
| **Identity Provider** | Identity Providers, including Smart Card (X509) IdP status and certificate-chain visibility |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -158,22 +164,29 @@ This is stricter than simply finding the same timeout value somewhere else in th
|
||||
|
||||
### Default Scopes
|
||||
|
||||
Prowler requests a fixed set of OAuth scopes on every token exchange. The defaults cover every bundled check across the Sign-On and Application services:
|
||||
Prowler requests a fixed set of OAuth scopes on every token exchange. The defaults cover every bundled check across the Sign-On, Application, Authenticator, Network, API Token, User, System Log, and Identity Provider services:
|
||||
|
||||
- `okta.policies.read`
|
||||
- `okta.brands.read`
|
||||
- `okta.apps.read`
|
||||
- `okta.authenticators.read`
|
||||
- `okta.networkZones.read`
|
||||
- `okta.apiTokens.read`
|
||||
- `okta.roles.read`
|
||||
- `okta.groups.read`
|
||||
- `okta.logStreams.read`
|
||||
- `okta.idps.read`
|
||||
|
||||
The service app must have these scopes granted in the **Okta API Scopes** tab. When the granted set is narrower than the requested set, the token request fails with an `invalid_scope` error and the scan stops at provider initialization.
|
||||
The service app must have these scopes granted in the **Okta API Scopes** tab. `okta.groups.read` is required so the API token Super Admin check can resolve admin roles inherited via group membership; without it the check falls back to direct-only role assignments and emits a best-effort caveat. When the granted set is narrower than the requested set, the token request fails with an `invalid_scope` error and the scan stops at provider initialization.
|
||||
|
||||
When additional checks are enabled — or when running against a service app that exposes a different scope set — override the default with `OKTA_SCOPES` (comma-separated string for the env var) or `--okta-scopes` (space-separated list for the CLI):
|
||||
|
||||
```bash
|
||||
# Environment variable — comma-separated
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.users.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read,okta.users.read"
|
||||
|
||||
# CLI flag — space-separated
|
||||
prowler okta --okta-scopes okta.policies.read okta.brands.read okta.apps.read okta.users.read
|
||||
prowler okta --okta-scopes okta.policies.read okta.brands.read okta.apps.read okta.authenticators.read okta.networkZones.read okta.apiTokens.read okta.roles.read okta.groups.read okta.logStreams.read okta.idps.read okta.users.read
|
||||
```
|
||||
|
||||
For the full catalog of OAuth scopes exposed by the Okta Management API, refer to the [Okta OAuth 2.0 scopes documentation](https://developer.okta.com/docs/api/oauth2/).
|
||||
|
||||
@@ -47,7 +47,11 @@ Follow these steps to remove a user of your account:
|
||||
1. Navigate to **Users** from the side menu.
|
||||
2. Click the delete button of your current user.
|
||||
|
||||
> **Note: Each user will be able to delete himself and not others, regardless of his permissions.**
|
||||
> **Note: Each user can only delete their own account, regardless of their permissions. For this reason, the delete button is only shown on your own row and not on other users' rows.**
|
||||
|
||||
Deleting a user removes the **entire user account** from Prowler, not just its membership in your organization. Because a single account can belong to more than one tenant, allowing one administrator to delete it outright could affect organizations they don't manage and irreversibly remove another person's identity. To keep this destructive action under the control of the account owner, the API only permits a user to delete themselves (it rejects any other target with a `400` response), and the UI mirrors this by showing the delete button exclusively on your own row.
|
||||
|
||||
To remove **another** user from your organization, use the [_Expel from organization_](/user-guide/tutorials/prowler-app-multi-tenant#expelling-a-user-from-an-organization) action instead. Expelling removes the user's membership, role grants, and active sessions for your tenant only, and deletes the underlying account just for that user if your organization was their last remaining membership. This action is reserved for tenant **owners**.
|
||||
|
||||
<img src="/images/prowler-app/rbac/user_remove.png" alt="Remove User" width="700" />
|
||||
|
||||
|
||||
@@ -108,10 +108,10 @@ Prowler App updates user attributes each time a user logs in. Any changes made i
|
||||
The `userType` attribute controls which Prowler role is assigned to the user:
|
||||
|
||||
- If `userType` matches an existing Prowler role name, the user receives that role automatically.
|
||||
- If `userType` does not match any existing role, Prowler App creates a new role with that name **without permissions**.
|
||||
- If `userType` is not set, the user receives the `no_permissions` role.
|
||||
- If `userType` does not match any existing role, Prowler App creates a new role with that name **with read-only access** (visibility over all providers, no management permissions). A Prowler administrator can adjust its permissions afterward through the [RBAC Management](/user-guide/tutorials/prowler-app-rbac) tab.
|
||||
- If `userType` is not set, the user's existing roles are left unchanged.
|
||||
|
||||
In all cases where the resulting role has no permissions, a Prowler administrator must configure the appropriate permissions through the [RBAC Management](/user-guide/tutorials/prowler-app-rbac) tab. The `userType` value is **case-sensitive** - for example, `Backend` and `backend` are treated as different roles.
|
||||
The `userType` value is **case-sensitive** - for example, `Backend` and `backend` are treated as different roles.
|
||||
|
||||
</Warning>
|
||||
|
||||
@@ -223,9 +223,9 @@ To test the `userType` → role mapping, set the **Department** attribute in the
|
||||
After a successful SSO login, the user profile in Prowler App reflects the attributes sent by Google Workspace:
|
||||
|
||||
- **Name**: Populated from the `firstName` and `lastName` attributes.
|
||||
- **Role**: Created automatically from the `userType` attribute (e.g., `Backend`). If the role did not exist previously, it is created with no permissions by default.
|
||||
- **Permissions**: In the screenshot below, the user has no permissions because the `Backend` role did not exist prior to login and was created automatically without any permissions. To resolve this, a Prowler administrator can either:
|
||||
- Assign the appropriate permissions to the new role via the [RBAC Management](/user-guide/tutorials/prowler-app-rbac) tab.
|
||||
- **Role**: Created automatically from the `userType` attribute (e.g., `Backend`). If the role did not exist previously, it is created with read-only access by default.
|
||||
- **Permissions**: If the assigned permissions need to be adjusted, a Prowler administrator can either:
|
||||
- Edit the permissions of the new role via the [RBAC Management](/user-guide/tutorials/prowler-app-rbac) tab.
|
||||
- Set the `userType` attribute in the IdP to match an existing Prowler role that already has the desired permissions. The updated role is applied on the next SAML login.
|
||||
|
||||
For more details on role assignment behavior and attribute mapping, refer to the [SAML SSO Configuration](/user-guide/tutorials/prowler-app-sso#configure-attribute-mapping-in-the-idp) page.
|
||||
|
||||
@@ -87,7 +87,7 @@ Choose a Method:
|
||||
|----------------|---------------------------------------------------------------------------------------------------------|----------|
|
||||
| `firstName` | The user's first name. | Yes |
|
||||
| `lastName` | The user's last name. | Yes |
|
||||
| `userType` | Determines which Prowler role the user receives (e.g., `admin`, `auditor`). If a role with that name already exists, the user receives it automatically; if it does not exist, Prowler App creates a new role with that name without permissions. If `userType` is not defined, the user is assigned the `no_permissions` role. Role permissions can be edited in the [RBAC Management tab](/user-guide/tutorials/prowler-app-rbac). | No |
|
||||
| `userType` | Determines which Prowler role the user receives (e.g., `admin`, `auditor`). If a role with that name already exists, the user receives it automatically; if it does not exist, Prowler App creates a new role with that name with read-only access (visibility over all providers, no management permissions). If `userType` is not defined, the user's existing roles are left unchanged. Role permissions can be edited in the [RBAC Management tab](/user-guide/tutorials/prowler-app-rbac). | No |
|
||||
| `organization` | The user's company name. | No |
|
||||
|
||||
<Info>
|
||||
@@ -140,7 +140,7 @@ Choose a Method:
|
||||

|
||||
|
||||
* **Organization** (`organization`): Maps to the company name displayed in Prowler App. This attribute is optional.
|
||||
* **User type** (`userType`): Determines the Prowler role assigned to the user. This attribute is **case-sensitive** and must match the exact name of an existing role in Prowler App.
|
||||
* **User type** (`userType`): Determines the Prowler role assigned to the user. This attribute is **case-sensitive**: if it matches the exact name of an existing role in Prowler App the user receives that role; if no role with that name exists, a new one is created with read-only access.
|
||||
|
||||

|
||||
|
||||
@@ -152,14 +152,10 @@ Choose a Method:
|
||||
The `userType` attribute controls which Prowler role is assigned to the user:
|
||||
|
||||
* If a role with the specified name already exists in Prowler App, the user automatically receives that role.
|
||||
* If the role does not exist, Prowler App creates a new role with that exact name but without any permissions, preventing the user from performing any actions.
|
||||
* If `userType` is not defined in the user's Okta profile, the user is assigned the `no_permissions` role.
|
||||
* If the role does not exist, Prowler App creates a new role with that exact name with read-only access: the user can see all providers and their findings but cannot manage anything. A Prowler administrator (a user whose role includes the "Manage Account" permission) can adjust its permissions afterward through the [RBAC Management tab](/user-guide/tutorials/prowler-app-rbac).
|
||||
* If `userType` is not defined in the user's Okta profile, the user's existing roles in Prowler App are left unchanged.
|
||||
|
||||
In all cases where the resulting role has no permissions, a Prowler administrator (a user whose role includes the "Manage Account" permission) must configure the appropriate permissions through the [RBAC Management tab](/user-guide/tutorials/prowler-app-rbac).
|
||||
|
||||
This behavior is intentional: by defaulting to no permissions, Prowler App ensures that a misconfiguration in Okta cannot inadvertently grant elevated access.
|
||||
|
||||
**Example:** To assign the `IT` role to a user, set the `userType` value to `IT` in Okta. If a role named `IT` already exists in Prowler App, the user receives it automatically upon login. If it does not exist, Prowler App creates a new role called `IT` without permissions, and a Prowler administrator must configure the desired permissions for it.
|
||||
**Example:** To assign the `IT` role to a user, set the `userType` value to `IT` in Okta. If a role named `IT` already exists in Prowler App, the user receives it automatically upon login. If it does not exist, Prowler App creates a new role called `IT` with read-only access, and a Prowler administrator can adjust its permissions as needed.
|
||||
|
||||
</Warning>
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# =============================================================================
|
||||
# Build stage - Install dependencies and build the application
|
||||
# =============================================================================
|
||||
FROM ghcr.io/astral-sh/uv:python3.13-alpine@sha256:8f53782bb232ab0b5558f3071e86e2bbfde884e18815f2b19cc57f2d336e9ee2 AS builder
|
||||
FROM ghcr.io/astral-sh/uv:0.11.21-python3.13-alpine3.23@sha256:f09cc61ffc001f202701fdeae14dbdd50f6ca4cfcf248f41fd3234a302c8534f AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -25,7 +25,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
# =============================================================================
|
||||
# Final stage - Minimal runtime environment
|
||||
# =============================================================================
|
||||
FROM python:3.13-alpine@sha256:bb1f2fdb1065c85468775c9d680dcd344f6442a2d1181ef7916b60a623f11d40
|
||||
FROM python:3.13.14-alpine3.23@sha256:b0513989fa9be54569cac73f48a60320b74bb0f9ffa886568eea7e48a2432c04
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud"
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user