mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-09 21:04:53 +00:00
Compare commits
77 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9a50dffaa0 | |||
| e710ebff1c | |||
| b3caee88e4 | |||
| d9f90e50b8 | |||
| 58efb719fa | |||
| 355b7071aa | |||
| b994b0b14e | |||
| 6c559fbb8d | |||
| b2d74711d9 | |||
| 7e60e8f8da | |||
| 62955dd16b | |||
| 1f7caa6394 | |||
| 662e7e9e18 | |||
| e3013d9918 | |||
| 0ea2f6d67e | |||
| 7692a1d76a | |||
| 1c9afc714e | |||
| 466f1a3d73 | |||
| 061fbaa7bb | |||
| 28b045302f | |||
| 5a2226c02c | |||
| 6f172a5c19 | |||
| a7d180ea5b | |||
| d4bbc8b5ad | |||
| a5bc226f11 | |||
| 3a3d9d6146 | |||
| bcd282d3d0 | |||
| eb7949c884 | |||
| e60a4462e5 | |||
| f7f8747512 | |||
| d573af911d | |||
| cf9beb8234 | |||
| 7f67eac1bf | |||
| a652e28b4a | |||
| 1b17304c4a | |||
| c2cef99b33 | |||
| a769e37615 | |||
| 9d2a8d9108 | |||
| e05519ff9f | |||
| 67b26072f8 | |||
| 2222082631 | |||
| 8b0cb4b981 | |||
| 9422eff8ab | |||
| e3c4368d32 | |||
| 2a641b39c8 | |||
| 02b713572b | |||
| 74251350bc | |||
| 8f745cdbe6 | |||
| 81226cd837 | |||
| a2824f7166 | |||
| edbbd86828 | |||
| c58dad2ca4 | |||
| b4befe3a10 | |||
| d98933c2e7 | |||
| 03dfa3816d | |||
| ad1261ce54 | |||
| 3252f9cf19 | |||
| f1cdf3df15 | |||
| 03ddb8a708 | |||
| 2678c6bc9f | |||
| 48c071297f | |||
| 7e9a16d022 | |||
| 84b388f649 | |||
| 671d0c746c | |||
| 0e4b117161 | |||
| a70bc3c1c7 | |||
| 723d161c63 | |||
| d560020592 | |||
| 00451f8239 | |||
| 329dfdf8e6 | |||
| 4c59af93eb | |||
| 6ca8e726f7 | |||
| 546eb2d85a | |||
| ec3efc94f5 | |||
| 6cffd0d17f | |||
| 528d32601b | |||
| 56b3044aae |
+8
-1
@@ -11,7 +11,14 @@ envs = "wt step copy-ignored"
|
||||
[[pre-start]]
|
||||
deps = "uv sync"
|
||||
|
||||
# Block 3: reminder - last visible output before `wt switch` returns.
|
||||
# Block 3: prepare pnpm via corepack.
|
||||
[[pre-start]]
|
||||
corepack-enable = "corepack enable"
|
||||
|
||||
[[pre-start]]
|
||||
corepack-install = "cd ui && corepack install"
|
||||
|
||||
# Block 4: reminder - last visible output before `wt switch` returns.
|
||||
# Hooks can't mutate the parent shell, so venv activation is manual.
|
||||
[[pre-start]]
|
||||
reminder = "echo '>> Reminder: activate the venv in this shell with: source .venv/bin/activate'"
|
||||
|
||||
@@ -145,7 +145,7 @@ SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.28.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.30.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
name: 'Docs: Markdown Lint'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
markdown-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
release-assets.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: ui/.nvmrc
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
||||
with:
|
||||
package_json_file: ui/package.json
|
||||
run_install: false
|
||||
|
||||
- name: Run markdownlint
|
||||
# Pin must match .pre-commit-config.yaml so prek and CI behave identically.
|
||||
# pnpm dlx doesn't accept --ignore-scripts as a flag; the env var
|
||||
# disables postinstall scripts on transitives the same way.
|
||||
env:
|
||||
pnpm_config_ignore_scripts: 'true'
|
||||
run: pnpm dlx markdownlint-cli@0.45.0 '**/*.md'
|
||||
@@ -540,7 +540,82 @@ jobs:
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-vercel
|
||||
files: ./vercel_coverage.xml
|
||||
|
||||
# Scaleway Provider
|
||||
- name: Check if Scaleway files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-scaleway
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/scaleway/**
|
||||
./tests/**/scaleway/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run Scaleway tests
|
||||
if: steps.changed-scaleway.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/scaleway --cov-report=xml:scaleway_coverage.xml tests/providers/scaleway
|
||||
|
||||
- name: Upload Scaleway coverage to Codecov
|
||||
if: steps.changed-scaleway.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-scaleway
|
||||
files: ./scaleway_coverage.xml
|
||||
|
||||
# StackIT Provider
|
||||
- name: Check if StackIT files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-stackit
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/stackit/**
|
||||
./tests/**/stackit/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run StackIT tests
|
||||
if: steps.changed-stackit.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/stackit --cov-report=xml:stackit_coverage.xml tests/providers/stackit
|
||||
|
||||
- name: Upload StackIT coverage to Codecov
|
||||
if: steps.changed-stackit.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-stackit
|
||||
files: ./stackit_coverage.xml
|
||||
|
||||
# External Provider (dynamic loading)
|
||||
- name: Check if External Provider files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-external
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/common/**
|
||||
./prowler/config/**
|
||||
./prowler/lib/**
|
||||
./tests/providers/external/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run External Provider tests
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/common --cov=./prowler/config --cov=./prowler/lib --cov-report=xml:external_coverage.xml tests/providers/external
|
||||
|
||||
- name: Upload External Provider coverage to Codecov
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-external
|
||||
files: ./external_coverage.xml
|
||||
|
||||
# Lib
|
||||
- name: Check if Lib files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: '24.13.0'
|
||||
node-version-file: 'ui/.nvmrc'
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
||||
|
||||
@@ -16,7 +16,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: "24.13.0"
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -93,11 +92,11 @@ jobs:
|
||||
ui/vitest.config.ts
|
||||
ui/vitest.setup.ts
|
||||
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
- name: Setup Node.js
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
node-version-file: 'ui/.nvmrc'
|
||||
|
||||
- name: Setup pnpm
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "markdownlint/style/prettier",
|
||||
"first-line-h1": false,
|
||||
"no-duplicate-heading": {
|
||||
"siblings_only": true
|
||||
},
|
||||
"no-inline-html": false,
|
||||
"line-length": false,
|
||||
"no-bare-urls": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
node_modules/
|
||||
ui/node_modules/
|
||||
.git/
|
||||
.venv/
|
||||
**/.venv/
|
||||
dist/
|
||||
build/
|
||||
htmlcov/
|
||||
.next/
|
||||
ui/.next/
|
||||
ui/out/
|
||||
contrib/
|
||||
|
||||
# Auto-generated content (keepachangelog format legitimately repeats section headings).
|
||||
# Revisit with the team — see beads task on markdownlint rule triage.
|
||||
**/CHANGELOG.md
|
||||
@@ -133,6 +133,13 @@ repos:
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
## MARKDOWN
|
||||
- repo: https://github.com/igorshubovych/markdownlint-cli
|
||||
rev: v0.45.0
|
||||
hooks:
|
||||
- id: markdownlint
|
||||
priority: 30
|
||||
|
||||
## CONTAINERS
|
||||
- repo: https://github.com/hadolint/hadolint
|
||||
rev: v2.14.0
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
Use these skills for detailed patterns on-demand:
|
||||
|
||||
### Generic Skills (Any Project)
|
||||
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
@@ -28,6 +29,7 @@ Use these skills for detailed patterns on-demand:
|
||||
| `tdd` | Test-Driven Development workflow | [SKILL.md](skills/tdd/SKILL.md) |
|
||||
|
||||
### Prowler-Specific Skills
|
||||
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
|
||||
|
||||
+4
-3
@@ -1,4 +1,4 @@
|
||||
# Do you want to learn on how to...
|
||||
# Do you want to learn on how to
|
||||
|
||||
- [Contribute with your code or fixes to Prowler](https://docs.prowler.com/developer-guide/introduction)
|
||||
- [Create a new provider](https://docs.prowler.com/developer-guide/provider)
|
||||
@@ -32,5 +32,6 @@ Provider-specific developer notes:
|
||||
|
||||
Want some swag as appreciation for your contribution?
|
||||
|
||||
# Prowler Developer Guide
|
||||
https://goto.prowler.com/devguide
|
||||
## Prowler Developer Guide
|
||||
|
||||
<https://goto.prowler.com/devguide>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<p align="center">
|
||||
<img align="center" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-black.png#gh-light-mode-only" width="50%" height="50%">
|
||||
<img align="center" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-black.png#gh-light-mode-only" width="50%" height="50%">
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
</p>
|
||||
<p align="center">
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
@@ -22,8 +22,8 @@
|
||||
<a href="https://pypistats.org/packages/prowler"><img alt="PyPI Downloads" src="https://img.shields.io/pypi/dw/prowler.svg?label=downloads"></a>
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img alt="Codecov coverage" src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img alt="Linux Foundation insights health score" src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://github.com/prowler-cloud/prowler/releases"><img alt="Version" src="https://img.shields.io/github/v/release/prowler-cloud/prowler"></a>
|
||||
@@ -36,7 +36,7 @@
|
||||
</p>
|
||||
<hr>
|
||||
<p align="center">
|
||||
<img align="center" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
<img align="center" alt="Prowler Cloud demo" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
</p>
|
||||
|
||||
# Description
|
||||
@@ -122,6 +122,7 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
| Vercel | 26 | 6 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 4 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
@@ -146,11 +147,13 @@ Prowler App offers flexible installation methods tailored to various environment
|
||||
|
||||
### Docker Compose
|
||||
|
||||
**Requirements**
|
||||
#### Requirements
|
||||
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
- `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands**
|
||||
#### Commands
|
||||
|
||||
_macOS/Linux:_
|
||||
|
||||
``` console
|
||||
VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name)
|
||||
@@ -160,6 +163,16 @@ curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${V
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
_Windows PowerShell:_
|
||||
|
||||
``` powershell
|
||||
$VERSION = (Invoke-RestMethod -Uri "https://api.github.com/repos/prowler-cloud/prowler/releases/latest").tag_name
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/docker-compose.yml" -OutFile "docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/.env" -OutFile ".env"
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
|
||||
@@ -175,14 +188,14 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
|
||||
### From GitHub
|
||||
|
||||
**Requirements**
|
||||
#### Requirements
|
||||
|
||||
* `git` installed.
|
||||
* `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
* `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
- `git` installed.
|
||||
- `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
- `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
- `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands to run the API**
|
||||
#### Commands to run the API
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -199,7 +212,7 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
> After completing the setup, access the API documentation at http://localhost:8080/api/v1/docs.
|
||||
|
||||
**Commands to run the API Worker**
|
||||
#### Commands to run the API Worker
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -212,7 +225,7 @@ cd src/backend
|
||||
python -m celery -A config.celery worker -l info -E
|
||||
```
|
||||
|
||||
**Commands to run the API Scheduler**
|
||||
#### Commands to run the API Scheduler
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -225,7 +238,7 @@ cd src/backend
|
||||
python -m celery -A config.celery beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
```
|
||||
|
||||
**Commands to run the UI**
|
||||
#### Commands to run the UI
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -237,7 +250,7 @@ pnpm start
|
||||
|
||||
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
**Pre-commit Hooks Setup**
|
||||
#### Pre-commit Hooks Setup
|
||||
|
||||
Some pre-commit hooks require tools installed on your system:
|
||||
|
||||
@@ -257,14 +270,14 @@ prowler -v
|
||||
|
||||
### Containers
|
||||
|
||||
**Available Versions of Prowler CLI**
|
||||
#### Available Versions of Prowler CLI
|
||||
|
||||
The following versions of Prowler CLI are available, depending on your requirements:
|
||||
|
||||
- `latest`: Synchronizes with the `master` branch. Note that this version is not stable.
|
||||
- `v4-latest`: Synchronizes with the `v4` branch. Note that this version is not stable.
|
||||
- `v3-latest`: Synchronizes with the `v3` branch. Note that this version is not stable.
|
||||
- `<x.y.z>` (release): Stable releases corresponding to specific versions. You can find the complete list of releases [here](https://github.com/prowler-cloud/prowler/releases).
|
||||
- `<x.y.z>` (release): Stable releases corresponding to specific versions. See the [complete list of Prowler releases](https://github.com/prowler-cloud/prowler/releases).
|
||||
- `stable`: Always points to the latest release.
|
||||
- `v4-stable`: Always points to the latest release for v4.
|
||||
- `v3-stable`: Always points to the latest release for v3.
|
||||
@@ -293,7 +306,7 @@ python prowler-cli.py -v
|
||||
|
||||
# 🛡️ GitHub Action
|
||||
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
|
||||
```yaml
|
||||
name: Prowler IaC Scan
|
||||
@@ -338,7 +351,7 @@ Full configuration, per-provider authentication, and SARIF examples: [Prowler Gi
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
**Running Prowler**
|
||||
### Running Prowler
|
||||
|
||||
Prowler can be executed across various environments, offering flexibility to meet your needs. It can be run from:
|
||||
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@ inputs:
|
||||
required: false
|
||||
default: json-ocsf
|
||||
push-to-cloud:
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-import-findings#using-the-cli
|
||||
required: false
|
||||
default: "false"
|
||||
flags:
|
||||
@@ -299,7 +299,7 @@ runs:
|
||||
echo ""
|
||||
echo "**Get started in 3 steps:**"
|
||||
echo "1. Create an account at [cloud.prowler.com](https://cloud.prowler.com)"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli))"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-import-findings#using-the-cli))"
|
||||
echo "3. Add \`PROWLER_CLOUD_API_KEY\` to your GitHub secrets and set \`push-to-cloud: true\` on this action"
|
||||
echo ""
|
||||
echo "See [prowler.com/pricing](https://prowler.com/pricing) for plan details."
|
||||
|
||||
+4
-4
@@ -10,7 +10,7 @@
|
||||
> - [`jsonapi`](../skills/jsonapi/SKILL.md) - Strict JSON:API v1.1 spec compliance
|
||||
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
|
||||
|
||||
### Auto-invoke Skills
|
||||
## Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
@@ -81,7 +81,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
## DECISION TREES
|
||||
|
||||
### Serializer Selection
|
||||
```
|
||||
```text
|
||||
Read → <Model>Serializer
|
||||
Create → <Model>CreateSerializer
|
||||
Update → <Model>UpdateSerializer
|
||||
@@ -89,7 +89,7 @@ Nested read → <Model>IncludeSerializer
|
||||
```
|
||||
|
||||
### Task vs View
|
||||
```
|
||||
```text
|
||||
< 100ms → View
|
||||
> 100ms or external API → Celery task
|
||||
Needs retry → Celery task
|
||||
@@ -105,7 +105,7 @@ Django 5.1.x | DRF 3.15.x | djangorestframework-jsonapi 7.x | Celery 5.4.x | Pos
|
||||
|
||||
## PROJECT STRUCTURE
|
||||
|
||||
```
|
||||
```text
|
||||
api/src/backend/
|
||||
├── api/ # Main Django app
|
||||
│ ├── v1/ # API version 1 (views, serializers, urls)
|
||||
|
||||
+56
-1
@@ -2,6 +2,62 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.31.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Opt-in automatic recovery of allowlisted idempotent background tasks whose worker died during a deploy or crash: when enabled via `DJANGO_TASK_RECOVERY_ENABLED` (off by default), stuck summary and deletion tasks are detected and re-run instead of staying pending forever (scan and Jira tasks are excluded), with a `reconcile_orphan_tasks` management command for on-demand recovery [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- DORA compliance framework support [(#11131)](https://github.com/prowler-cloud/prowler/pull/11131)
|
||||
- Label Postgres connections with `application_name="<component>:<alias>"` (component injected per process via `DJANGO_APP_COMPONENT`) so connections are attributable by component in `pg_stat_activity` [(#11494)](https://github.com/prowler-cloud/prowler/pull/11494)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Allowlisted idempotent background tasks are no longer lost when a worker is stopped or crashes mid-task; tasks with external side effects are marked terminal instead of blindly re-running [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Workers now shut down gracefully on deploy or restart, finishing or re-queueing in-flight tasks instead of being force-killed and leaving them stuck [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `dulwich` from 0.23.0 to 1.2.5 and `pyjwt` from 2.12.1 to 2.13.0, patching `GHSA-897w-fcg9-f6xj` (arbitrary file write) and `PYSEC-2026-179` (HMAC/JWK key confusion) flagged by osv-scanner in `api/uv.lock` [(#11499)](https://github.com/prowler-cloud/prowler/pull/11499)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.3] (Prowler v5.29.3)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- API startup no longer crashes when Neo4j is unreachable, as the Neo4j driver now connects lazily on first use rather than during app initialization [(#11491)](https://github.com/prowler-cloud/prowler/pull/11491)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.1] (Prowler v5.29.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `GET /api/v1/findings` N+1 query loading `resources__tags` when listing findings [(#11420)](https://github.com/prowler-cloud/prowler/pull/11420)
|
||||
- Clean up the scan tmp output directory when `scan-report` fails so partial files do not accumulate and fill the worker disk (`No space left on device`) [(#11421)](https://github.com/prowler-cloud/prowler/pull/11421)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.0] (Prowler v5.29.0)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Scan finding ingestion: bulk-resolve `Resource`/`ResourceTag` rows, replace per-mapping `SELECT FOR UPDATE` with deferred `ResourceTagMapping.bulk_create(ignore_conflicts=True)`, wrap each micro-batch in a single `rls_transaction`, and raise `SCAN_DB_BATCH_SIZE` to 1000 [(#11249)](https://github.com/prowler-cloud/prowler/pull/11249)
|
||||
- Faster `GET /api/v1/finding-groups/latest` aggregation on tenants where one recent scan holds most findings [(#11380)](https://github.com/prowler-cloud/prowler/pull/11380)
|
||||
|
||||
---
|
||||
|
||||
## [1.29.1] (Prowler v5.28.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `finding-groups` slow response with finding-level filters such as `region`; check title and description are now read from the daily summaries, which drops sorting by `check_title` [(#11326)](https://github.com/prowler-cloud/prowler/pull/11326)
|
||||
|
||||
---
|
||||
|
||||
## [1.29.0] (Prowler v5.28.0)
|
||||
|
||||
### 🚀 Added
|
||||
@@ -29,7 +85,6 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- `perform_scan_task` and `perform_scheduled_scan_task` now short-circuit with a warning and `return None` when the target provider no longer exists, instead of letting `handle_provider_deletion` raise `ProviderDeletedException`. `perform_scheduled_scan_task` also removes any orphan `PeriodicTask` it finds so beat stops re-firing scans for deleted providers. Prevents queued messages for deleted providers from being recorded as `FAILURE` [(#11185)](https://github.com/prowler-cloud/prowler/pull/11185)
|
||||
- Attack Paths: `BEDROCK-001` and `BEDROCK-002` now target roles trusting `bedrock-agentcore.amazonaws.com` instead of `bedrock.amazonaws.com`, eliminating false positives against regular Bedrock service roles (Agents, Knowledge Bases, model invocation) [(#11141)](https://github.com/prowler-cloud/prowler/pull/11141)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## [1.27.1] (Prowler v5.26.1)
|
||||
|
||||
+29
-29
@@ -2,7 +2,7 @@
|
||||
|
||||
This repository contains the JSON API and Task Runner components for Prowler, which facilitate a complete backend that interacts with the Prowler SDK and is used by the Prowler UI.
|
||||
|
||||
# Components
|
||||
## Components
|
||||
The Prowler API is composed of the following components:
|
||||
|
||||
- The JSON API, which is an API built with Django Rest Framework.
|
||||
@@ -10,13 +10,13 @@ The Prowler API is composed of the following components:
|
||||
- The PostgreSQL database, which is used to store the data.
|
||||
- The Valkey database, which is an in-memory database which is used as a message broker for the Celery workers.
|
||||
|
||||
## Note about Valkey
|
||||
### Note about Valkey
|
||||
|
||||
[Valkey](https://valkey.io/) is an open source (BSD) high performance key/value datastore.
|
||||
|
||||
Valkey exposes a Redis 7.2 compliant API. Any service that exposes the Redis API can be used with Prowler API.
|
||||
|
||||
# Modify environment variables
|
||||
## Modify environment variables
|
||||
|
||||
Under the root path of the project, you can find a file called `.env`. This file shows all the environment variables that the project uses. You should review it and set the values for the variables you want to change.
|
||||
|
||||
@@ -24,7 +24,7 @@ If you don’t set `DJANGO_TOKEN_SIGNING_KEY` or `DJANGO_TOKEN_VERIFYING_KEY`, t
|
||||
|
||||
**Important note**: Every Prowler version (or repository branches and tags) could have different variables set in its `.env` file. Please use the `.env` file that corresponds with each version.
|
||||
|
||||
## Local deployment
|
||||
### Local deployment
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the virtual environment, not before. Otherwise, variables will not be loaded properly.
|
||||
|
||||
To do this, you can run:
|
||||
@@ -34,12 +34,12 @@ set -a
|
||||
source .env
|
||||
```
|
||||
|
||||
# 🚀 Production deployment
|
||||
## Docker deployment
|
||||
## 🚀 Production deployment
|
||||
### Docker deployment
|
||||
|
||||
This method requires `docker` and `docker compose`.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -50,13 +50,13 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Build the base image
|
||||
#### Build the base image
|
||||
|
||||
```console
|
||||
docker compose --profile prod build
|
||||
```
|
||||
|
||||
### Run the production service
|
||||
#### Run the production service
|
||||
|
||||
This command will start the Django production server and the Celery worker and also the Valkey and PostgreSQL databases.
|
||||
|
||||
@@ -68,7 +68,7 @@ You can access the server in `http://localhost:8080`.
|
||||
|
||||
> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts.
|
||||
|
||||
### View the Production Server Logs
|
||||
#### View the Production Server Logs
|
||||
|
||||
To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern:
|
||||
|
||||
@@ -133,13 +133,13 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
> By default, the Gunicorn server will try to use as many workers as your machine can handle. You can manually change that in the `src/backend/config/guniconf.py` file.
|
||||
|
||||
# 🧪 Development guide
|
||||
## 🧪 Development guide
|
||||
|
||||
## Local deployment
|
||||
### Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -150,7 +150,7 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Start the PostgreSQL Database and Valkey
|
||||
#### Start the PostgreSQL Database and Valkey
|
||||
|
||||
The PostgreSQL database (version 16.3) and Valkey (version 7) are required for the development environment. To make development easier, we have provided a `docker-compose` file that will start these components for you.
|
||||
|
||||
@@ -161,7 +161,7 @@ The PostgreSQL database (version 16.3) and Valkey (version 7) are required for t
|
||||
docker compose up postgres valkey -d
|
||||
```
|
||||
|
||||
### Install the Python dependencies
|
||||
#### Install the Python dependencies
|
||||
|
||||
> You must have uv installed
|
||||
|
||||
@@ -169,7 +169,7 @@ docker compose up postgres valkey -d
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Apply migrations
|
||||
#### Apply migrations
|
||||
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
@@ -178,7 +178,7 @@ cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
### Run the Django development server
|
||||
#### Run the Django development server
|
||||
|
||||
```console
|
||||
cd src/backend
|
||||
@@ -188,7 +188,7 @@ python manage.py runserver
|
||||
You can access the server in `http://localhost:8000`.
|
||||
All changes in the code will be automatically reloaded in the server.
|
||||
|
||||
### Run the Celery worker
|
||||
#### Run the Celery worker
|
||||
|
||||
```console
|
||||
python -m celery -A config.celery worker -l info -E
|
||||
@@ -196,11 +196,11 @@ python -m celery -A config.celery worker -l info -E
|
||||
|
||||
The Celery worker does not detect and reload changes in the code, so you need to restart it manually when you make changes.
|
||||
|
||||
## Docker deployment
|
||||
### Docker deployment
|
||||
|
||||
This method requires `docker` and `docker compose`.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -211,13 +211,13 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Build the base image
|
||||
#### Build the base image
|
||||
|
||||
```console
|
||||
docker compose --profile dev build
|
||||
```
|
||||
|
||||
### Run the development service
|
||||
#### Run the development service
|
||||
|
||||
This command will start the Django development server and the Celery worker and also the Valkey and PostgreSQL databases.
|
||||
|
||||
@@ -230,7 +230,7 @@ All changes in the code will be automatically reloaded in the server.
|
||||
|
||||
> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts.
|
||||
|
||||
### View the development server logs
|
||||
#### View the development server logs
|
||||
|
||||
To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern:
|
||||
|
||||
@@ -238,7 +238,7 @@ To view the logs for any component (e.g., Django, Celery worker), you can use th
|
||||
docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
```
|
||||
|
||||
## Applying migrations
|
||||
### Applying migrations
|
||||
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
@@ -247,7 +247,7 @@ cd src/backend
|
||||
uv run python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
## Apply fixtures
|
||||
### Apply fixtures
|
||||
|
||||
Fixtures are used to populate the database with initial development data.
|
||||
|
||||
@@ -258,7 +258,7 @@ uv run python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
|
||||
> The default credentials are `dev@prowler.com:Thisisapassword123@` or `dev2@prowler.com:Thisisapassword123@`
|
||||
|
||||
## Run tests
|
||||
### Run tests
|
||||
|
||||
Note that the tests will fail if you use the same `.env` file as the development environment.
|
||||
|
||||
@@ -269,7 +269,7 @@ cd src/backend
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
# Custom commands
|
||||
## Custom commands
|
||||
|
||||
Django provides a way to create custom commands that can be run from the command line.
|
||||
|
||||
@@ -281,7 +281,7 @@ To run a custom command, you need to be in the `prowler/api/src/backend` directo
|
||||
uv run python manage.py <command_name>
|
||||
```
|
||||
|
||||
## Generate dummy data
|
||||
### Generate dummy data
|
||||
|
||||
```console
|
||||
python manage.py findings --tenant
|
||||
@@ -298,7 +298,7 @@ This command creates, for a given tenant, a provider, scan and a set of findings
|
||||
>
|
||||
> The last step is required to access the findings details, since the UI needs that to print all the information.
|
||||
|
||||
### Example
|
||||
#### Example
|
||||
|
||||
```console
|
||||
~/backend $ uv run python manage.py findings --tenant
|
||||
|
||||
@@ -22,12 +22,12 @@ apply_fixtures() {
|
||||
|
||||
start_dev_server() {
|
||||
echo "Starting the development server..."
|
||||
uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
exec uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
}
|
||||
|
||||
start_prod_server() {
|
||||
echo "Starting the Gunicorn server..."
|
||||
uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
exec uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
}
|
||||
|
||||
resolve_worker_hostname() {
|
||||
@@ -47,7 +47,7 @@ resolve_worker_hostname() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
uv run python -m celery -A config.celery worker \
|
||||
exec uv run python -m celery -A config.celery worker \
|
||||
-n "$(resolve_worker_hostname)" \
|
||||
-l "${DJANGO_LOGGING_LEVEL:-info}" \
|
||||
-Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \
|
||||
@@ -56,7 +56,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
exec uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
manage_db_partitions() {
|
||||
@@ -68,6 +68,15 @@ manage_db_partitions() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Identify this process to Postgres (application_name=<component>:<alias>) so
|
||||
# connections are attributable by component in pg_stat_activity. Web tiers
|
||||
# report "api"; everything else uses the launch subcommand.
|
||||
case "$1" in
|
||||
prod|dev) DJANGO_APP_COMPONENT="api" ;;
|
||||
*) DJANGO_APP_COMPONENT="$1" ;;
|
||||
esac
|
||||
export DJANGO_APP_COMPONENT
|
||||
|
||||
case "$1" in
|
||||
dev)
|
||||
apply_migrations
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# Orphan Celery task recovery
|
||||
|
||||
When a worker is terminated mid-task (a deploy, an OOM kill, a node eviction), the
|
||||
task it was running can be left non-terminal forever: the `TaskResult` stays
|
||||
`STARTED` and nothing re-runs it. This page describes the mechanisms that detect and
|
||||
recover allowlisted idempotent orphans so pending-task alerts do not fire. Scan tasks
|
||||
are not auto-recovered (re-running a scan is not safe to do automatically); the
|
||||
watchdog covers the summary/aggregation and deletion tasks.
|
||||
|
||||
## How recovery works
|
||||
|
||||
1. **Durable delivery.** The broker is configured so a task message is acknowledged
|
||||
only after the task finishes (`task_acks_late`), one task is reserved at a time
|
||||
(`worker_prefetch_multiplier = 1`), and an abruptly-lost worker re-queues its task
|
||||
(`task_reject_on_worker_lost`). On `SIGTERM` the worker is given a soft-shutdown
|
||||
window (`worker_soft_shutdown_timeout`) to finish or re-queue in-flight work
|
||||
before it is force-killed. `scan-perform`, `scan-perform-scheduled` and
|
||||
`integration-jira` opt out of redelivery with `acks_late=False`, so a crash drops
|
||||
them rather than re-running and duplicating findings or Jira issues. Other
|
||||
non-recovered side-effect tasks keep `acks_late=True`, so the broker can still
|
||||
re-deliver them after a worker loss: the S3 upload rebuilds from worker-local files
|
||||
that did not survive the crash and so no-ops, but Security Hub re-reads findings from
|
||||
the DB and re-sends them to AWS.
|
||||
|
||||
2. **Periodic watchdog.** A Beat task, `reconcile-orphan-tasks`, runs every couple of
|
||||
minutes (a `django_celery_beat` periodic task created by migration). For each
|
||||
in-flight task result with an allowlisted idempotent task name, it pings the
|
||||
worker recorded on the task's `TaskResult`:
|
||||
- worker responds -> the task is still running, leave it alone;
|
||||
- worker is gone (and the task started before a short grace window) -> it is a
|
||||
real orphan: the stale task is revoked and marked terminal (clearing the
|
||||
pending/started alert), and the task is re-enqueued from its stored name and
|
||||
kwargs.
|
||||
|
||||
The re-run is safe because only tasks with proven idempotency are allowlisted: the
|
||||
summary/aggregation tasks clear and re-write their own rows, and deletions are
|
||||
idempotent. Scan tasks and external side effects are excluded: re-running a scan is
|
||||
not safe to do automatically, Jira sends would create duplicate issues, the S3
|
||||
upload rebuilds from worker-local files that do not survive a crash, and
|
||||
report/Security Hub recovery is out of scope.
|
||||
|
||||
3. **Recovery cap.** A per-task Valkey counter limits how often the same task is
|
||||
re-enqueued. After `--max-attempts` recoveries (default 3) the orphan is marked
|
||||
terminal instead of re-enqueued, so a task that repeatedly kills its worker cannot
|
||||
loop forever.
|
||||
|
||||
A Postgres advisory lock ensures that, even with multiple API/worker replicas, only
|
||||
one reconciliation runs at a time; the others no-op.
|
||||
|
||||
## On-demand command
|
||||
|
||||
The same logic is available as a management command, useful right after a deploy or
|
||||
for manual intervention:
|
||||
|
||||
```bash
|
||||
python manage.py reconcile_orphan_tasks # recover now
|
||||
python manage.py reconcile_orphan_tasks --dry-run # report orphans, change nothing
|
||||
python manage.py reconcile_orphan_tasks --grace-minutes 5 --max-attempts 3
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings have safe defaults; override via environment variables.
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER` | `1` | Tasks reserved per worker process. |
|
||||
| `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT` | `60` | Seconds the worker drains/re-queues on `SIGTERM` before force-kill. |
|
||||
| `DJANGO_CELERY_TASK_TIME_LIMIT` | `21600` (6h) | Hard limit for most tasks; connection checks are capped at 120s. |
|
||||
| `DJANGO_CELERY_TASK_SOFT_TIME_LIMIT` | hard - 600 | Soft limit; raises `SoftTimeLimitExceeded` for cleanup. |
|
||||
| `DJANGO_CELERY_LONG_TASK_TIME_LIMIT` | `172800` (48h) | Hard limit for scans and provider/tenant deletions, which can legitimately run for more than a day. |
|
||||
| `DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT` | long hard - 600 | Soft limit for the long-running tasks above. |
|
||||
| `DJANGO_TASK_RECOVERY_ENABLED` | `false` | Master switch for orphan-task recovery, disabled by default (opt-in); set to `true` to enable. When off, no orphan is detected, marked terminal, or re-enqueued (attack-paths stale cleanup still runs). |
|
||||
| `DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED` | `true` | Auto re-enqueue orphaned scan summary/aggregation tasks. |
|
||||
| `DJANGO_TASK_RECOVERY_DELETIONS_ENABLED` | `true` | Auto re-enqueue orphaned provider/tenant deletion tasks. |
|
||||
|
||||
Recovery is opt-in: with the master flag off (the default) the sweep does nothing.
|
||||
Once enabled, the per-group flags default to on, so every group recovers unless you
|
||||
turn one off; a task whose group flag is off is marked terminal instead of
|
||||
re-enqueued.
|
||||
|
||||
Turning recovery off only disables this watchdog sweep; it does not change Celery's
|
||||
broker-level redelivery (`task_acks_late`/`task_reject_on_worker_lost`), which still
|
||||
re-delivers tasks that keep `acks_late=True` on worker loss, independently of this flag.
|
||||
|
||||
`task_acks_late` and `task_reject_on_worker_lost` are enabled in `config/celery.py`.
|
||||
|
||||
## Deployment requirement
|
||||
|
||||
Two conditions must both hold for the soft shutdown to actually drain work:
|
||||
|
||||
1. **The worker must receive `SIGTERM`.** The container entrypoint `exec`s the
|
||||
Celery process so it runs as PID 1; otherwise `SIGTERM` from `docker stop`/ECS
|
||||
hits the entrypoint shell, never reaches Celery, and the worker is hard-killed
|
||||
(SIGKILL) at the grace deadline without draining. Custom entrypoints must
|
||||
preserve the `exec`.
|
||||
2. **The orchestrator must give the worker enough time** before force-killing it.
|
||||
Set the stop grace period to exceed `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT`
|
||||
plus a margin:
|
||||
- **docker-compose:** `stop_grace_period` on the worker services (set to `120s`).
|
||||
- **AWS ECS:** the worker container `stopTimeout` (configured in the deployment
|
||||
repository).
|
||||
|
||||
If either condition is missing, long tasks are still recovered by the watchdog,
|
||||
but they are cut mid-run on every deploy instead of draining.
|
||||
+14
-4
@@ -68,7 +68,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.29.0"
|
||||
version = "1.31.0"
|
||||
|
||||
[tool.uv]
|
||||
# Transitive pins matching master to avoid silent drift; bump deliberately.
|
||||
@@ -226,7 +226,7 @@ constraint-dependencies = [
|
||||
"drf-simple-apikey==2.2.1",
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"dulwich==0.23.0",
|
||||
"dulwich==1.2.5",
|
||||
"duo-client==5.5.0",
|
||||
"durationpy==0.10",
|
||||
"email-validator==2.2.0",
|
||||
@@ -354,7 +354,7 @@ constraint-dependencies = [
|
||||
"pydantic-core==2.41.5",
|
||||
"pygithub==2.8.0",
|
||||
"pygments==2.20.0",
|
||||
"pyjwt==2.12.1",
|
||||
"pyjwt==2.13.0",
|
||||
"pylint==3.2.5",
|
||||
"pymsalruntime==0.18.1",
|
||||
"pynacl==1.6.2",
|
||||
@@ -443,7 +443,17 @@ constraint-dependencies = [
|
||||
# The microsoft-kiota-http security bump to 1.9.9 (GHSA-7j59-v9qr-6fq9) requires
|
||||
# microsoft-kiota-abstractions>=1.9.9, which a constraint cannot satisfy against the
|
||||
# SDK's hard pin; override it to the patched, kiota-aligned version.
|
||||
#
|
||||
# prowler@master hard-pins dulwich==0.23.0 and pyjwt==2.12.1 in [project.dependencies].
|
||||
# dulwich 1.2.5 patches GHSA-897w-fcg9-f6xj (arbitrary file write) and pyjwt 2.13.0
|
||||
# patches PYSEC-2026-179 (HMAC/JWK key-confusion); a constraint cannot satisfy these
|
||||
# against the SDK's hard pins, so override them to the patched versions until the SDK
|
||||
# bump propagates to the pinned master rev. pyjwt keeps the [crypto] extra because an
|
||||
# override replaces the whole requirement; bare pyjwt would drop it from the consumers
|
||||
# that request pyjwt[crypto] and leave cryptography (needed for RS256) only transitive.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2",
|
||||
"microsoft-kiota-abstractions==1.9.9"
|
||||
"microsoft-kiota-abstractions==1.9.9",
|
||||
"dulwich==1.2.5",
|
||||
"pyjwt[crypto]==2.13.0"
|
||||
]
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
@@ -30,7 +32,6 @@ class ApiConfig(AppConfig):
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.attack_paths import database as graph_database
|
||||
|
||||
# Generate required cryptographic keys if not present, but only if:
|
||||
# `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
|
||||
@@ -41,37 +42,8 @@ class ApiConfig(AppConfig):
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
# Commands that don't need Neo4j
|
||||
SKIP_NEO4J_DJANGO_COMMANDS = [
|
||||
"makemigrations",
|
||||
"migrate",
|
||||
"pgpartition",
|
||||
"check",
|
||||
"help",
|
||||
"showmigrations",
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook)
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
(
|
||||
"manage.py" in sys.argv[0]
|
||||
and sys.argv[1] in SKIP_NEO4J_DJANGO_COMMANDS
|
||||
)
|
||||
or "celery" in sys.argv[0]
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)"
|
||||
)
|
||||
|
||||
else:
|
||||
graph_database.init_driver()
|
||||
|
||||
# Neo4j driver is initialized at API startup (see api.attack_paths.database)
|
||||
# It remains lazy for Celery workers and selected Django commands
|
||||
# Neo4j driver is created lazily on first use (see api.attack_paths.database).
|
||||
# App init never contacts Neo4j, so a Neo4j outage cannot block API startup.
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
|
||||
@@ -1,22 +1,24 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Iterator
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
@@ -28,6 +30,9 @@ READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
|
||||
# Shorter than CONN_ACQUISITION_TIMEOUT — the driver requires acquisition to be
|
||||
# the longer of the two (it may include opening a new connection).
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
@@ -58,15 +63,24 @@ def init_driver() -> neo4j.Driver:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
_driver.verify_connectivity()
|
||||
# Publish the singleton only after connectivity is verified so a
|
||||
# failed probe does not leave an unverified driver behind. Close the
|
||||
# driver on failure so a repeatedly-probed outage cannot leak pools.
|
||||
try:
|
||||
driver.verify_connectivity()
|
||||
except Exception:
|
||||
driver.close()
|
||||
raise
|
||||
_driver = driver
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
|
||||
@@ -94,25 +96,22 @@ PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""List compliance frameworks the API can load for `provider_type`.
|
||||
"""List compliance framework identifiers available for `provider_type`.
|
||||
|
||||
The list is sourced from `Compliance.get_bulk` so that the names
|
||||
returned here are guaranteed to be loadable by the bulk loader. This
|
||||
prevents downstream key mismatches (e.g. CSV report generation iterating
|
||||
framework names and looking them up in the bulk dict).
|
||||
Includes both per-provider frameworks and universal top-level frameworks
|
||||
(e.g. ``dora``, ``csa_ccm_4.0``).
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type for which to retrieve
|
||||
available compliance frameworks (e.g., "aws", "azure", "gcp", "m365").
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type
|
||||
(e.g., "aws", "azure", "gcp", "m365").
|
||||
|
||||
Returns:
|
||||
list[str]: A list of framework identifiers (e.g., "cis_1.4_aws", "mitre_attack_azure") available
|
||||
for the given provider.
|
||||
list[str]: Framework identifiers (e.g., "cis_1.4_aws", "dora").
|
||||
"""
|
||||
global AVAILABLE_COMPLIANCE_FRAMEWORKS
|
||||
if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS:
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = list(
|
||||
Compliance.get_bulk(provider_type).keys()
|
||||
get_bulk_compliance_frameworks_universal(provider_type).keys()
|
||||
)
|
||||
|
||||
return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type]
|
||||
@@ -139,18 +138,14 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
|
||||
"""
|
||||
Retrieve the Prowler compliance data for a specified provider type.
|
||||
|
||||
This function fetches the compliance frameworks and their associated
|
||||
requirements for the given cloud provider.
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The provider type
|
||||
(e.g., 'aws', 'azure') for which to retrieve compliance data.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary mapping compliance framework names to their respective
|
||||
Compliance objects for the specified provider.
|
||||
dict: Mapping of framework name to `ComplianceFramework` for the provider.
|
||||
"""
|
||||
return Compliance.get_bulk(provider_type)
|
||||
return get_bulk_compliance_frameworks_universal(provider_type)
|
||||
|
||||
|
||||
def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
|
||||
@@ -209,8 +204,8 @@ def load_prowler_checks(
|
||||
for compliance_name, compliance_data in prowler_compliance.get(
|
||||
provider_type, {}
|
||||
).items():
|
||||
for requirement in compliance_data.Requirements:
|
||||
for check in requirement.Checks:
|
||||
for requirement in compliance_data.requirements:
|
||||
for check in requirement.checks.get(provider_type, []):
|
||||
try:
|
||||
checks[provider_type][check].add(compliance_name)
|
||||
except KeyError:
|
||||
@@ -290,24 +285,40 @@ def generate_compliance_overview_template(
|
||||
requirements_status = {"passed": 0, "failed": 0, "manual": 0}
|
||||
total_requirements = 0
|
||||
|
||||
for requirement in compliance_data.Requirements:
|
||||
for requirement in compliance_data.requirements:
|
||||
total_requirements += 1
|
||||
total_checks = len(requirement.Checks)
|
||||
checks_dict = {check: None for check in requirement.Checks}
|
||||
provider_check_list = list(requirement.checks.get(provider_type, []))
|
||||
total_checks = len(provider_check_list)
|
||||
checks_dict = {check: None for check in provider_check_list}
|
||||
|
||||
req_status_val = "MANUAL" if total_checks == 0 else "PASS"
|
||||
|
||||
# MITRE attrs are wrapped under `_raw_attributes` by the
|
||||
# universal adapter — unwrap so consumers see the flat list.
|
||||
requirement_attributes = requirement.attributes
|
||||
if (
|
||||
isinstance(requirement_attributes, dict)
|
||||
and "_raw_attributes" in requirement_attributes
|
||||
):
|
||||
attributes_payload = list(requirement_attributes["_raw_attributes"])
|
||||
elif isinstance(requirement_attributes, dict):
|
||||
attributes_payload = (
|
||||
[dict(requirement_attributes)] if requirement_attributes else []
|
||||
)
|
||||
else:
|
||||
attributes_payload = [
|
||||
dict(attribute) for attribute in requirement_attributes
|
||||
]
|
||||
|
||||
# Build requirement dictionary
|
||||
requirement_dict = {
|
||||
"name": requirement.Name or requirement.Id,
|
||||
"description": requirement.Description,
|
||||
"tactics": getattr(requirement, "Tactics", []),
|
||||
"subtechniques": getattr(requirement, "SubTechniques", []),
|
||||
"platforms": getattr(requirement, "Platforms", []),
|
||||
"technique_url": getattr(requirement, "TechniqueURL", ""),
|
||||
"attributes": [
|
||||
dict(attribute) for attribute in requirement.Attributes
|
||||
],
|
||||
"name": requirement.name or requirement.id,
|
||||
"description": requirement.description,
|
||||
"tactics": requirement.tactics or [],
|
||||
"subtechniques": requirement.sub_techniques or [],
|
||||
"platforms": requirement.platforms or [],
|
||||
"technique_url": requirement.technique_url or "",
|
||||
"attributes": attributes_payload,
|
||||
"checks": checks_dict,
|
||||
"checks_status": {
|
||||
"pass": 0,
|
||||
@@ -325,15 +336,15 @@ def generate_compliance_overview_template(
|
||||
requirements_status["passed"] += 1
|
||||
|
||||
# Add requirement to compliance requirements
|
||||
compliance_requirements[requirement.Id] = requirement_dict
|
||||
compliance_requirements[requirement.id] = requirement_dict
|
||||
|
||||
# Build compliance dictionary
|
||||
compliance_dict = {
|
||||
"framework": compliance_data.Framework,
|
||||
"name": compliance_data.Name,
|
||||
"version": compliance_data.Version,
|
||||
"framework": compliance_data.framework,
|
||||
"name": compliance_data.name,
|
||||
"version": compliance_data.version,
|
||||
"provider": provider_type,
|
||||
"description": compliance_data.Description,
|
||||
"description": compliance_data.description,
|
||||
"requirements": compliance_requirements,
|
||||
"requirements_status": requirements_status,
|
||||
"total_requirements": total_requirements,
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from tasks.jobs.orphan_recovery import reconcile_orphans
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Recover orphaned allowlisted Celery tasks whose worker is gone and mark "
|
||||
"other stale task results terminal. Single-flight via a Postgres advisory lock."
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--grace-minutes",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Skip tasks started within this window (worker may still register).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-attempts",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Give up re-running a task after this many recovery attempts; it is then left terminal instead of re-enqueued.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect and report orphans without revoking or re-enqueuing.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
result = reconcile_orphans(
|
||||
grace_minutes=options["grace_minutes"],
|
||||
max_attempts=options["max_attempts"],
|
||||
dry_run=options["dry_run"],
|
||||
)
|
||||
|
||||
if not result.get("acquired"):
|
||||
self.stdout.write("Reconcile skipped: another run holds the lock.")
|
||||
return
|
||||
|
||||
if result.get("enabled") is False:
|
||||
message = (
|
||||
"Task recovery is disabled (DJANGO_TASK_RECOVERY_ENABLED is off); "
|
||||
"no orphans were recovered."
|
||||
)
|
||||
if result.get("attack_paths") is not None:
|
||||
message += " Attack-paths stale cleanup still ran."
|
||||
self.stdout.write(message)
|
||||
return
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
"Orphan reconcile complete: "
|
||||
f"recovered={len(result.get('recovered', []))} "
|
||||
f"failed={len(result.get('failed', []))} "
|
||||
f"skipped(in-flight)={len(result.get('skipped', []))}"
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
TASK_NAME = "reconcile-orphan-tasks"
|
||||
INTERVAL_MINUTES = 2
|
||||
|
||||
|
||||
def create_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
schedule, _ = IntervalSchedule.objects.get_or_create(
|
||||
every=INTERVAL_MINUTES,
|
||||
period="minutes",
|
||||
)
|
||||
|
||||
PeriodicTask.objects.update_or_create(
|
||||
name=TASK_NAME,
|
||||
defaults={
|
||||
"task": TASK_NAME,
|
||||
"interval": schedule,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def delete_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
PeriodicTask.objects.filter(name=TASK_NAME).delete()
|
||||
|
||||
# Clean up the schedule if no other task references it
|
||||
IntervalSchedule.objects.filter(
|
||||
every=INTERVAL_MINUTES,
|
||||
period="minutes",
|
||||
periodictask__isnull=True,
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0093_okta_provider"),
|
||||
("django_celery_beat", "0019_alter_periodictasks_options"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_periodic_task, delete_periodic_task),
|
||||
]
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.29.0
|
||||
version: 1.31.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
@@ -13137,8 +13137,59 @@ paths:
|
||||
responses:
|
||||
'200':
|
||||
description: CSV file containing the compliance report
|
||||
'202':
|
||||
description: The task is in progress
|
||||
'403':
|
||||
description: There is a problem with credentials
|
||||
'404':
|
||||
description: Compliance report not found
|
||||
description: Compliance report not found, or the scan has no reports yet
|
||||
/api/v1/scans/{id}/compliance/{name}/ocsf:
|
||||
get:
|
||||
operationId: scans_compliance_ocsf_retrieve
|
||||
description: Download a specific compliance report as an OCSF JSON file. Only
|
||||
universal frameworks that declare an output configuration produce this artifact
|
||||
(currently 'dora' and 'csa_ccm_4.0'); any other framework returns 404.
|
||||
summary: Retrieve compliance report as OCSF JSON
|
||||
parameters:
|
||||
- in: query
|
||||
name: fields[scan-reports]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- name
|
||||
description: endpoint return only specific fields in the response on a per-type
|
||||
basis by including a fields[TYPE] query parameter.
|
||||
explode: false
|
||||
- in: path
|
||||
name: id
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
description: A UUID string identifying this scan.
|
||||
required: true
|
||||
- in: path
|
||||
name: name
|
||||
schema:
|
||||
type: string
|
||||
description: The compliance report name, like 'dora'
|
||||
required: true
|
||||
tags:
|
||||
- Scan
|
||||
security:
|
||||
- JWT or API Key: []
|
||||
responses:
|
||||
'200':
|
||||
description: OCSF JSON file containing the compliance report
|
||||
'202':
|
||||
description: The task is in progress
|
||||
'403':
|
||||
description: There is a problem with credentials
|
||||
'404':
|
||||
description: Compliance report not found, the framework does not provide
|
||||
an OCSF export, or the scan has no reports yet
|
||||
/api/v1/scans/{id}/csa:
|
||||
get:
|
||||
operationId: scans_csa_retrieve
|
||||
|
||||
@@ -182,23 +182,19 @@ def _make_app():
|
||||
return ApiConfig("api", api)
|
||||
|
||||
|
||||
def test_ready_initializes_driver_for_api_process(monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
["gunicorn"],
|
||||
["celery", "-A", "api"],
|
||||
["manage.py", "migrate"],
|
||||
],
|
||||
ids=["api", "celery", "manage_py"],
|
||||
)
|
||||
def test_ready_never_eagerly_initializes_neo4j_driver(monkeypatch, argv):
|
||||
"""ready() must never contact Neo4j; the driver is created lazily on first use."""
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_called_once()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["celery", "-A", "api"])
|
||||
_set_argv(monkeypatch, argv)
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
@@ -208,31 +204,3 @@ def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_manage_py_skip_command(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["manage.py", "migrate"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_when_testing(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, True)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
"""
|
||||
Tests for Neo4j database lazy initialization.
|
||||
|
||||
The Neo4j driver connects on first use by default. API processes may
|
||||
eagerly initialize the driver during app startup, while Celery workers
|
||||
remain lazy. These tests validate the database module behavior itself.
|
||||
The Neo4j driver is created on first use for every process type; app startup
|
||||
never contacts Neo4j. These tests validate the database module behavior itself.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
import pytest
|
||||
|
||||
import api.attack_paths.database as db_module
|
||||
@@ -59,6 +60,32 @@ class TestLazyInitialization:
|
||||
assert result is mock_driver
|
||||
assert db_module._driver is mock_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_leaves_driver_none_when_verify_fails(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""A failed verify_connectivity() must not publish or leak the driver."""
|
||||
mock_driver = MagicMock()
|
||||
mock_driver.verify_connectivity.side_effect = (
|
||||
neo4j.exceptions.ServiceUnavailable("down")
|
||||
)
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
with pytest.raises(neo4j.exceptions.ServiceUnavailable):
|
||||
db_module.init_driver()
|
||||
|
||||
assert db_module._driver is None
|
||||
mock_driver.close.assert_called_once()
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_returns_cached_driver_on_subsequent_calls(
|
||||
@@ -116,21 +143,23 @@ class TestConnectionAcquisitionTimeout:
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
original_driver = db_module._driver
|
||||
original_timeout = db_module.CONN_ACQUISITION_TIMEOUT
|
||||
original_acq_timeout = db_module.CONN_ACQUISITION_TIMEOUT
|
||||
original_conn_timeout = db_module.CONNECTION_TIMEOUT
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = original_timeout
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = original_acq_timeout
|
||||
db_module.CONNECTION_TIMEOUT = original_conn_timeout
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_driver_receives_configured_timeout(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""init_driver() should pass CONN_ACQUISITION_TIMEOUT to the neo4j driver."""
|
||||
"""init_driver() should pass the configured timeouts to the neo4j driver."""
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
@@ -141,11 +170,13 @@ class TestConnectionAcquisitionTimeout:
|
||||
}
|
||||
}
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = 42
|
||||
db_module.CONNECTION_TIMEOUT = 7
|
||||
|
||||
db_module.init_driver()
|
||||
|
||||
_, kwargs = mock_driver_factory.call_args
|
||||
assert kwargs["connection_acquisition_timeout"] == 42
|
||||
assert kwargs["connection_timeout"] == 7
|
||||
|
||||
|
||||
class TestAtexitRegistration:
|
||||
|
||||
@@ -12,7 +12,9 @@ from api.compliance import (
|
||||
load_prowler_checks,
|
||||
)
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
|
||||
|
||||
class TestCompliance:
|
||||
@@ -28,16 +30,16 @@ class TestCompliance:
|
||||
assert set(checks) == {"check1", "check2", "check3"}
|
||||
mock_check_metadata.get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.compliance.Compliance")
|
||||
def test_get_prowler_provider_compliance(self, mock_compliance):
|
||||
@patch("api.compliance.get_bulk_compliance_frameworks_universal")
|
||||
def test_get_prowler_provider_compliance(self, mock_get_bulk):
|
||||
provider_type = Provider.ProviderChoices.AWS
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
mock_get_bulk.return_value = {
|
||||
"compliance1": MagicMock(),
|
||||
"compliance2": MagicMock(),
|
||||
}
|
||||
compliance_data = get_prowler_provider_compliance(provider_type)
|
||||
assert compliance_data == mock_compliance.get_bulk.return_value
|
||||
mock_compliance.get_bulk.assert_called_once_with(provider_type)
|
||||
assert compliance_data == mock_get_bulk.return_value
|
||||
mock_get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.compliance.get_prowler_provider_checks")
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
@@ -51,9 +53,9 @@ class TestCompliance:
|
||||
prowler_compliance = {
|
||||
"aws": {
|
||||
"compliance1": MagicMock(
|
||||
Requirements=[
|
||||
requirements=[
|
||||
MagicMock(
|
||||
Checks=["check1", "check2"],
|
||||
checks={"aws": ["check1", "check2"]},
|
||||
),
|
||||
],
|
||||
),
|
||||
@@ -167,35 +169,38 @@ class TestCompliance:
|
||||
def test_generate_compliance_overview_template(self, mock_provider_choices):
|
||||
mock_provider_choices.values = ["aws"]
|
||||
|
||||
# ``name`` is a reserved MagicMock kwarg (it labels the mock for repr,
|
||||
# it does NOT set a ``.name`` attribute), so it must be assigned
|
||||
# explicitly after construction.
|
||||
requirement1 = MagicMock(
|
||||
Id="requirement1",
|
||||
Name="Requirement 1",
|
||||
Description="Description of requirement 1",
|
||||
Attributes=[],
|
||||
Checks=["check1", "check2"],
|
||||
Tactics=["tactic1"],
|
||||
SubTechniques=["subtechnique1"],
|
||||
Platforms=["platform1"],
|
||||
TechniqueURL="https://example.com",
|
||||
id="requirement1",
|
||||
description="Description of requirement 1",
|
||||
attributes=[],
|
||||
checks={"aws": ["check1", "check2"]},
|
||||
tactics=["tactic1"],
|
||||
sub_techniques=["subtechnique1"],
|
||||
platforms=["platform1"],
|
||||
technique_url="https://example.com",
|
||||
)
|
||||
requirement1.name = "Requirement 1"
|
||||
requirement2 = MagicMock(
|
||||
Id="requirement2",
|
||||
Name="Requirement 2",
|
||||
Description="Description of requirement 2",
|
||||
Attributes=[],
|
||||
Checks=[],
|
||||
Tactics=[],
|
||||
SubTechniques=[],
|
||||
Platforms=[],
|
||||
TechniqueURL="",
|
||||
id="requirement2",
|
||||
description="Description of requirement 2",
|
||||
attributes=[],
|
||||
checks={"aws": []},
|
||||
tactics=[],
|
||||
sub_techniques=[],
|
||||
platforms=[],
|
||||
technique_url="",
|
||||
)
|
||||
requirement2.name = "Requirement 2"
|
||||
compliance1 = MagicMock(
|
||||
Requirements=[requirement1, requirement2],
|
||||
Framework="Framework 1",
|
||||
Version="1.0",
|
||||
Description="Description of compliance1",
|
||||
Name="Compliance 1",
|
||||
requirements=[requirement1, requirement2],
|
||||
framework="Framework 1",
|
||||
version="1.0",
|
||||
description="Description of compliance1",
|
||||
)
|
||||
compliance1.name = "Compliance 1"
|
||||
prowler_compliance = {"aws": {"compliance1": compliance1}}
|
||||
|
||||
template = generate_compliance_overview_template(prowler_compliance)
|
||||
@@ -271,24 +276,28 @@ def reset_compliance_cache():
|
||||
|
||||
class TestGetComplianceFrameworks:
|
||||
def test_returns_keys_from_compliance_get_bulk(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
with patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk:
|
||||
mock_get_bulk.return_value = {
|
||||
"cis_1.4_aws": MagicMock(),
|
||||
"mitre_attack_aws": MagicMock(),
|
||||
}
|
||||
result = get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
assert sorted(result) == ["cis_1.4_aws", "mitre_attack_aws"]
|
||||
mock_compliance.get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
mock_get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
|
||||
def test_caches_result_per_provider(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
with patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk:
|
||||
mock_get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
# Cached after first call.
|
||||
assert mock_compliance.get_bulk.call_count == 1
|
||||
assert mock_get_bulk.call_count == 1
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_type",
|
||||
@@ -296,17 +305,19 @@ class TestGetComplianceFrameworks:
|
||||
)
|
||||
def test_listing_is_subset_of_bulk(self, reset_compliance_cache, provider_type):
|
||||
"""Regression for CLOUD-API-40S: every name returned by
|
||||
``get_compliance_frameworks`` must be loadable via ``Compliance.get_bulk``.
|
||||
``get_compliance_frameworks`` must be loadable via
|
||||
``get_bulk_compliance_frameworks_universal``.
|
||||
|
||||
A divergence here is what produced ``KeyError: 'csa_ccm_4.0'`` in
|
||||
``generate_outputs_task`` after universal/multi-provider compliance
|
||||
JSONs were introduced at the top-level ``prowler/compliance/`` path.
|
||||
"""
|
||||
bulk_keys = set(Compliance.get_bulk(provider_type).keys())
|
||||
bulk_keys = set(get_bulk_compliance_frameworks_universal(provider_type).keys())
|
||||
listed = set(get_compliance_frameworks(provider_type))
|
||||
|
||||
missing = listed - bulk_keys
|
||||
assert not missing, (
|
||||
f"get_compliance_frameworks({provider_type!r}) returned names not "
|
||||
f"loadable by Compliance.get_bulk: {sorted(missing)}"
|
||||
f"loadable by get_bulk_compliance_frameworks_universal: "
|
||||
f"{sorted(missing)}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from config.django.base import label_postgres_connections
|
||||
|
||||
|
||||
class TestLabelPostgresConnections:
|
||||
def test_labels_postgres_and_skips_neo4j(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "scan")
|
||||
databases = {
|
||||
"default": {"ENGINE": "psqlextra.backend"},
|
||||
"neo4j": {"HOST": "neo4j", "PORT": "7687"},
|
||||
}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["default"]["OPTIONS"]["application_name"] == "scan:default"
|
||||
assert "OPTIONS" not in databases["neo4j"]
|
||||
|
||||
def test_labels_plain_postgresql_backend(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "api")
|
||||
databases = {"saas": {"ENGINE": "django.db.backends.postgresql"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["saas"]["OPTIONS"]["application_name"] == "api:saas"
|
||||
|
||||
def test_defaults_component_to_api_when_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("DJANGO_APP_COMPONENT", raising=False)
|
||||
databases = {"default": {"ENGINE": "psqlextra.backend"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["default"]["OPTIONS"]["application_name"] == "api:default"
|
||||
|
||||
def test_preserves_existing_options(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "worker")
|
||||
databases = {
|
||||
"replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"OPTIONS": {"sslmode": "require"},
|
||||
}
|
||||
}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["replica"]["OPTIONS"] == {
|
||||
"sslmode": "require",
|
||||
"application_name": "worker:replica",
|
||||
}
|
||||
|
||||
def test_truncates_application_name_to_63_bytes(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "c" * 80)
|
||||
databases = {"default": {"ENGINE": "psqlextra.backend"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert len(databases["default"]["OPTIONS"]["application_name"]) == 63
|
||||
@@ -24,9 +24,11 @@ from conftest import (
|
||||
today_after_n_days,
|
||||
)
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
from django.db.models import Count
|
||||
from django.http import JsonResponse
|
||||
from django.test import RequestFactory
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from django.urls import reverse
|
||||
from django_celery_results.models import TaskResult
|
||||
from rest_framework import status
|
||||
@@ -64,6 +66,7 @@ from api.models import (
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
ResourceTag,
|
||||
Role,
|
||||
RoleProviderGroupRelationship,
|
||||
SAMLConfiguration,
|
||||
@@ -3856,16 +3859,20 @@ class TestScanViewSet:
|
||||
scan.output_location = "dummy"
|
||||
scan.save()
|
||||
|
||||
dummy_task = Task.objects.create(tenant_id=scan.tenant_id)
|
||||
dummy_task.id = "dummy-task-id"
|
||||
dummy_task_data = {"id": dummy_task.id, "state": StateChoices.EXECUTING}
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
)
|
||||
task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
dummy_task_data = {"id": str(task.id), "state": StateChoices.EXECUTING}
|
||||
|
||||
with (
|
||||
patch("api.v1.views.Task.objects.get", return_value=dummy_task),
|
||||
patch(
|
||||
"api.v1.views.TaskSerializer",
|
||||
return_value=type("DummySerializer", (), {"data": dummy_task_data}),
|
||||
),
|
||||
with patch(
|
||||
"api.v1.views.TaskSerializer",
|
||||
return_value=type("DummySerializer", (), {"data": dummy_task_data}),
|
||||
):
|
||||
url = reverse("scan-report", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
@@ -4186,6 +4193,88 @@ class TestScanViewSet:
|
||||
assert resp.status_code == status.HTTP_302_FOUND
|
||||
assert resp["Location"] == presigned_url
|
||||
|
||||
def test_compliance_s3_returns_latest_match(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
):
|
||||
"""When several files match, the most recently modified one is served."""
|
||||
scan = scans_fixture[0]
|
||||
bucket = "bucket"
|
||||
scan.output_location = f"s3://{bucket}/path/scan.zip"
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.save()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
old_key = "path/compliance/prowler-output-aws-20240101000000_cis_1.4_aws.csv"
|
||||
latest_key = "path/compliance/prowler-output-aws-20240202000000_cis_1.4_aws.csv"
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
return {
|
||||
"Contents": [
|
||||
{
|
||||
"Key": old_key,
|
||||
"LastModified": datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
},
|
||||
{
|
||||
"Key": latest_key,
|
||||
"LastModified": datetime(2024, 2, 2, tzinfo=timezone.utc),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
assert Params["Key"] == latest_key
|
||||
return "https://test-bucket.s3.amazonaws.com/latest"
|
||||
|
||||
monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client())
|
||||
|
||||
url = reverse("scan-compliance", kwargs={"pk": scan.id, "name": "cis_1.4_aws"})
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_302_FOUND
|
||||
assert resp["Location"].endswith("/latest")
|
||||
|
||||
def test_compliance_local_returns_latest_match(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
):
|
||||
"""The local branch serves the most recently modified matching file."""
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
comp_dir = Path(tmp) / "reports" / "compliance"
|
||||
comp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
old_file = comp_dir / "prowler-output-aws-20240101000000_cis_1.4_aws.csv"
|
||||
old_file.write_bytes(b"old")
|
||||
latest_file = comp_dir / "prowler-output-aws-20240202000000_cis_1.4_aws.csv"
|
||||
latest_file.write_bytes(b"latest")
|
||||
# Make `latest_file` newer regardless of creation order.
|
||||
os.utime(old_file, (1_700_000_000, 1_700_000_000))
|
||||
os.utime(latest_file, (1_700_000_100, 1_700_000_100))
|
||||
|
||||
scan.output_location = str(Path(tmp) / "reports" / "scan.zip")
|
||||
scan.save()
|
||||
|
||||
monkeypatch.setattr(
|
||||
glob,
|
||||
"glob",
|
||||
lambda p: [str(old_file), str(latest_file)],
|
||||
)
|
||||
|
||||
url = reverse(
|
||||
"scan-compliance", kwargs={"pk": scan.id, "name": "cis_1.4_aws"}
|
||||
)
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
assert resp.content == b"latest"
|
||||
assert resp["Content-Disposition"].endswith(
|
||||
f'filename="{latest_file.name}"'
|
||||
)
|
||||
|
||||
def test_compliance_s3_not_found(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
):
|
||||
@@ -4294,18 +4383,24 @@ class TestScanViewSet:
|
||||
assert cd.startswith('attachment; filename="')
|
||||
assert cd.endswith(f'filename="{fname.name}"')
|
||||
|
||||
@patch("api.v1.views.Task.objects.get")
|
||||
@patch("api.v1.views.TaskSerializer")
|
||||
def test__get_task_status_returns_none_if_task_not_executing(
|
||||
self, mock_task_serializer, mock_task_get, authenticated_client, scans_fixture
|
||||
self, mock_task_serializer, authenticated_client, scans_fixture
|
||||
):
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.output_location = "dummy"
|
||||
scan.save()
|
||||
|
||||
task = Task.objects.create(tenant_id=scan.tenant_id)
|
||||
mock_task_get.return_value = task
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
)
|
||||
task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
mock_task_serializer.return_value.data = {
|
||||
"id": str(task.id),
|
||||
"state": StateChoices.COMPLETED,
|
||||
@@ -4326,6 +4421,7 @@ class TestScanViewSet:
|
||||
scan.save()
|
||||
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
)
|
||||
@@ -4346,6 +4442,51 @@ class TestScanViewSet:
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert response.data["id"] == str(task.id)
|
||||
|
||||
@patch("api.v1.views.TaskSerializer")
|
||||
def test__get_task_status_returns_latest_task(
|
||||
self, mock_task_serializer, authenticated_client, scans_fixture
|
||||
):
|
||||
"""With several scan-report tasks for the scan, the most recent is used."""
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.output_location = "dummy"
|
||||
scan.save()
|
||||
|
||||
old_task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
),
|
||||
)
|
||||
new_task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
),
|
||||
)
|
||||
# `inserted_at` is `auto_now_add`, and within the test transaction the DB
|
||||
# `now()` is constant, so force distinct timestamps to make order_by stable.
|
||||
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
Task.objects.filter(pk=old_task.pk).update(inserted_at=base)
|
||||
Task.objects.filter(pk=new_task.pk).update(
|
||||
inserted_at=base + timedelta(hours=1)
|
||||
)
|
||||
|
||||
mock_task_serializer.side_effect = lambda instance, *a, **k: SimpleNamespace(
|
||||
data={"id": str(instance.id), "state": StateChoices.EXECUTING}
|
||||
)
|
||||
|
||||
url = reverse("scan-report", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert str(new_task.id) in response["Content-Location"]
|
||||
assert str(old_task.id) not in response["Content-Location"]
|
||||
|
||||
@patch("api.v1.views.get_s3_client")
|
||||
@patch("api.v1.views.sentry_sdk.capture_exception")
|
||||
def test_compliance_list_objects_client_error(
|
||||
@@ -6916,6 +7057,80 @@ class TestFindingViewSet:
|
||||
== findings_fixture[0].status
|
||||
)
|
||||
|
||||
def test_findings_list_resource_tags_no_n_plus_one(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
"""Listing findings must load every resource's tags in a constant
|
||||
number of queries, no matter how many findings/resources are returned.
|
||||
|
||||
This guards ``FindingViewSet._optimize_tags_loading`` against
|
||||
regressions that would reintroduce one extra query per resource (the
|
||||
N+1 the prefetch was added to remove).
|
||||
"""
|
||||
scan = findings_fixture[0].scan
|
||||
tenant_id = findings_fixture[0].tenant_id
|
||||
provider = scan.provider
|
||||
|
||||
def _create_finding_with_tagged_resource(index):
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider,
|
||||
uid=f"arn:aws:ec2:us-east-1:123456789012:instance/n-plus-one-{index}",
|
||||
name=f"N+1 Instance {index}",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="prowler-test",
|
||||
)
|
||||
resource.upsert_or_delete_tags(
|
||||
[
|
||||
ResourceTag.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
key=f"key-{index}",
|
||||
value=f"value-{index}",
|
||||
)
|
||||
]
|
||||
)
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
uid=f"n_plus_one_finding_{index}",
|
||||
scan=scan,
|
||||
status=Status.FAIL,
|
||||
status_extended="n+1 status",
|
||||
impact=Severity.medium,
|
||||
severity=Severity.medium,
|
||||
check_id="test_check_id",
|
||||
check_metadata={"CheckId": "test_check_id", "servicename": "ec2"},
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
return finding
|
||||
|
||||
params = {"filter[inserted_at]": TODAY, "include": "resources"}
|
||||
|
||||
# Baseline: the two findings provided by the fixture.
|
||||
with CaptureQueriesContext(connection) as baseline:
|
||||
response = authenticated_client.get(reverse("finding-list"), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
# Add more findings, each with its own resource carrying tags.
|
||||
extra_findings = 5
|
||||
for index in range(extra_findings):
|
||||
_create_finding_with_tagged_resource(index)
|
||||
|
||||
with CaptureQueriesContext(connection) as scaled:
|
||||
response = authenticated_client.get(reverse("finding-list"), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == len(findings_fixture) + extra_findings
|
||||
|
||||
# The query count must not grow with the number of findings/resources.
|
||||
assert len(scaled.captured_queries) == len(baseline.captured_queries), (
|
||||
"Resource tags are not being prefetched: "
|
||||
f"{len(baseline.captured_queries)} queries for {len(findings_fixture)} "
|
||||
f"findings vs {len(scaled.captured_queries)} for "
|
||||
f"{len(findings_fixture) + extra_findings}. Likely an N+1 regression "
|
||||
"in FindingViewSet._optimize_tags_loading."
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"include_values, expected_resources",
|
||||
[
|
||||
@@ -9345,6 +9560,16 @@ class TestComplianceOverviewViewSet:
|
||||
assert "platforms" in attributes["attributes"]["technique_details"]
|
||||
assert "technique_url" in attributes["attributes"]["technique_details"]
|
||||
|
||||
# Guard against the `_raw_attributes` wrapper leaking through —
|
||||
# the UI reads metadata[i].Category / .AWSService directly.
|
||||
metadata = attributes["attributes"]["metadata"]
|
||||
assert isinstance(metadata, list) and len(metadata) > 0
|
||||
first_attr = metadata[0]
|
||||
assert isinstance(first_attr, dict)
|
||||
assert "_raw_attributes" not in first_attr
|
||||
assert "Category" in first_attr
|
||||
assert "AWSService" in first_attr
|
||||
|
||||
def test_compliance_overview_attributes_missing_compliance_id(
|
||||
self, authenticated_client
|
||||
):
|
||||
@@ -15921,6 +16146,12 @@ class TestFindingGroupViewSet:
|
||||
assert attrs["fail_count"] == 0
|
||||
assert attrs["resources_total"] == 1
|
||||
assert attrs["resources_fail"] == 0
|
||||
# check_title / check_description are resolved post-pagination from the
|
||||
# summary table, not from the finding's check_metadata.
|
||||
assert attrs["check_title"] == "Ensure EC2 instances do not have public IPs"
|
||||
assert (
|
||||
attrs["check_description"] == "EC2 instances should use private IPs only."
|
||||
)
|
||||
|
||||
def test_finding_groups_status_pass_when_no_fail(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
@@ -17162,6 +17393,12 @@ class TestFindingGroupViewSet:
|
||||
assert attrs["fail_count"] == 0
|
||||
assert attrs["resources_total"] == 1
|
||||
assert attrs["resources_fail"] == 0
|
||||
# check_title / check_description are resolved post-pagination from the
|
||||
# summary table, not from the finding's check_metadata.
|
||||
assert attrs["check_title"] == "Ensure EC2 instances do not have public IPs"
|
||||
assert (
|
||||
attrs["check_description"] == "EC2 instances should use private IPs only."
|
||||
)
|
||||
|
||||
def test_finding_groups_latest_status_in_filter(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
@@ -17419,18 +17656,20 @@ class TestFindingGroupViewSet:
|
||||
check_ids = [item["id"] for item in data]
|
||||
assert check_ids == sorted(check_ids)
|
||||
|
||||
def test_finding_groups_latest_sort_by_check_title(
|
||||
def test_finding_groups_latest_sort_by_check_title_not_supported(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""Test /latest supports sorting by check_title."""
|
||||
"""check_title is not a sortable field for finding groups.
|
||||
|
||||
Titles live in the TOASTed check_metadata blob and are resolved after
|
||||
pagination from the summary table, so they cannot drive DB-level
|
||||
ordering. Requesting that sort is rejected.
|
||||
"""
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-group-latest"),
|
||||
{"sort": "check_title"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
check_titles = [item["attributes"]["check_title"] for item in data]
|
||||
assert check_titles == sorted(check_titles)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
|
||||
+182
-68
@@ -116,6 +116,7 @@ from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
|
||||
from api.compliance import (
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
|
||||
get_compliance_frameworks,
|
||||
get_prowler_provider_compliance,
|
||||
)
|
||||
from api.constants import SEVERITY_ORDER
|
||||
from api.db_router import MainRouter
|
||||
@@ -1849,7 +1850,42 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet):
|
||||
200: OpenApiResponse(
|
||||
description="CSV file containing the compliance report"
|
||||
),
|
||||
404: OpenApiResponse(description="Compliance report not found"),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="Compliance report not found, or the scan has no reports yet"
|
||||
),
|
||||
},
|
||||
request=None,
|
||||
),
|
||||
compliance_ocsf=extend_schema(
|
||||
tags=["Scan"],
|
||||
summary="Retrieve compliance report as OCSF JSON",
|
||||
description=(
|
||||
"Download a specific compliance report as an OCSF JSON file. "
|
||||
"Only universal frameworks that declare an output configuration "
|
||||
"produce this artifact (currently 'dora' and 'csa_ccm_4.0'); any "
|
||||
"other framework returns 404."
|
||||
),
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name="name",
|
||||
type=str,
|
||||
location=OpenApiParameter.PATH,
|
||||
required=True,
|
||||
description="The compliance report name, like 'dora'",
|
||||
),
|
||||
],
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
description="OCSF JSON file containing the compliance report"
|
||||
),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="Compliance report not found, the framework does "
|
||||
"not provide an OCSF export, or the scan has no reports yet"
|
||||
),
|
||||
},
|
||||
request=None,
|
||||
),
|
||||
@@ -1992,35 +2028,23 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
return queryset.select_related("provider", "task")
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "create":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return ScanCreateSerializer
|
||||
elif self.action == "partial_update":
|
||||
if self.action == "partial_update":
|
||||
return ScanUpdateSerializer
|
||||
elif self.action == "report":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return ScanReportSerializer
|
||||
elif self.action == "compliance":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return ScanComplianceReportSerializer
|
||||
elif self.action == "threatscore":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "ens":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "nis2":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "csa":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "cis":
|
||||
|
||||
action_defaults = {
|
||||
"create": ScanCreateSerializer,
|
||||
"report": ScanReportSerializer,
|
||||
"compliance": ScanComplianceReportSerializer,
|
||||
"compliance_ocsf": ScanComplianceReportSerializer,
|
||||
}
|
||||
response_only_actions = {"threatscore", "ens", "nis2", "csa", "cis"}
|
||||
|
||||
if self.action in action_defaults or self.action in response_only_actions:
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
if self.action in action_defaults:
|
||||
return action_defaults[self.action]
|
||||
|
||||
return super().get_serializer_class()
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
@@ -2059,12 +2083,17 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
if scan_instance.state == StateChoices.EXECUTING and scan_instance.task:
|
||||
task = scan_instance.task
|
||||
else:
|
||||
try:
|
||||
task = Task.objects.get(
|
||||
# A scan can have several `scan-report` tasks (e.g. re-runs); take the
|
||||
# most recent one. `.first()` also avoids `MultipleObjectsReturned`.
|
||||
task = (
|
||||
Task.objects.filter(
|
||||
task_runner_task__task_name="scan-report",
|
||||
task_runner_task__task_kwargs__contains=str(scan_instance.id),
|
||||
)
|
||||
except Task.DoesNotExist:
|
||||
.order_by("-inserted_at")
|
||||
.first()
|
||||
)
|
||||
if task is None:
|
||||
return None
|
||||
|
||||
self.response_serializer_class = TaskSerializer
|
||||
@@ -2139,27 +2168,32 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
contents = resp.get("Contents", [])
|
||||
keys = []
|
||||
matches = []
|
||||
for obj in contents:
|
||||
key = obj["Key"]
|
||||
key_basename = os.path.basename(key)
|
||||
if any(ch in suffix for ch in ("*", "?", "[")):
|
||||
if fnmatch.fnmatch(key_basename, suffix):
|
||||
keys.append(key)
|
||||
matches.append(obj)
|
||||
elif key_basename == suffix:
|
||||
keys.append(key)
|
||||
matches.append(obj)
|
||||
elif key.endswith(suffix):
|
||||
# Backward compatibility if suffix already includes directories
|
||||
keys.append(key)
|
||||
if not keys:
|
||||
matches.append(obj)
|
||||
if not matches:
|
||||
return Response(
|
||||
{
|
||||
"detail": f"No compliance file found for name '{os.path.splitext(suffix)[0]}'."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
# path_pattern here is prefix, but in compliance we build correct suffix check before
|
||||
key = keys[0]
|
||||
# Return the most recently modified match (latest report) when
|
||||
# several files share the prefix/suffix. `list_objects_v2` always
|
||||
# returns `LastModified`; the fallback keeps ordering deterministic
|
||||
# if it is ever absent.
|
||||
key = max(matches, key=lambda o: (o.get("LastModified", ""), o["Key"]))[
|
||||
"Key"
|
||||
]
|
||||
else:
|
||||
# path_pattern is exact key; HEAD before presigning to preserve the 404 contract.
|
||||
key = path_pattern
|
||||
@@ -2209,7 +2243,9 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
filepath = files[0]
|
||||
# Return the most recently modified match (latest report) when the
|
||||
# pattern resolves to several files.
|
||||
filepath = max(files, key=os.path.getmtime)
|
||||
with open(filepath, "rb") as f:
|
||||
content = f.read()
|
||||
filename = os.path.basename(filepath)
|
||||
@@ -2257,20 +2293,16 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "application/x-zip-compressed")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="compliance/(?P<name>[^/]+)",
|
||||
url_name="compliance",
|
||||
)
|
||||
def compliance(self, request, pk=None, name=None):
|
||||
scan = self.get_object()
|
||||
if name not in get_compliance_frameworks(scan.provider.provider):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
def _serve_compliance_artifact(self, scan, name, file_extension, content_type):
|
||||
"""Resolve and serve a per-framework compliance artifact from disk/S3.
|
||||
|
||||
Shared by the CSV and OCSF compliance download actions. Both are
|
||||
path-based (no query params) on purpose: ``get_object`` runs
|
||||
``filter_queryset``, which triggers JSON:API's
|
||||
``QueryParameterValidationFilter`` and 400s on any non-JSON:API
|
||||
query param, so a ``?format=`` / ``?type=`` selector is not viable
|
||||
here — the format is encoded in the route instead.
|
||||
"""
|
||||
running_resp = self._get_task_status(scan)
|
||||
if running_resp:
|
||||
return running_resp
|
||||
@@ -2287,25 +2319,66 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix), "compliance", f"{name}.csv"
|
||||
os.path.dirname(key_prefix), "compliance", f"{name}.{file_extension}"
|
||||
)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="text/csv",
|
||||
content_type=content_type,
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "compliance", f"*_{name}.csv")
|
||||
pattern = os.path.join(base, "compliance", f"*_{name}.{file_extension}")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "text/csv")
|
||||
return self._serve_file(content, filename, content_type)
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="compliance/(?P<name>[^/]+)",
|
||||
url_name="compliance",
|
||||
)
|
||||
def compliance(self, request, pk=None, name=None):
|
||||
scan = self.get_object()
|
||||
if name not in get_compliance_frameworks(scan.provider.provider):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
return self._serve_compliance_artifact(scan, name, "csv", "text/csv")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="compliance/(?P<name>[^/]+)/ocsf",
|
||||
url_name="compliance-ocsf",
|
||||
)
|
||||
def compliance_ocsf(self, request, pk=None, name=None):
|
||||
scan = self.get_object()
|
||||
if name not in get_compliance_frameworks(scan.provider.provider):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
universal_bulk = get_prowler_provider_compliance(scan.provider.provider)
|
||||
framework_obj = universal_bulk.get(name)
|
||||
if not (framework_obj and getattr(framework_obj, "outputs", None)):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' does not provide an OCSF export."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
return self._serve_compliance_artifact(
|
||||
scan, name, "ocsf.json", "application/json"
|
||||
)
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
@@ -3749,6 +3822,16 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
return queryset
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
def _optimize_tags_loading(self, queryset):
|
||||
"""Prefetch resource tags to avoid N+1 queries when serializing findings"""
|
||||
return queryset.prefetch_related(
|
||||
Prefetch(
|
||||
"resources__tags",
|
||||
queryset=ResourceTag.objects.filter(tenant_id=self.request.tenant_id),
|
||||
to_attr="prefetched_tags",
|
||||
)
|
||||
)
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
return self.paginate_by_pk(
|
||||
@@ -7369,6 +7452,15 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
|
||||
# `check_title` / `check_description` are intentionally NOT resolved
|
||||
# here. They live in the large JSONB `check_metadata` blob (TOASTed),
|
||||
# so reading them per finding row is very expensive, and pulling them
|
||||
# in via a correlated subquery makes Django add the subquery to GROUP
|
||||
# BY, which re-evaluates it once per input row. They are identical for
|
||||
# every finding of a `check_id`, so `_post_process_aggregation` fills
|
||||
# them from the summary table's plain columns in a single batched
|
||||
# lookup scoped to the paginated page.
|
||||
|
||||
# `pass_count`, `fail_count` and `manual_count` only count non-muted
|
||||
# findings. Muted findings are tracked separately via the
|
||||
# `*_muted_count` fields.
|
||||
@@ -7439,15 +7531,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
agg_failing_since=Min(
|
||||
"first_seen_at", filter=Q(status="FAIL", muted=False)
|
||||
),
|
||||
check_title=Coalesce(
|
||||
Max(KeyTextTransform("checktitle", "check_metadata")),
|
||||
Max(KeyTextTransform("CheckTitle", "check_metadata")),
|
||||
Max(KeyTextTransform("Checktitle", "check_metadata")),
|
||||
),
|
||||
check_description=Coalesce(
|
||||
Max(KeyTextTransform("description", "check_metadata")),
|
||||
Max(KeyTextTransform("Description", "check_metadata")),
|
||||
),
|
||||
)
|
||||
.annotate(
|
||||
# Group is muted only if it has zero non-muted findings.
|
||||
@@ -7484,14 +7567,17 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
|
||||
def _get_latest_findings_per_provider(self, filtered_queryset):
|
||||
"""Keep only findings from each provider's most recent completed scan."""
|
||||
latest_scan_ids = (
|
||||
# Materialize to a literal IN list. Left as a subquery, Postgres can't
|
||||
# estimate the match count and picks a serial nested loop on
|
||||
# resource_finding_mappings when one scan dominates findings
|
||||
latest_scan_ids = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=self.request.tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values("id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
return filtered_queryset.filter(scan_id__in=latest_scan_ids)
|
||||
|
||||
@@ -7503,9 +7589,38 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
- Computes aggregated status (FAIL > PASS > MANUAL); the orthogonal
|
||||
``muted`` boolean is already on the row from the SQL aggregation
|
||||
- Converts provider string to list
|
||||
- Fills check_title / check_description for the findings path
|
||||
"""
|
||||
rows = list(aggregated_data)
|
||||
|
||||
# The findings-aggregation path omits check_title / check_description
|
||||
# (they sit in TOASTed JSONB; see _aggregate_findings). Fill them from
|
||||
# the summary table's plain columns in one query scoped to this page.
|
||||
# The summary-aggregation path already carries them, so skip it there.
|
||||
if rows and "check_title" not in rows[0]:
|
||||
check_ids = [row["check_id"] for row in rows]
|
||||
role = get_role(self.request.user, self.request.tenant_id)
|
||||
summaries = FindingGroupDailySummary.objects.filter(
|
||||
tenant_id=self.request.tenant_id,
|
||||
check_id__in=check_ids,
|
||||
)
|
||||
# Scope to the user's providers, mirroring get_queryset(), so titles
|
||||
# are read only from providers the user can see.
|
||||
if not role.unlimited_visibility:
|
||||
summaries = summaries.filter(provider__in=get_providers(role))
|
||||
metadata_by_check = {
|
||||
item["check_id"]: item
|
||||
for item in summaries.order_by("check_id", "-inserted_at")
|
||||
.distinct("check_id")
|
||||
.values("check_id", "check_title", "check_description")
|
||||
}
|
||||
for row in rows:
|
||||
metadata = metadata_by_check.get(row["check_id"], {})
|
||||
row["check_title"] = metadata.get("check_title")
|
||||
row["check_description"] = metadata.get("check_description")
|
||||
|
||||
results = []
|
||||
for row in aggregated_data:
|
||||
for row in rows:
|
||||
# Convert severity order back to string
|
||||
severity_order = row.get("severity_order", 1)
|
||||
row["severity"] = SEVERITY_ORDER_REVERSE.get(
|
||||
@@ -7551,7 +7666,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
|
||||
_FINDING_GROUP_SORT_MAP = {
|
||||
"check_id": "check_id",
|
||||
"check_title": "check_title",
|
||||
"severity": "severity_order",
|
||||
"status": "status_order",
|
||||
"muted": "muted",
|
||||
|
||||
@@ -26,6 +26,61 @@ celery_app.conf.result_backend_transport_options = {
|
||||
}
|
||||
celery_app.conf.visibility_timeout = BROKER_VISIBILITY_TIMEOUT
|
||||
|
||||
# Durable delivery: keep the message until the task finishes, so a worker killed
|
||||
# mid-task (deploy/OOM/eviction) does not silently drop it. Reserve one task at a
|
||||
# time so a crash exposes at most one extra reserved message.
|
||||
celery_app.conf.task_acks_late = True
|
||||
celery_app.conf.task_reject_on_worker_lost = True
|
||||
celery_app.conf.worker_prefetch_multiplier = env.int(
|
||||
"DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER", default=1
|
||||
)
|
||||
# On SIGTERM, give the worker time to finish or re-queue in-flight tasks before
|
||||
# it is forcefully killed (Celery 5.5+ soft shutdown).
|
||||
celery_app.conf.worker_soft_shutdown_timeout = env.int(
|
||||
"DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT", default=60
|
||||
)
|
||||
# Bound execution so a blocked task cannot pin a worker forever. Connection
|
||||
# checks get a tight limit; scans and provider/tenant deletions can legitimately
|
||||
# run for more than a day on large tenants, so they get a much higher cap.
|
||||
# The default for every other task is set as the global limit, not as a "*"
|
||||
# annotation: Celery applies the "*" entry AFTER the per-task one, so a "*" in
|
||||
# task_annotations would silently overwrite every specific limit defined below.
|
||||
_TASK_HARD_LIMIT = env.int("DJANGO_CELERY_TASK_TIME_LIMIT", default=6 * 60 * 60)
|
||||
_TASK_SOFT_LIMIT = env.int(
|
||||
"DJANGO_CELERY_TASK_SOFT_TIME_LIMIT", default=_TASK_HARD_LIMIT - 600
|
||||
)
|
||||
_LONG_TASK_HARD_LIMIT = env.int(
|
||||
"DJANGO_CELERY_LONG_TASK_TIME_LIMIT", default=48 * 60 * 60
|
||||
)
|
||||
_LONG_TASK_SOFT_LIMIT = env.int(
|
||||
"DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT", default=_LONG_TASK_HARD_LIMIT - 600
|
||||
)
|
||||
celery_app.conf.task_time_limit = _TASK_HARD_LIMIT
|
||||
celery_app.conf.task_soft_time_limit = _TASK_SOFT_LIMIT
|
||||
celery_app.conf.task_annotations = {
|
||||
**{
|
||||
name: {"soft_time_limit": 60, "time_limit": 120}
|
||||
for name in (
|
||||
"provider-connection-check",
|
||||
"integration-connection-check",
|
||||
"lighthouse-connection-check",
|
||||
"lighthouse-provider-connection-check",
|
||||
)
|
||||
},
|
||||
**{
|
||||
name: {
|
||||
"soft_time_limit": _LONG_TASK_SOFT_LIMIT,
|
||||
"time_limit": _LONG_TASK_HARD_LIMIT,
|
||||
}
|
||||
for name in (
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"provider-deletion",
|
||||
"tenant-deletion",
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
celery_app.autodiscover_tasks(["api"])
|
||||
|
||||
|
||||
|
||||
@@ -306,3 +306,32 @@ SESSION_COOKIE_SECURE = True
|
||||
ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
|
||||
) # 48h
|
||||
|
||||
# Orphan task recovery feature flags. The master switch is OFF by default, so task
|
||||
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
|
||||
# toggles default to enabled, so once the master is on every group recovers unless a
|
||||
# group is explicitly turned off.
|
||||
TASK_RECOVERY_ENABLED = env.bool("DJANGO_TASK_RECOVERY_ENABLED", False)
|
||||
TASK_RECOVERY_SUMMARIES_ENABLED = env.bool(
|
||||
"DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED", True
|
||||
)
|
||||
TASK_RECOVERY_DELETIONS_ENABLED = env.bool(
|
||||
"DJANGO_TASK_RECOVERY_DELETIONS_ENABLED", True
|
||||
)
|
||||
|
||||
|
||||
def label_postgres_connections(databases):
|
||||
"""Tag each Postgres connection with ``application_name="<component>:<alias>"``
|
||||
so connections are attributable by component in ``pg_stat_activity`` (and any
|
||||
tooling that surfaces ``application_name``). The component (api / worker /
|
||||
scan / ...) is injected per process by the container entrypoint via
|
||||
``DJANGO_APP_COMPONENT``; the alias distinguishes which pool inside the
|
||||
process owns the connection. The neo4j entry is skipped (not a Postgres
|
||||
backend). Postgres truncates ``application_name`` at 63 bytes.
|
||||
"""
|
||||
component = env.str("DJANGO_APP_COMPONENT", default="api")
|
||||
for alias, config in databases.items():
|
||||
engine = config.get("ENGINE", "")
|
||||
if engine.startswith("psqlextra") or "postgresql" in engine:
|
||||
name = f"{component}:{alias}"[:63]
|
||||
config.setdefault("OPTIONS", {})["application_name"] = name
|
||||
|
||||
@@ -54,6 +54,8 @@ DATABASES = {
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
label_postgres_connections(DATABASES) # noqa: F405
|
||||
|
||||
REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = tuple( # noqa: F405
|
||||
render_class
|
||||
for render_class in REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] # noqa: F405
|
||||
|
||||
@@ -58,3 +58,5 @@ DATABASES = {
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
label_postgres_connections(DATABASES) # noqa: F405
|
||||
|
||||
@@ -34,3 +34,8 @@ DRF_API_KEY = {
|
||||
# JWT
|
||||
|
||||
SIMPLE_JWT["ALGORITHM"] = "HS256" # noqa: F405
|
||||
# pyjwt >= 2.13.0 rejects an empty HMAC signing key, so HS256 tests need a real
|
||||
# key (>= 32 bytes also avoids the InsecureKeyLengthWarning). Production uses RS256.
|
||||
SIMPLE_JWT["SIGNING_KEY"] = env.str( # noqa: F405
|
||||
"DJANGO_TOKEN_SIGNING_KEY", "insecure-testing-jwt-signing-key-do-not-use-in-prod"
|
||||
)
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from celery import current_app, states
|
||||
from celery import states
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES
|
||||
from tasks.jobs.attack_paths.db_utils import (
|
||||
_mark_scan_finished,
|
||||
recover_graph_data_ready,
|
||||
)
|
||||
from tasks.jobs.orphan_recovery import is_worker_alive as _is_worker_alive
|
||||
from tasks.jobs.orphan_recovery import revoke_task as _revoke_task
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
@@ -150,32 +152,6 @@ def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _is_worker_alive(worker: str) -> bool:
|
||||
"""Ping a specific Celery worker. Returns `True` if it responds or on error."""
|
||||
try:
|
||||
response = current_app.control.inspect(destination=[worker], timeout=1.0).ping()
|
||||
return response is not None and worker in response
|
||||
except Exception:
|
||||
logger.exception(f"Failed to ping worker {worker}, treating as alive")
|
||||
return True
|
||||
|
||||
|
||||
def _revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task. Non-fatal on failure.
|
||||
|
||||
`terminate=True` SIGTERMs the worker if the task is mid-execution; use
|
||||
for EXECUTING cleanup. `terminate=False` only marks the task id revoked
|
||||
across workers, so any worker pulling the queued message discards it;
|
||||
use for SCHEDULED cleanup where the task hasn't run yet.
|
||||
"""
|
||||
try:
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
|
||||
|
||||
def _cleanup_scan(scan, task_result, reason: str) -> bool:
|
||||
"""
|
||||
Clean up a single stale `AttackPathsScan`:
|
||||
|
||||
@@ -39,11 +39,6 @@ from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS
|
||||
from prowler.lib.outputs.compliance.cisa_scuba.cisa_scuba_googleworkspace import (
|
||||
GoogleWorkspaceCISASCuBA,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.csa.csa_alibabacloud import AlibabaCloudCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_aws import AWSCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_azure import AzureCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_gcp import GCPCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_oraclecloud import OracleCloudCSA
|
||||
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
|
||||
@@ -102,7 +97,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "prowler_threatscore_aws", ProwlerThreatScoreAWS),
|
||||
(lambda name: name.startswith("ccc_"), CCC_AWS),
|
||||
(lambda name: name.startswith("c5_"), AWSC5),
|
||||
(lambda name: name.startswith("csa_"), AWSCSA),
|
||||
(lambda name: name == "asd_essential_eight_aws", ASDEssentialEightAWS),
|
||||
],
|
||||
"azure": [
|
||||
@@ -113,7 +107,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("ccc_"), CCC_Azure),
|
||||
(lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure),
|
||||
(lambda name: name == "c5_azure", AzureC5),
|
||||
(lambda name: name.startswith("csa_"), AzureCSA),
|
||||
],
|
||||
"gcp": [
|
||||
(lambda name: name.startswith("cis_"), GCPCIS),
|
||||
@@ -123,7 +116,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP),
|
||||
(lambda name: name.startswith("ccc_"), CCC_GCP),
|
||||
(lambda name: name == "c5_gcp", GCPC5),
|
||||
(lambda name: name.startswith("csa_"), GCPCSA),
|
||||
],
|
||||
"kubernetes": [
|
||||
(lambda name: name.startswith("cis_"), KubernetesCIS),
|
||||
@@ -152,11 +144,9 @@ COMPLIANCE_CLASS_MAP = {
|
||||
"image": [],
|
||||
"oraclecloud": [
|
||||
(lambda name: name.startswith("cis_"), OracleCloudCIS),
|
||||
(lambda name: name.startswith("csa_"), OracleCloudCSA),
|
||||
],
|
||||
"alibabacloud": [
|
||||
(lambda name: name.startswith("cis_"), AlibabaCloudCIS),
|
||||
(lambda name: name.startswith("csa_"), AlibabaCloudCSA),
|
||||
(
|
||||
lambda name: name == "prowler_threatscore_alibabacloud",
|
||||
ProwlerThreatScoreAlibaba,
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Detect and recover orphaned Celery tasks.
|
||||
|
||||
A task is "orphaned" when its result row is non-terminal (STARTED/RECEIVED) but the
|
||||
worker that was running it is gone (deploy, OOM, eviction). We tell a real orphan
|
||||
from a still-running task by pinging the worker recorded on its `TaskResult`:
|
||||
|
||||
- worker responds -> the task is in flight, leave it alone (never double-run);
|
||||
- worker is gone -> real orphan: mark the stale result terminal (so pending/started
|
||||
alerts clear), then re-enqueue the task from its stored name + kwargs.
|
||||
|
||||
This recovers only allowlisted tasks with local, proven idempotency. Celery's
|
||||
`result_extended=True` gives us the stored `task_name`/`task_kwargs`/`worker` once
|
||||
the task starts, but external side-effect tasks are failed instead of blindly
|
||||
re-run. A small recovery cap stops a task that repeatedly kills its worker from
|
||||
looping forever.
|
||||
|
||||
This is the shared engine behind both the periodic Beat watchdog and the
|
||||
`reconcile_orphan_tasks` management command.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from celery import current_app, states
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db import connections
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Arbitrary constant key for pg_try_advisory_lock so only one reconciliation
|
||||
# runs at a time across replicas / the watchdog / the command.
|
||||
ORPHAN_RECOVERY_LOCK_KEY = 0x70726F77 # "prow"
|
||||
|
||||
# Non-terminal states that mean "a worker had this and may have died with it".
|
||||
IN_FLIGHT_STATES = (states.STARTED, states.RECEIVED)
|
||||
|
||||
# Tasks with proven idempotency are eligible for auto re-enqueue, grouped so each
|
||||
# group can be toggled independently by a feature flag (see config.django.base).
|
||||
# Summaries clear and rewrite their own rows and deletions are idempotent. Tasks with
|
||||
# external side effects are never eligible: integration-jira would create duplicate
|
||||
# issues, integration-s3 rebuilds its upload from worker-local files that do not
|
||||
# survive a crash, and report/Security Hub recovery is out of scope.
|
||||
RECOVERY_TASK_GROUPS = {
|
||||
"summaries": {
|
||||
"scan-summary",
|
||||
"scan-compliance-overviews",
|
||||
"scan-provider-compliance-scores",
|
||||
"scan-daily-severity",
|
||||
"scan-finding-group-summaries",
|
||||
"scan-reset-ephemeral-resources",
|
||||
},
|
||||
"deletions": {"provider-deletion", "tenant-deletion"},
|
||||
}
|
||||
|
||||
|
||||
def reenqueueable_tasks() -> set[str]:
|
||||
"""Task names eligible for auto re-enqueue, honoring the per-group feature flags.
|
||||
|
||||
A group whose flag is disabled is dropped, so its orphaned tasks are marked
|
||||
terminal instead of re-enqueued.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
group_enabled = {
|
||||
"summaries": settings.TASK_RECOVERY_SUMMARIES_ENABLED,
|
||||
"deletions": settings.TASK_RECOVERY_DELETIONS_ENABLED,
|
||||
}
|
||||
return {
|
||||
task
|
||||
for group, tasks in RECOVERY_TASK_GROUPS.items()
|
||||
if group_enabled[group]
|
||||
for task in tasks
|
||||
}
|
||||
|
||||
|
||||
# Tasks the watchdog ignores entirely (not even marked terminal): scan tasks are not
|
||||
# auto-recovered, since re-running a scan is not safe to do automatically; attack-paths
|
||||
# scans are handled by their own stale-cleanup (which also drops the temp Neo4j db);
|
||||
# and the maintenance tasks must not self-recover (they run again on their own schedule).
|
||||
_SKIP_RECOVERY = {
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"attack-paths-scan-perform",
|
||||
"attack-paths-cleanup-stale-scans",
|
||||
"reconcile-orphan-tasks",
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def advisory_lock(key: int = ORPHAN_RECOVERY_LOCK_KEY, using: str = "default"):
|
||||
"""Yield True if this session won a Postgres advisory lock, else False.
|
||||
|
||||
Non-blocking: losers get False and should no-op. The lock is released on
|
||||
exit (and implicitly if the session dies).
|
||||
"""
|
||||
with connections[using].cursor() as cursor:
|
||||
cursor.execute("SELECT pg_try_advisory_lock(%s)", [key])
|
||||
acquired = bool(cursor.fetchone()[0])
|
||||
try:
|
||||
yield acquired
|
||||
finally:
|
||||
if acquired:
|
||||
cursor.execute("SELECT pg_advisory_unlock(%s)", [key])
|
||||
|
||||
|
||||
def is_worker_alive(worker: str, timeout: float = 1.0) -> bool:
|
||||
"""Ping a specific Celery worker. Returns True if it responds, or on error.
|
||||
|
||||
Erring on the side of "alive" means an unreachable control bus never causes
|
||||
a still-running task to be re-enqueued.
|
||||
"""
|
||||
try:
|
||||
response = current_app.control.inspect(
|
||||
destination=[worker], timeout=timeout
|
||||
).ping()
|
||||
return response is not None and worker in response
|
||||
except Exception:
|
||||
logger.exception(f"Failed to ping worker {worker}, treating as alive")
|
||||
return True
|
||||
|
||||
|
||||
def revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task by its TaskResult. Non-fatal on failure.
|
||||
|
||||
terminate=True SIGTERMs the worker if the task is mid-execution; terminate=False
|
||||
only marks the id revoked so any worker pulling the queued message discards it
|
||||
(use before re-enqueuing, so a later broker redelivery of the stale message is
|
||||
dropped).
|
||||
"""
|
||||
try:
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
|
||||
|
||||
def _decode_celery_field(value, default):
|
||||
"""Decode django-celery-results' stored task_args/task_kwargs to a Python object.
|
||||
|
||||
The backend stores them as a (sometimes double-encoded) repr/JSON string. An
|
||||
empty or missing field returns ``default``; a non-empty value that cannot be
|
||||
decoded raises ``ValueError`` so the caller can avoid re-enqueuing a task with
|
||||
the wrong arguments.
|
||||
"""
|
||||
obj = value
|
||||
for _ in range(2): # values can be double-encoded (a string holding a repr)
|
||||
if not isinstance(obj, str):
|
||||
break
|
||||
text = obj.strip()
|
||||
if not text:
|
||||
return default
|
||||
parsed = None
|
||||
for parser in (ast.literal_eval, json.loads):
|
||||
try:
|
||||
parsed = parser(text)
|
||||
break
|
||||
except (ValueError, SyntaxError, TypeError):
|
||||
continue
|
||||
if parsed is None:
|
||||
raise ValueError(f"undecodable celery field: {text[:120]!r}")
|
||||
obj = parsed
|
||||
return default if obj is None else obj
|
||||
|
||||
|
||||
def reconcile_orphans(
|
||||
grace_minutes: int = 2,
|
||||
max_attempts: int = 3,
|
||||
window_hours: int = 6,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Run the full orphan sweep under a single-flight advisory lock.
|
||||
|
||||
Recovers any orphaned in-flight task and delegates attack-paths scans that
|
||||
never reached a worker to their existing stale-cleanup. Returns a summary;
|
||||
a no-op (lock not won) is reported too.
|
||||
"""
|
||||
with advisory_lock() as acquired:
|
||||
if not acquired:
|
||||
logger.info("Orphan reconcile skipped: another run holds the lock")
|
||||
return {"acquired": False}
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
if settings.TASK_RECOVERY_ENABLED:
|
||||
# Populate the task registry so we can re-enqueue any task by name.
|
||||
import tasks.tasks # noqa: F401
|
||||
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=grace_minutes,
|
||||
max_attempts=max_attempts,
|
||||
window_hours=window_hours,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
result["enabled"] = True
|
||||
else:
|
||||
logger.info("Orphan task recovery disabled by feature flag")
|
||||
result = {"recovered": [], "failed": [], "skipped": [], "enabled": False}
|
||||
|
||||
if not dry_run:
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
result["attack_paths"] = cleanup_stale_attack_paths_scans()
|
||||
|
||||
return {"acquired": True, **result}
|
||||
|
||||
|
||||
def _reconcile_task_results(
|
||||
grace_minutes: int, max_attempts: int, window_hours: int, dry_run: bool
|
||||
) -> dict:
|
||||
from django_celery_results.models import TaskResult
|
||||
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(minutes=grace_minutes)
|
||||
candidates = list(
|
||||
TaskResult.objects.filter(status__in=IN_FLIGHT_STATES, date_created__lt=cutoff)
|
||||
.exclude(worker__isnull=True)
|
||||
.exclude(worker="")
|
||||
.exclude(task_name__in=_SKIP_RECOVERY)
|
||||
)
|
||||
|
||||
# Ping each distinct worker at most once.
|
||||
worker_alive = {w: is_worker_alive(w) for w in {tr.worker for tr in candidates}}
|
||||
|
||||
recovered, failed, skipped = [], [], []
|
||||
for task_result in candidates:
|
||||
if worker_alive.get(task_result.worker, True):
|
||||
skipped.append(task_result.task_id) # in flight, do not double-run
|
||||
continue
|
||||
if dry_run:
|
||||
recovered.append(task_result.task_id)
|
||||
continue
|
||||
outcome = _recover_task(task_result, max_attempts, window_hours)
|
||||
(recovered if outcome == "recovered" else failed).append(task_result.task_id)
|
||||
|
||||
logger.info(
|
||||
"Orphan reconcile: recovered=%d failed=%d skipped(in-flight)=%d",
|
||||
len(recovered),
|
||||
len(failed),
|
||||
len(skipped),
|
||||
)
|
||||
return {"recovered": recovered, "failed": failed, "skipped": skipped}
|
||||
|
||||
|
||||
def _recovery_attempt_count(name: str, kwargs_repr, window_hours: int) -> int:
|
||||
"""Increment and return the recovery count for this (task, kwargs) within the
|
||||
window. Backed by Valkey so it survives result-row churn (a worker processing
|
||||
the revoke can blank the TaskResult fields). Fail-open if Valkey is down (the
|
||||
broker being unreachable means nothing is running anyway).
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
try:
|
||||
import redis
|
||||
|
||||
client = redis.from_url(settings.CELERY_BROKER_URL)
|
||||
signature = f"{name}|{kwargs_repr}".encode()
|
||||
key = (
|
||||
"orphan-recovery:"
|
||||
+ hashlib.sha1(signature, usedforsecurity=False).hexdigest()
|
||||
)
|
||||
count = client.incr(key)
|
||||
if count == 1:
|
||||
client.expire(key, max(1, window_hours) * 3600)
|
||||
return int(count)
|
||||
except Exception:
|
||||
logger.exception("Recovery-attempt counter unavailable; allowing recovery")
|
||||
return 1
|
||||
|
||||
|
||||
def _recover_task(task_result, max_attempts: int, window_hours: int) -> str:
|
||||
"""Recover one orphaned task. Returns 'recovered' or 'failed'."""
|
||||
# Capture name/args/kwargs now: revoking can let a worker blank the row.
|
||||
name = task_result.task_name
|
||||
args_repr = task_result.task_args
|
||||
kwargs_repr = task_result.task_kwargs
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
|
||||
# Drop any future broker redelivery of the stale message.
|
||||
revoke_task(task_result, terminate=False)
|
||||
|
||||
# Mark the stale result terminal so "pending/started forever" alerts clear.
|
||||
task_result.status = states.REVOKED
|
||||
task_result.date_done = now
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
if name not in reenqueueable_tasks():
|
||||
logger.warning(
|
||||
"Orphan %s (%s) not re-enqueued: not allowlisted for auto recovery",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
# Count the attempt only once the task is allowlisted, so a task sitting in a
|
||||
# disabled group does not burn its recovery budget while the flag is off (and is
|
||||
# not already over the cap the moment the group is re-enabled).
|
||||
attempt = _recovery_attempt_count(name, kwargs_repr, window_hours)
|
||||
if attempt > max_attempts:
|
||||
logger.warning(
|
||||
"Orphan %s (%s) not re-enqueued: recovery cap reached (%d/%d)",
|
||||
task_result.task_id,
|
||||
name,
|
||||
attempt,
|
||||
max_attempts,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
task_obj = current_app.tasks.get(name)
|
||||
if task_obj is None:
|
||||
logger.error(
|
||||
"Orphan %s: task %s not registered, cannot re-enqueue",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
try:
|
||||
args = _decode_celery_field(args_repr, [])
|
||||
kwargs = _decode_celery_field(kwargs_repr, {})
|
||||
except ValueError:
|
||||
logger.error(
|
||||
"Orphan %s (%s): could not decode stored args/kwargs, not re-enqueuing",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
new_task_id = str(uuid4())
|
||||
task_obj.apply_async(
|
||||
args=list(args) if isinstance(args, (list, tuple)) else [],
|
||||
kwargs=kwargs if isinstance(kwargs, dict) else {},
|
||||
task_id=new_task_id,
|
||||
)
|
||||
logger.info(
|
||||
"Re-enqueued orphan %s (%s) as %s", task_result.task_id, name, new_task_id
|
||||
)
|
||||
return "recovered"
|
||||
@@ -29,7 +29,10 @@ from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
@@ -571,7 +574,7 @@ def generate_csa_report(
|
||||
Args:
|
||||
tenant_id: The tenant ID for Row-Level Security context.
|
||||
scan_id: ID of the scan executed by Prowler.
|
||||
compliance_id: ID of the compliance framework (e.g., "csa_ccm_4.0_aws").
|
||||
compliance_id: ID of the compliance framework (e.g., "csa_ccm_4.0").
|
||||
output_path: Output PDF file path.
|
||||
provider_id: Provider ID for the scan.
|
||||
only_failed: If True, only include failed requirements in detailed section.
|
||||
@@ -883,9 +886,11 @@ def generate_compliance_reports(
|
||||
frameworks_bulk.get(f"nis2_{provider_type}")
|
||||
)
|
||||
if generate_csa:
|
||||
pending_checks_by_framework["csa"] = _get_compliance_check_ids(
|
||||
frameworks_bulk.get(f"csa_ccm_4.0_{provider_type}")
|
||||
)
|
||||
# csa_ccm_4.0 lives at the top level, not under compliance/{provider}/.
|
||||
csa_framework = frameworks_bulk.get(
|
||||
"csa_ccm_4.0"
|
||||
) or get_bulk_compliance_frameworks_universal(provider_type).get("csa_ccm_4.0")
|
||||
pending_checks_by_framework["csa"] = _get_compliance_check_ids(csa_framework)
|
||||
if generate_cis and latest_cis:
|
||||
pending_checks_by_framework["cis"] = _get_compliance_check_ids(
|
||||
frameworks_bulk.get(latest_cis)
|
||||
@@ -1183,7 +1188,7 @@ def generate_compliance_reports(
|
||||
if generate_csa:
|
||||
generated_report_keys.append("csa")
|
||||
csa_path = output_paths["csa"]
|
||||
compliance_id_csa = f"csa_ccm_4.0_{provider_type}"
|
||||
compliance_id_csa = "csa_ccm_4.0"
|
||||
pdf_path_csa = f"{csa_path}_csa_report.pdf"
|
||||
logger.info("Generating CSA CCM report with compliance %s", compliance_id_csa)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
@@ -26,7 +27,10 @@ from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, StatusChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
from .components import (
|
||||
@@ -222,6 +226,46 @@ def get_requirement_metadata(
|
||||
return None
|
||||
|
||||
|
||||
def _universal_attributes_to_list(attributes) -> list:
|
||||
"""Flatten a universal requirement's ``attributes`` into a list of objects
|
||||
with attribute access. MITRE wraps its list under ``_raw_attributes``."""
|
||||
if isinstance(attributes, dict) and "_raw_attributes" in attributes:
|
||||
entries = attributes.get("_raw_attributes") or []
|
||||
return [
|
||||
SimpleNamespace(**entry) for entry in entries if isinstance(entry, dict)
|
||||
]
|
||||
if isinstance(attributes, dict):
|
||||
return [SimpleNamespace(**attributes)] if attributes else []
|
||||
return list(attributes or [])
|
||||
|
||||
|
||||
def _adapt_universal_to_legacy(framework, provider_type: str) -> SimpleNamespace:
|
||||
"""Expose a universal ``ComplianceFramework`` under the legacy ``Compliance``
|
||||
attribute names used by the PDF pipeline."""
|
||||
provider_key = (provider_type or "").lower()
|
||||
requirements = []
|
||||
for requirement in framework.requirements:
|
||||
checks_by_provider = (
|
||||
requirement.checks if isinstance(requirement.checks, dict) else {}
|
||||
)
|
||||
requirements.append(
|
||||
SimpleNamespace(
|
||||
Id=requirement.id,
|
||||
Description=requirement.description or "",
|
||||
Checks=list(checks_by_provider.get(provider_key, [])),
|
||||
Attributes=_universal_attributes_to_list(requirement.attributes),
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
Framework=framework.framework,
|
||||
Name=framework.name,
|
||||
Version=framework.version or "",
|
||||
Description=framework.description or "",
|
||||
Provider=framework.provider or provider_type,
|
||||
Requirements=requirements,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PDF Styles Cache
|
||||
# =============================================================================
|
||||
@@ -869,9 +913,18 @@ class BaseComplianceReportGenerator(ABC):
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
compliance_obj = frameworks_bulk.get(compliance_id)
|
||||
# Load compliance framework — fall back to the universal loader
|
||||
# for top-level JSONs (e.g. csa_ccm_4.0) that Compliance.get_bulk
|
||||
# does not scan.
|
||||
compliance_obj = Compliance.get_bulk(provider_type).get(compliance_id)
|
||||
if not compliance_obj:
|
||||
universal_framework = get_bulk_compliance_frameworks_universal(
|
||||
provider_type
|
||||
).get(compliance_id)
|
||||
if universal_framework:
|
||||
compliance_obj = _adapt_universal_to_legacy(
|
||||
universal_framework, provider_type
|
||||
)
|
||||
|
||||
if not compliance_obj:
|
||||
raise ValueError(f"Compliance framework not found: {compliance_id}")
|
||||
|
||||
+465
-275
@@ -42,7 +42,6 @@ from api.db_utils import (
|
||||
SET_CONFIG_QUERY,
|
||||
psycopg_connection,
|
||||
rls_transaction,
|
||||
update_objects_in_batches,
|
||||
)
|
||||
from api.exceptions import ProviderConnectionError
|
||||
from api.models import (
|
||||
@@ -59,6 +58,7 @@ from api.models import (
|
||||
ResourceFindingMapping,
|
||||
ResourceScanSummary,
|
||||
ResourceTag,
|
||||
ResourceTagMapping,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
@@ -97,8 +97,16 @@ COMPLIANCE_REQUIREMENT_COPY_COLUMNS = (
|
||||
)
|
||||
# Controls how many findings we process per micro-batch before flushing to DB writes
|
||||
FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=3000)
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=500)
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres.
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=1000)
|
||||
# Throttle scan progress persistence: minimum progress delta (fraction 0-1)
|
||||
# between two persisted progress updates.
|
||||
PROGRESS_THROTTLE_DELTA = env.float("DJANGO_SCAN_PROGRESS_THROTTLE_DELTA", default=0.01)
|
||||
# Throttle scan progress persistence: maximum seconds without persisting progress
|
||||
# regardless of delta (so slow checks still show progress in the UI).
|
||||
PROGRESS_THROTTLE_SECONDS = env.float(
|
||||
"DJANGO_SCAN_PROGRESS_THROTTLE_SECONDS", default=10.0
|
||||
)
|
||||
|
||||
ATTACK_SURFACE_PROVIDER_COMPATIBILITY = {
|
||||
"internet-exposed": None, # Compatible with all providers
|
||||
@@ -468,9 +476,12 @@ def _create_compliance_summaries(
|
||||
)
|
||||
)
|
||||
|
||||
# Bulk insert summaries
|
||||
if summary_objects:
|
||||
with rls_transaction(tenant_id):
|
||||
# Idempotent re-run: clear this scan's prior summaries before re-inserting, so a
|
||||
# recovered scan-compliance-overviews run reflects its own re-derived rows instead
|
||||
# of keeping a stale one (bulk_create ignore_conflicts alone would keep the old).
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceOverviewSummary.objects.filter(scan_id=scan_id).delete()
|
||||
if summary_objects:
|
||||
ComplianceOverviewSummary.objects.bulk_create(
|
||||
summary_objects, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
@@ -528,16 +539,26 @@ def _process_finding_micro_batch(
|
||||
"""
|
||||
# Accumulate objects for bulk operations
|
||||
findings_to_create = []
|
||||
mappings_to_create = []
|
||||
dirty_resources = {}
|
||||
resources_with_new_tag_mappings: set[str] = set()
|
||||
resource_denormalized_data = [] # (finding_instance, resource_instance) pairs
|
||||
tag_mappings_to_create: list[ResourceTagMapping] = []
|
||||
skipped_findings_count = 0 # Track findings skipped due to UID length
|
||||
|
||||
# Prefetch last statuses for all findings in this batch
|
||||
# TEMPORARY WORKAROUND: Filter out UIDs > 300 chars to avoid query errors
|
||||
finding_uids = [
|
||||
f.uid for f in findings_batch if f is not None and len(f.uid) <= 300
|
||||
]
|
||||
# Separate findings into those persistable (uid <= 300) and over-limit.
|
||||
# Resources/tags ARE still resolved for over-limit findings to preserve the
|
||||
# original behavior (resources are persisted even when their finding is dropped).
|
||||
non_null_findings = [f for f in findings_batch if f is not None]
|
||||
persistable_findings = [f for f in non_null_findings if len(f.uid) <= 300]
|
||||
skipped_findings_count = len(non_null_findings) - len(persistable_findings)
|
||||
none_count = len(findings_batch) - len(non_null_findings)
|
||||
if none_count:
|
||||
logger.error(
|
||||
f"{none_count} None finding(s) detected on scan {scan_instance.id}."
|
||||
)
|
||||
|
||||
# Prefetch last statuses for all persistable findings in this batch (read replica)
|
||||
finding_uids = [f.uid for f in persistable_findings]
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
last_statuses = {
|
||||
item["uid"]: (item["status"], item["first_seen_at"])
|
||||
@@ -548,281 +569,411 @@ def _process_finding_micro_batch(
|
||||
.order_by("uid", "-inserted_at")
|
||||
.distinct("uid")
|
||||
}
|
||||
# Update cache
|
||||
for uid, data in last_statuses.items():
|
||||
if uid not in last_status_cache:
|
||||
last_status_cache[uid] = data
|
||||
|
||||
# Process each finding in the batch
|
||||
for finding in findings_batch:
|
||||
if finding is None:
|
||||
logger.error(f"None finding detected on scan {scan_instance.id}.")
|
||||
continue
|
||||
# All DB writes for this micro-batch run inside ONE rls_transaction,
|
||||
# with deadlock-retry at micro-batch granularity instead of per-finding.
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
# 1) Pre-resolve Resources in bulk
|
||||
# Collect all uids referenced by this batch that are not in cache yet.
|
||||
# NOTE: we intentionally include empty-string uids here. The SDK
|
||||
# explicitly emits findings with `resource_uid=""` for some flows
|
||||
# (IaC scans, some Azure/GCP/K8s checks). The original
|
||||
# `get_or_create` behavior was to create/share a Resource with
|
||||
# uid="" for these findings rather than dropping them. Preserve
|
||||
# that behavior; do NOT filter by truthiness.
|
||||
batch_resource_uids: set[str] = set()
|
||||
for f in non_null_findings:
|
||||
if f.resource_uid not in resource_cache:
|
||||
batch_resource_uids.add(f.resource_uid)
|
||||
|
||||
# Process resource with deadlock retry
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
resource_uid = finding.resource_uid
|
||||
if resource_uid not in resource_cache:
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
resource_instance, _ = Resource.objects.get_or_create(
|
||||
if batch_resource_uids:
|
||||
existing_resources = {
|
||||
r.uid: r
|
||||
for r in Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider_instance,
|
||||
uid=resource_uid,
|
||||
defaults={
|
||||
"region": finding.region,
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
"name": finding.resource_name,
|
||||
"groups": [group] if group else None,
|
||||
},
|
||||
provider_id=provider_instance.id,
|
||||
uid__in=batch_resource_uids,
|
||||
)
|
||||
resource_cache[resource_uid] = resource_instance
|
||||
resource_failed_findings_cache[resource_uid] = 0
|
||||
else:
|
||||
resource_instance = resource_cache[resource_uid]
|
||||
break
|
||||
except (OperationalError, IntegrityError) as db_err:
|
||||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||||
logger.warning(
|
||||
f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} "
|
||||
f"detected when processing resource {resource_uid} on scan {scan_instance.id}. Retrying..."
|
||||
}
|
||||
missing_uids = batch_resource_uids - existing_resources.keys()
|
||||
if missing_uids:
|
||||
# Build defaults from the first finding referencing each uid.
|
||||
first_finding_per_uid: dict[str, ProwlerFinding] = {}
|
||||
for f in non_null_findings:
|
||||
if f.resource_uid in missing_uids:
|
||||
first_finding_per_uid.setdefault(f.resource_uid, f)
|
||||
resources_to_create = []
|
||||
for uid in missing_uids:
|
||||
f = first_finding_per_uid[uid]
|
||||
check_metadata = f.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
resources_to_create.append(
|
||||
Resource(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider_instance,
|
||||
uid=uid,
|
||||
region=f.region,
|
||||
service=f.service_name,
|
||||
type=f.resource_type,
|
||||
name=f.resource_name,
|
||||
groups=[group] if group else None,
|
||||
)
|
||||
)
|
||||
Resource.objects.bulk_create(
|
||||
resources_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "provider_id", "uid"],
|
||||
)
|
||||
# Re-fetch to obtain instances we just created AND any
|
||||
# created concurrently by another scan against the same provider.
|
||||
existing_resources.update(
|
||||
{
|
||||
r.uid: r
|
||||
for r in Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_instance.id,
|
||||
uid__in=missing_uids,
|
||||
)
|
||||
}
|
||||
)
|
||||
for uid, r in existing_resources.items():
|
||||
resource_cache[uid] = r
|
||||
resource_failed_findings_cache.setdefault(uid, 0)
|
||||
|
||||
# 2) Pre-resolve ResourceTags in bulk
|
||||
batch_tag_kv: set[tuple[str, str]] = set()
|
||||
for f in non_null_findings:
|
||||
for k, v in f.resource_tags.items():
|
||||
if (k, v) not in tag_cache:
|
||||
batch_tag_kv.add((k, v))
|
||||
|
||||
if batch_tag_kv:
|
||||
keys_to_query = {k for k, _ in batch_tag_kv}
|
||||
existing_tags = {
|
||||
(t.key, t.value): t
|
||||
for t in ResourceTag.objects.filter(
|
||||
tenant_id=tenant_id, key__in=keys_to_query
|
||||
)
|
||||
if (t.key, t.value) in batch_tag_kv
|
||||
}
|
||||
missing_kv = batch_tag_kv - existing_tags.keys()
|
||||
if missing_kv:
|
||||
ResourceTag.objects.bulk_create(
|
||||
[
|
||||
ResourceTag(tenant_id=tenant_id, key=k, value=v)
|
||||
for k, v in missing_kv
|
||||
],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "key", "value"],
|
||||
)
|
||||
existing_tags.update(
|
||||
{
|
||||
(t.key, t.value): t
|
||||
for t in ResourceTag.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
key__in={k for k, _ in missing_kv},
|
||||
)
|
||||
if (t.key, t.value) in missing_kv
|
||||
}
|
||||
)
|
||||
tag_cache.update(existing_tags)
|
||||
|
||||
# 3) Per-finding in-memory processing
|
||||
for finding in non_null_findings:
|
||||
resource_uid = finding.resource_uid
|
||||
resource_instance = resource_cache.get(resource_uid)
|
||||
if resource_instance is None:
|
||||
# Should be unreachable after the pre-resolve step. Defensive log.
|
||||
logger.error(
|
||||
f"Resource {resource_uid} missing from cache after pre-resolve "
|
||||
f"on scan {scan_instance.id}; skipping finding."
|
||||
)
|
||||
continue
|
||||
|
||||
# Detect resource field changes (defer save until end-of-batch bulk_update).
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
updated = False
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
if resource_instance.service != finding.service_name:
|
||||
resource_instance.service = finding.service_name
|
||||
updated = True
|
||||
if resource_instance.type != finding.resource_type:
|
||||
resource_instance.type = finding.resource_type
|
||||
updated = True
|
||||
if resource_instance.metadata != finding.resource_metadata:
|
||||
resource_instance.metadata = json.dumps(
|
||||
finding.resource_metadata, cls=CustomEncoder
|
||||
)
|
||||
updated = True
|
||||
if resource_instance.details != finding.resource_details:
|
||||
resource_instance.details = finding.resource_details
|
||||
updated = True
|
||||
if resource_instance.partition != finding.partition:
|
||||
resource_instance.partition = finding.partition
|
||||
updated = True
|
||||
if group and (
|
||||
not resource_instance.groups
|
||||
or group not in resource_instance.groups
|
||||
):
|
||||
resource_instance.groups = (resource_instance.groups or []) + [
|
||||
group
|
||||
]
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
dirty_resources[resource_uid] = resource_instance
|
||||
|
||||
# Accumulate ResourceTagMapping rows; bulk_create at end of block.
|
||||
for k, v in finding.resource_tags.items():
|
||||
tag_instance = tag_cache.get((k, v))
|
||||
if tag_instance is None:
|
||||
# Should not happen after pre-resolve; skip defensively.
|
||||
continue
|
||||
tag_mappings_to_create.append(
|
||||
ResourceTagMapping(
|
||||
tenant_id=tenant_id,
|
||||
resource=resource_instance,
|
||||
tag=tag_instance,
|
||||
)
|
||||
)
|
||||
|
||||
unique_resources.add(
|
||||
(resource_instance.uid, resource_instance.region)
|
||||
)
|
||||
time.sleep(0.1 * (2**attempt))
|
||||
continue
|
||||
else:
|
||||
raise db_err
|
||||
|
||||
# Track resource field changes (defer save)
|
||||
updated = False
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
if resource_instance.service != finding.service_name:
|
||||
resource_instance.service = finding.service_name
|
||||
updated = True
|
||||
if resource_instance.type != finding.resource_type:
|
||||
resource_instance.type = finding.resource_type
|
||||
updated = True
|
||||
if resource_instance.metadata != finding.resource_metadata:
|
||||
resource_instance.metadata = json.dumps(
|
||||
finding.resource_metadata, cls=CustomEncoder
|
||||
)
|
||||
updated = True
|
||||
if resource_instance.details != finding.resource_details:
|
||||
resource_instance.details = finding.resource_details
|
||||
updated = True
|
||||
if resource_instance.partition != finding.partition:
|
||||
resource_instance.partition = finding.partition
|
||||
updated = True
|
||||
if group and (
|
||||
not resource_instance.groups or group not in resource_instance.groups
|
||||
):
|
||||
resource_instance.groups = (resource_instance.groups or []) + [group]
|
||||
updated = True
|
||||
# TEMPORARY WORKAROUND: Skip findings with UID > 300 chars
|
||||
# TODO: Remove this after implementing text field migration for finding.uid
|
||||
if len(finding.uid) > 300:
|
||||
logger.warning(
|
||||
f"Skipping finding with UID exceeding 300 characters. "
|
||||
f"Length: {len(finding.uid)}, "
|
||||
f"Check: {finding.check_id}, "
|
||||
f"Resource: {finding.resource_name}, "
|
||||
f"UID: {finding.uid}"
|
||||
)
|
||||
continue
|
||||
|
||||
if updated:
|
||||
dirty_resources[resource_uid] = resource_instance
|
||||
|
||||
# Process tags
|
||||
tags = []
|
||||
with rls_transaction(tenant_id):
|
||||
for key, value in finding.resource_tags.items():
|
||||
tag_key = (key, value)
|
||||
if tag_key not in tag_cache:
|
||||
tag_instance, _ = ResourceTag.objects.get_or_create(
|
||||
tenant_id=tenant_id, key=key, value=value
|
||||
finding_uid = finding.uid
|
||||
last_status, last_first_seen_at = last_status_cache.get(
|
||||
finding_uid, (None, None)
|
||||
)
|
||||
tag_cache[tag_key] = tag_instance
|
||||
else:
|
||||
tag_instance = tag_cache[tag_key]
|
||||
tags.append(tag_instance)
|
||||
resource_instance.upsert_or_delete_tags(tags=tags)
|
||||
|
||||
unique_resources.add((resource_instance.uid, resource_instance.region))
|
||||
status = FindingStatus[finding.status]
|
||||
delta = _create_finding_delta(last_status, status)
|
||||
|
||||
# Prepare finding data
|
||||
finding_uid = finding.uid
|
||||
if not last_first_seen_at:
|
||||
last_first_seen_at = datetime.now(tz=timezone.utc)
|
||||
|
||||
# TEMPORARY WORKAROUND: Skip findings with UID > 300 chars
|
||||
# TODO: Remove this after implementing text field migration for finding.uid
|
||||
if len(finding_uid) > 300:
|
||||
skipped_findings_count += 1
|
||||
logger.warning(
|
||||
f"Skipping finding with UID exceeding 300 characters. "
|
||||
f"Length: {len(finding_uid)}, "
|
||||
f"Check: {finding.check_id}, "
|
||||
f"Resource: {finding.resource_name}, "
|
||||
f"UID: {finding_uid}"
|
||||
)
|
||||
continue
|
||||
# Determine if finding should be muted and why
|
||||
# Priority: mutelist processor (highest) > manual mute rules
|
||||
is_muted = False
|
||||
muted_reason = None
|
||||
if finding.muted:
|
||||
is_muted = True
|
||||
muted_reason = "Muted by mutelist"
|
||||
elif finding_uid in mute_rules_cache:
|
||||
is_muted = True
|
||||
muted_reason = mute_rules_cache[finding_uid]
|
||||
|
||||
last_status, last_first_seen_at = last_status_cache.get(
|
||||
finding_uid, (None, None)
|
||||
)
|
||||
if status == FindingStatus.FAIL and not is_muted:
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
|
||||
status = FindingStatus[finding.status]
|
||||
delta = _create_finding_delta(last_status, status)
|
||||
check_metadata["compliance"] = finding.compliance
|
||||
finding_instance = Finding(
|
||||
tenant_id=tenant_id,
|
||||
uid=finding_uid,
|
||||
delta=delta,
|
||||
check_metadata=check_metadata,
|
||||
status=status,
|
||||
status_extended=finding.status_extended,
|
||||
severity=finding.severity,
|
||||
impact=finding.severity,
|
||||
raw_result=finding.raw,
|
||||
check_id=finding.check_id,
|
||||
scan=scan_instance,
|
||||
first_seen_at=last_first_seen_at,
|
||||
muted=is_muted,
|
||||
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||||
# Denormalized resource arrays populated directly on insert
|
||||
# (was previously a separate bulk_update; saves a CASE WHEN
|
||||
# over thousands of rows per micro-batch).
|
||||
resource_regions=[resource_instance.region]
|
||||
if resource_instance.region
|
||||
else [],
|
||||
resource_services=[resource_instance.service]
|
||||
if resource_instance.service
|
||||
else [],
|
||||
resource_types=[resource_instance.type]
|
||||
if resource_instance.type
|
||||
else [],
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append(
|
||||
(finding_instance, resource_instance)
|
||||
)
|
||||
|
||||
if not last_first_seen_at:
|
||||
last_first_seen_at = datetime.now(tz=timezone.utc)
|
||||
scan_resource_cache.add(
|
||||
(
|
||||
str(resource_instance.id),
|
||||
resource_instance.service,
|
||||
resource_instance.region,
|
||||
resource_instance.type,
|
||||
)
|
||||
)
|
||||
|
||||
# Determine if finding should be muted and why
|
||||
# Priority: mutelist processor (highest) > manual mute rules
|
||||
is_muted = False
|
||||
muted_reason = None
|
||||
aggregate_category_counts(
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
|
||||
# Check mutelist processor first (highest priority)
|
||||
if finding.muted:
|
||||
is_muted = True
|
||||
muted_reason = "Muted by mutelist"
|
||||
# If not muted by mutelist, check manual mute rules
|
||||
elif finding_uid in mute_rules_cache:
|
||||
is_muted = True
|
||||
muted_reason = mute_rules_cache[finding_uid]
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=check_metadata.get("resourcegroup") or None,
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
resource_uid=resource_instance.uid if resource_instance else "",
|
||||
cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Increment failed_findings_count cache if needed
|
||||
if status == FindingStatus.FAIL and not is_muted:
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
# 4) Bulk create ResourceTagMappings
|
||||
# Replaces the original per-resource `upsert_or_delete_tags`
|
||||
# (which did one `update_or_create` + SELECT FOR UPDATE per mapping).
|
||||
if tag_mappings_to_create:
|
||||
# Pre-SELECT existing pairs: `bulk_create(ignore_conflicts=True)`
|
||||
# does not populate `pk`, so we cannot tell new vs existing from
|
||||
# the result; we need that to bump `updated_at` only on resources
|
||||
# that actually gain a mapping.
|
||||
candidate_resource_ids = {
|
||||
m.resource_id for m in tag_mappings_to_create
|
||||
}
|
||||
candidate_tag_ids = {m.tag_id for m in tag_mappings_to_create}
|
||||
existing_pairs = set(
|
||||
ResourceTagMapping.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
resource_id__in=candidate_resource_ids,
|
||||
tag_id__in=candidate_tag_ids,
|
||||
).values_list("resource_id", "tag_id")
|
||||
)
|
||||
resource_uid_by_id = {
|
||||
str(r.id): uid for uid, r in resource_cache.items()
|
||||
}
|
||||
for m in tag_mappings_to_create:
|
||||
if (m.resource_id, m.tag_id) not in existing_pairs:
|
||||
uid = resource_uid_by_id.get(str(m.resource_id))
|
||||
if uid is not None:
|
||||
resources_with_new_tag_mappings.add(uid)
|
||||
|
||||
# Create finding object (don't save yet)
|
||||
check_metadata = finding.get_metadata()
|
||||
check_metadata["compliance"] = finding.compliance
|
||||
finding_instance = Finding(
|
||||
tenant_id=tenant_id,
|
||||
uid=finding_uid,
|
||||
delta=delta,
|
||||
check_metadata=check_metadata,
|
||||
status=status,
|
||||
status_extended=finding.status_extended,
|
||||
severity=finding.severity,
|
||||
impact=finding.severity,
|
||||
raw_result=finding.raw,
|
||||
check_id=finding.check_id,
|
||||
scan=scan_instance,
|
||||
first_seen_at=last_first_seen_at,
|
||||
muted=is_muted,
|
||||
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append((finding_instance, resource_instance))
|
||||
ResourceTagMapping.objects.bulk_create(
|
||||
tag_mappings_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "resource_id", "tag_id"],
|
||||
)
|
||||
|
||||
# Track for scan summary
|
||||
scan_resource_cache.add(
|
||||
(
|
||||
str(resource_instance.id),
|
||||
resource_instance.service,
|
||||
resource_instance.region,
|
||||
resource_instance.type,
|
||||
)
|
||||
)
|
||||
# 5) Bulk create Findings
|
||||
if findings_to_create:
|
||||
Finding.objects.bulk_create(
|
||||
findings_to_create, batch_size=SCAN_DB_BATCH_SIZE
|
||||
)
|
||||
|
||||
# Track categories with counts for ScanCategorySummary by (category, severity)
|
||||
aggregate_category_counts(
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
# 6) Bulk create ResourceFindingMapping rows
|
||||
mappings_to_create = [
|
||||
ResourceFindingMapping(
|
||||
tenant_id=tenant_id,
|
||||
resource=resource_instance,
|
||||
finding=finding_instance,
|
||||
)
|
||||
for finding_instance, resource_instance in resource_denormalized_data
|
||||
]
|
||||
if mappings_to_create:
|
||||
created_mappings = ResourceFindingMapping.objects.bulk_create(
|
||||
mappings_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "resource_id", "finding_id"],
|
||||
)
|
||||
inserted = sum(1 for m in created_mappings if m.pk)
|
||||
if inserted != len(mappings_to_create):
|
||||
logger.error(
|
||||
f"scan {scan_instance.id}: expected "
|
||||
f"{len(mappings_to_create)} ResourceFindingMapping rows, "
|
||||
f"inserted {inserted}. Rolling back micro-batch."
|
||||
)
|
||||
|
||||
# Track resource groups with counts for ScanGroupSummary
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=check_metadata.get("resourcegroup") or None,
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
resource_uid=resource_instance.uid if resource_instance else "",
|
||||
cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Bulk operations within single transaction
|
||||
with rls_transaction(tenant_id):
|
||||
# Bulk create findings
|
||||
if findings_to_create:
|
||||
Finding.objects.bulk_create(
|
||||
findings_to_create, batch_size=SCAN_DB_BATCH_SIZE
|
||||
)
|
||||
|
||||
# Bulk create resource-finding mappings
|
||||
for finding_instance, resource_instance in resource_denormalized_data:
|
||||
mappings_to_create.append(
|
||||
ResourceFindingMapping(
|
||||
tenant_id=tenant_id,
|
||||
resource=resource_instance,
|
||||
finding=finding_instance,
|
||||
# 7) Bulk update Resources
|
||||
# Union of:
|
||||
# - resources whose fields changed (dirty_resources)
|
||||
# - resources that got new tag mappings (need updated_at bump,
|
||||
# preserving the original `self.save(update_fields=["updated_at"])`
|
||||
# behavior of `upsert_or_delete_tags`)
|
||||
all_resource_uids_to_touch = (
|
||||
set(dirty_resources.keys()) | resources_with_new_tag_mappings
|
||||
)
|
||||
)
|
||||
|
||||
if mappings_to_create:
|
||||
created_mappings = ResourceFindingMapping.objects.bulk_create(
|
||||
mappings_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "resource_id", "finding_id"],
|
||||
)
|
||||
inserted = sum(1 for m in created_mappings if m.pk)
|
||||
if inserted != len(mappings_to_create):
|
||||
logger.error(
|
||||
f"scan {scan_instance.id}: expected "
|
||||
f"{len(mappings_to_create)} ResourceFindingMapping rows, "
|
||||
f"inserted {inserted}. Rolling back micro-batch."
|
||||
if all_resource_uids_to_touch:
|
||||
now_utc = datetime.now(tz=timezone.utc)
|
||||
resources_to_bulk_update = []
|
||||
for uid in all_resource_uids_to_touch:
|
||||
# Use the instance from dirty_resources if present (has mutated
|
||||
# fields), otherwise the cached one (for updated_at bump only).
|
||||
r = dirty_resources.get(uid) or resource_cache.get(uid)
|
||||
if r is None:
|
||||
continue
|
||||
# Manually bump updated_at since bulk_update bypasses auto_now.
|
||||
r.updated_at = now_utc
|
||||
resources_to_bulk_update.append(r)
|
||||
if resources_to_bulk_update:
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_bulk_update,
|
||||
[
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"region",
|
||||
"service",
|
||||
"type",
|
||||
"groups",
|
||||
"updated_at",
|
||||
],
|
||||
batch_size=1000,
|
||||
)
|
||||
# Successful execution: leave deadlock retry loop.
|
||||
break
|
||||
except (OperationalError, IntegrityError) as db_err:
|
||||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||||
logger.warning(
|
||||
f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} "
|
||||
f"on micro-batch for scan {scan_instance.id}. Retrying (attempt {attempt + 1})..."
|
||||
)
|
||||
|
||||
# Update finding denormalized arrays
|
||||
findings_to_update = []
|
||||
for finding_instance, resource_instance in resource_denormalized_data:
|
||||
if not finding_instance.resource_regions:
|
||||
finding_instance.resource_regions = []
|
||||
if not finding_instance.resource_services:
|
||||
finding_instance.resource_services = []
|
||||
if not finding_instance.resource_types:
|
||||
finding_instance.resource_types = []
|
||||
|
||||
if resource_instance.region not in finding_instance.resource_regions:
|
||||
finding_instance.resource_regions.append(resource_instance.region)
|
||||
if resource_instance.service not in finding_instance.resource_services:
|
||||
finding_instance.resource_services.append(resource_instance.service)
|
||||
if resource_instance.type not in finding_instance.resource_types:
|
||||
finding_instance.resource_types.append(resource_instance.type)
|
||||
|
||||
findings_to_update.append(finding_instance)
|
||||
|
||||
if findings_to_update:
|
||||
Finding.objects.bulk_update(
|
||||
findings_to_update,
|
||||
["resource_regions", "resource_services", "resource_types"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
|
||||
# Bulk update dirty resources
|
||||
if dirty_resources:
|
||||
update_objects_in_batches(
|
||||
tenant_id=tenant_id,
|
||||
model=Resource,
|
||||
objects=list(dirty_resources.values()),
|
||||
fields=[
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"region",
|
||||
"service",
|
||||
"type",
|
||||
"groups",
|
||||
],
|
||||
batch_size=1000,
|
||||
)
|
||||
time.sleep(0.1 * (2**attempt))
|
||||
# Clear accumulators that we appended to inside the failed transaction
|
||||
# so the retry produces consistent results.
|
||||
findings_to_create.clear()
|
||||
resource_denormalized_data.clear()
|
||||
tag_mappings_to_create.clear()
|
||||
dirty_resources.clear()
|
||||
resources_with_new_tag_mappings.clear()
|
||||
continue
|
||||
raise
|
||||
|
||||
# Log skipped findings summary
|
||||
if skipped_findings_count > 0:
|
||||
@@ -873,7 +1024,7 @@ def perform_prowler_scan(
|
||||
scan_instance = Scan.objects.get(pk=scan_id)
|
||||
scan_instance.state = StateChoices.EXECUTING
|
||||
scan_instance.started_at = datetime.now(tz=timezone.utc)
|
||||
scan_instance.save()
|
||||
scan_instance.save(update_fields=["state", "started_at", "updated_at"])
|
||||
|
||||
# Find the mutelist processor if it exists
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
@@ -918,7 +1069,13 @@ def perform_prowler_scan(
|
||||
provider_instance.connection_last_checked_at = datetime.now(
|
||||
tz=timezone.utc
|
||||
)
|
||||
provider_instance.save()
|
||||
provider_instance.save(
|
||||
update_fields=[
|
||||
"connected",
|
||||
"connection_last_checked_at",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
|
||||
# If the provider is not connected, raise an exception outside the transaction.
|
||||
# If raised within the transaction, the transaction will be rolled back and the provider will not be marked
|
||||
@@ -933,6 +1090,13 @@ def perform_prowler_scan(
|
||||
last_status_cache = {}
|
||||
resource_failed_findings_cache = defaultdict(int)
|
||||
|
||||
# Throttle scan_instance progress writes to avoid hammering the writer:
|
||||
# only persist when progress moves by at least `PROGRESS_THROTTLE_DELTA`
|
||||
# OR `PROGRESS_THROTTLE_SECONDS` have elapsed. The final progress (1.0)
|
||||
# always persists in the `finally` block below.
|
||||
last_persisted_progress = -1.0
|
||||
last_persisted_progress_at = 0.0
|
||||
|
||||
for progress, findings in prowler_scan.scan():
|
||||
# Process findings in micro-batches
|
||||
findings_list = list(findings)
|
||||
@@ -959,10 +1123,20 @@ def perform_prowler_scan(
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Update scan progress
|
||||
with rls_transaction(tenant_id):
|
||||
scan_instance.progress = progress
|
||||
scan_instance.save()
|
||||
# Throttled progress save (the final save in the `finally` block
|
||||
# below always runs regardless of throttle).
|
||||
now = time.time()
|
||||
progress_delta = progress - last_persisted_progress
|
||||
elapsed = now - last_persisted_progress_at
|
||||
if (
|
||||
progress_delta >= PROGRESS_THROTTLE_DELTA
|
||||
or elapsed >= PROGRESS_THROTTLE_SECONDS
|
||||
):
|
||||
with rls_transaction(tenant_id):
|
||||
scan_instance.progress = progress
|
||||
scan_instance.save(update_fields=["progress", "updated_at"])
|
||||
last_persisted_progress = progress
|
||||
last_persisted_progress_at = now
|
||||
|
||||
scan_instance.state = StateChoices.COMPLETED
|
||||
|
||||
@@ -976,13 +1150,16 @@ def perform_prowler_scan(
|
||||
resources_to_update.append(resource_instance)
|
||||
|
||||
if resources_to_update:
|
||||
update_objects_in_batches(
|
||||
tenant_id=tenant_id,
|
||||
model=Resource,
|
||||
objects=resources_to_update,
|
||||
fields=["failed_findings_count"],
|
||||
batch_size=1000,
|
||||
)
|
||||
# Single rls_transaction wrapping the bulk_update (previously
|
||||
# `update_objects_in_batches` opened one rls_transaction per
|
||||
# chunk; for tenants with many resources this collapsed N
|
||||
# BEGINs/COMMITs into 1).
|
||||
with rls_transaction(tenant_id):
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_update,
|
||||
["failed_findings_count"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error performing scan {scan_id}: {e}")
|
||||
@@ -994,7 +1171,16 @@ def perform_prowler_scan(
|
||||
scan_instance.duration = time.time() - start_time
|
||||
scan_instance.completed_at = datetime.now(tz=timezone.utc)
|
||||
scan_instance.unique_resource_count = len(unique_resources)
|
||||
scan_instance.save()
|
||||
scan_instance.save(
|
||||
update_fields=[
|
||||
"state",
|
||||
"duration",
|
||||
"completed_at",
|
||||
"unique_resource_count",
|
||||
"progress",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
|
||||
if exception is not None:
|
||||
raise exception
|
||||
@@ -1468,6 +1654,10 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
elif requirement_status == "PASS":
|
||||
requirement_statuses[key]["pass_count"] += 1
|
||||
|
||||
# Idempotent re-run: COPY can't ON CONFLICT, so clear this scan's rows first.
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete()
|
||||
|
||||
# Bulk create requirement records using PostgreSQL COPY
|
||||
_persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows)
|
||||
|
||||
|
||||
@@ -359,35 +359,40 @@ def _load_findings_for_requirement_checks(
|
||||
def _get_compliance_check_ids(compliance_obj) -> set[str]:
|
||||
"""Return the union of all check_ids referenced by a compliance framework.
|
||||
|
||||
Used by the master report orchestrator to know which checks each
|
||||
framework consumes from the shared ``findings_cache``, so that once a
|
||||
framework finishes the entries no other pending framework needs can be
|
||||
evicted from the cache (PROWLER-1733).
|
||||
Used by the master report orchestrator to evict entries from
|
||||
``findings_cache`` once no pending framework needs them (PROWLER-1733).
|
||||
|
||||
Args:
|
||||
compliance_obj: A loaded Compliance framework object exposing a
|
||||
``Requirements`` iterable, each requirement carrying ``Checks``.
|
||||
``None`` is treated as "no checks" rather than raising, so the
|
||||
caller can pass ``frameworks_bulk.get(...)`` directly without
|
||||
an extra existence check.
|
||||
|
||||
Returns:
|
||||
Set of check_id strings (empty if ``compliance_obj`` is ``None``).
|
||||
Accepts the legacy ``Compliance`` shape (``Requirements`` / ``Checks``
|
||||
lists) and the universal ``ComplianceFramework`` shape (``requirements``
|
||||
/ ``checks`` dict keyed by provider). ``None`` returns an empty set so
|
||||
callers can pass ``frameworks_bulk.get(...)`` directly.
|
||||
"""
|
||||
if compliance_obj is None:
|
||||
return set()
|
||||
checks: set[str] = set()
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or []
|
||||
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or getattr(
|
||||
compliance_obj, "requirements", None
|
||||
)
|
||||
if not requirements:
|
||||
return set()
|
||||
|
||||
check_ids: set[str] = set()
|
||||
try:
|
||||
# Defensive: Mock objects (used in unit tests) return another Mock
|
||||
# for any attribute access, which is truthy but not iterable. Treat
|
||||
# any non-iterable Requirements value as "no checks".
|
||||
for req in requirements:
|
||||
req_checks = getattr(req, "Checks", None) or []
|
||||
# Mock objects in unit tests return another Mock for any attribute
|
||||
# access — truthy but not iterable. Treat that as "no checks".
|
||||
for requirement in requirements:
|
||||
requirement_checks = getattr(requirement, "Checks", None)
|
||||
if requirement_checks is None:
|
||||
checks_by_provider = getattr(requirement, "checks", None) or {}
|
||||
requirement_checks = [
|
||||
check_id
|
||||
for check_ids_list in checks_by_provider.values()
|
||||
for check_id in check_ids_list
|
||||
]
|
||||
try:
|
||||
checks.update(req_checks)
|
||||
check_ids.update(requirement_checks)
|
||||
except TypeError:
|
||||
continue
|
||||
except TypeError:
|
||||
return set()
|
||||
return checks
|
||||
return check_ids
|
||||
|
||||
@@ -46,6 +46,7 @@ from tasks.jobs.lighthouse_providers import (
|
||||
refresh_lighthouse_provider_models,
|
||||
)
|
||||
from tasks.jobs.muting import mute_historical_findings
|
||||
from tasks.jobs.orphan_recovery import reconcile_orphans
|
||||
from tasks.jobs.report import (
|
||||
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
|
||||
_cleanup_stale_tmp_output_directories,
|
||||
@@ -67,7 +68,10 @@ from tasks.utils import (
|
||||
get_next_execution_datetime,
|
||||
)
|
||||
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.compliance import (
|
||||
get_compliance_frameworks,
|
||||
get_prowler_provider_compliance,
|
||||
)
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import delete_related_daily_task, rls_transaction
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
@@ -75,6 +79,9 @@ from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateC
|
||||
from api.utils import initialize_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.compliance.compliance import (
|
||||
process_universal_compliance_frameworks,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
@@ -253,7 +260,9 @@ def delete_provider_task(provider_id: str, tenant_id: str):
|
||||
return delete_provider(tenant_id=tenant_id, pk=provider_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-perform", queue="scans")
|
||||
# acks_late=False: a re-run would duplicate findings and the task is not auto-recovered,
|
||||
# so a crashed scan is dropped rather than redelivered by the broker (as before #11416).
|
||||
@shared_task(base=RLSTask, name="scan-perform", queue="scans", acks_late=False)
|
||||
@handle_provider_deletion
|
||||
def perform_scan_task(
|
||||
tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None
|
||||
@@ -297,7 +306,14 @@ def perform_scan_task(
|
||||
return result
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, bind=True, name="scan-perform-scheduled", queue="scans")
|
||||
# acks_late=False: like scan-perform; a dropped run is re-fired by Beat on the next tick.
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
bind=True,
|
||||
name="scan-perform-scheduled",
|
||||
queue="scans",
|
||||
acks_late=False,
|
||||
)
|
||||
@handle_provider_deletion
|
||||
def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
"""
|
||||
@@ -462,13 +478,42 @@ def cleanup_stale_attack_paths_scans_task():
|
||||
return cleanup_stale_attack_paths_scans()
|
||||
|
||||
|
||||
@shared_task(name="reconcile-orphan-tasks", queue="celery")
|
||||
def reconcile_orphan_tasks_task():
|
||||
"""Periodic watchdog: recover tasks whose worker is gone (deploys, crashes)."""
|
||||
return reconcile_orphans()
|
||||
|
||||
|
||||
@shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,))
|
||||
def delete_tenant_task(tenant_id: str):
|
||||
return delete_tenant(pk=tenant_id)
|
||||
|
||||
|
||||
def _scan_tmp_output_directory(tenant_id: str, scan_id: str) -> Path:
|
||||
"""Root tmp output directory for a scan ({tmp}/{tenant_id}/{scan_id})."""
|
||||
return Path(DJANGO_TMP_OUTPUT_DIRECTORY) / str(tenant_id) / str(scan_id)
|
||||
|
||||
|
||||
class ScanReportRLSTask(RLSTask):
|
||||
"""
|
||||
RLS task that removes the scan's tmp output directory when the task fails.
|
||||
|
||||
Covers failures both inside and outside the task body (e.g. ENOSPC mid-write,
|
||||
or setup errors) so partial artifacts do not accumulate on the worker disk.
|
||||
"""
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002
|
||||
del args # Required by Celery's Task.on_failure signature; not used.
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
scan_id = kwargs.get("scan_id")
|
||||
|
||||
if tenant_id and scan_id:
|
||||
logger.error(f"Scan report task {task_id} failed: {exc}")
|
||||
rmtree(_scan_tmp_output_directory(tenant_id, scan_id), ignore_errors=True)
|
||||
|
||||
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
base=ScanReportRLSTask,
|
||||
name="scan-report",
|
||||
queue="scan-reports",
|
||||
)
|
||||
@@ -513,11 +558,23 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
provider_uid = provider_obj.uid
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Per-framework exporters in `COMPLIANCE_CLASS_MAP` consume the legacy bulk.
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
# Universal-only frameworks (top-level JSONs like `dora.json`) are emitted
|
||||
# via `process_universal_compliance_frameworks` below.
|
||||
universal_bulk = get_prowler_provider_compliance(provider_type)
|
||||
universal_only_names = {
|
||||
name
|
||||
for name in universal_bulk
|
||||
if name not in frameworks_bulk and universal_bulk[name].outputs
|
||||
}
|
||||
frameworks_avail = get_compliance_frameworks(provider_type)
|
||||
out_dir, comp_dir = _generate_output_directory(
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
|
||||
)
|
||||
# Removed on success here and on failure by ScanReportRLSTask.on_failure,
|
||||
# so partial artifacts do not accumulate and fill the disk (ENOSPC).
|
||||
scan_tmp_dir = _scan_tmp_output_directory(tenant_id, scan_id)
|
||||
|
||||
def get_writer(writer_map, name, factory, is_last):
|
||||
"""
|
||||
@@ -535,6 +592,10 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
|
||||
output_writers = {}
|
||||
compliance_writers = {}
|
||||
# Shared across batches so universal writers are created once and reused.
|
||||
universal_compliance_state: dict[str, list] = {"compliance": []}
|
||||
universal_base_dir = os.path.dirname(out_dir)
|
||||
universal_output_filename = os.path.basename(out_dir)
|
||||
|
||||
scan_summary = FindingOutput._transform_findings_stats(
|
||||
ScanSummary.objects.filter(scan_id=scan_id)
|
||||
@@ -589,8 +650,30 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
writer.batch_write_data_to_file(**extra)
|
||||
writer._data.clear()
|
||||
|
||||
# Compliance CSVs
|
||||
# Universal-only frameworks (e.g. `dora.json`).
|
||||
if universal_only_names:
|
||||
process_universal_compliance_frameworks(
|
||||
input_compliance_frameworks=universal_only_names,
|
||||
universal_frameworks=universal_bulk,
|
||||
finding_outputs=fos,
|
||||
output_directory=universal_base_dir,
|
||||
output_filename=universal_output_filename,
|
||||
provider=provider_type,
|
||||
generated_outputs=universal_compliance_state,
|
||||
from_cli=False,
|
||||
is_last=is_last,
|
||||
)
|
||||
|
||||
# Compliance CSVs (per-framework exporters).
|
||||
for name in frameworks_avail:
|
||||
if name in universal_only_names:
|
||||
continue
|
||||
if name not in frameworks_bulk:
|
||||
logger.warning(
|
||||
"Compliance framework '%s' missing from bulk; skipping CSV export",
|
||||
name,
|
||||
)
|
||||
continue
|
||||
compliance_obj = frameworks_bulk[name]
|
||||
|
||||
klass = GenericCompliance
|
||||
@@ -666,7 +749,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
# TODO: We need to create a new periodic task to delete the output files
|
||||
# This task shouldn't be responsible for deleting the output files
|
||||
try:
|
||||
rmtree(Path(compressed).parent, ignore_errors=True)
|
||||
rmtree(scan_tmp_dir, ignore_errors=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting output files: {e}")
|
||||
final_location, did_upload = upload_uri, True
|
||||
@@ -1077,10 +1160,13 @@ def security_hub_integration_task(
|
||||
return upload_security_hub_integration(tenant_id, provider_id, scan_id)
|
||||
|
||||
|
||||
# acks_late=False: Jira sends are not deduplicated and the task is not auto-recovered,
|
||||
# so a crashed send is dropped rather than redelivered (avoids duplicate Jira issues).
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
name="integration-jira",
|
||||
queue="integrations",
|
||||
acks_late=False,
|
||||
)
|
||||
def jira_integration_task(
|
||||
tenant_id: str,
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from celery import states
|
||||
from django.test import override_settings
|
||||
from django_celery_results.models import TaskResult
|
||||
|
||||
from tasks.jobs.orphan_recovery import (
|
||||
_decode_celery_field,
|
||||
_reconcile_task_results,
|
||||
_recovery_attempt_count,
|
||||
advisory_lock,
|
||||
is_worker_alive,
|
||||
reconcile_orphans,
|
||||
reenqueueable_tasks,
|
||||
)
|
||||
|
||||
|
||||
def _orphan_result(*, name, kwargs, worker, created_minutes_ago, status=states.STARTED):
|
||||
"""Create a TaskResult mimicking an in-flight task, backdated past the grace."""
|
||||
tr = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
status=status,
|
||||
task_name=name,
|
||||
worker=worker,
|
||||
task_kwargs=repr(kwargs),
|
||||
task_args=repr([]),
|
||||
)
|
||||
TaskResult.objects.filter(pk=tr.pk).update(
|
||||
date_created=datetime.now(tz=timezone.utc)
|
||||
- timedelta(minutes=created_minutes_ago)
|
||||
)
|
||||
tr.refresh_from_db()
|
||||
return tr
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestDecodeCeleryField:
|
||||
def test_decodes_single_encoded_repr(self):
|
||||
assert _decode_celery_field("{'tenant_id': 'abc'}", {}) == {"tenant_id": "abc"}
|
||||
|
||||
def test_decodes_double_encoded(self):
|
||||
import json
|
||||
|
||||
stored = json.dumps(repr({"tenant_id": "abc", "scan_id": "s1"}))
|
||||
assert _decode_celery_field(stored, {}) == {"tenant_id": "abc", "scan_id": "s1"}
|
||||
|
||||
def test_empty_returns_default(self):
|
||||
assert _decode_celery_field(None, {}) == {}
|
||||
assert _decode_celery_field("", []) == []
|
||||
|
||||
def test_unparseable_raises(self):
|
||||
with pytest.raises(ValueError):
|
||||
_decode_celery_field("<<not a literal>>", {})
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestReconcileTaskResults:
|
||||
def _patches(self, alive):
|
||||
"""Patch worker liveness, revoke, and the task registry for re-enqueue."""
|
||||
mock_app = MagicMock()
|
||||
mock_task = MagicMock()
|
||||
mock_app.tasks.get.return_value = mock_task
|
||||
return (
|
||||
patch("tasks.jobs.orphan_recovery.is_worker_alive", return_value=alive),
|
||||
patch("tasks.jobs.orphan_recovery.revoke_task"),
|
||||
patch("tasks.jobs.orphan_recovery.current_app", mock_app),
|
||||
mock_task,
|
||||
)
|
||||
|
||||
def test_recovers_non_scan_task(self, tenants_fixture):
|
||||
"""A NON-scan task (tenant-deletion) left orphaned is re-enqueued too."""
|
||||
tenant = tenants_fixture[0]
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenant.id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["recovered"]
|
||||
tr.refresh_from_db()
|
||||
assert tr.status == states.REVOKED # stale result cleared (no pending alert)
|
||||
mock_task.apply_async.assert_called_once()
|
||||
call = mock_task.apply_async.call_args.kwargs
|
||||
assert call["kwargs"] == {"tenant_id": str(tenant.id)}
|
||||
assert call["task_id"] != tr.task_id # fresh task id
|
||||
|
||||
def test_external_integration_task_is_not_reenqueued_by_default(
|
||||
self, tenants_fixture
|
||||
):
|
||||
"""External side-effect tasks without proven idempotency stay terminal.
|
||||
|
||||
integration-s3 rebuilds its upload from worker-local files that do not
|
||||
survive the crash, so re-enqueuing it would upload nothing.
|
||||
"""
|
||||
tr = _orphan_result(
|
||||
name="integration-s3",
|
||||
kwargs={
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
"provider_id": str(uuid4()),
|
||||
"output_directory": "/tmp/gone",
|
||||
},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@override_settings(TASK_RECOVERY_SUMMARIES_ENABLED=False)
|
||||
def test_disabled_group_task_is_not_reenqueued(self, tenants_fixture):
|
||||
"""A task whose group feature flag is off stays terminal, not re-enqueued."""
|
||||
tr = _orphan_result(
|
||||
name="scan-summary",
|
||||
kwargs={
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
"scan_id": str(uuid4()),
|
||||
},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@override_settings(TASK_RECOVERY_SUMMARIES_ENABLED=False)
|
||||
def test_disabled_group_task_does_not_consume_recovery_attempt(
|
||||
self, tenants_fixture
|
||||
):
|
||||
"""A disabled-group task is failed without incrementing its Valkey attempt
|
||||
counter, so re-enabling the group does not start it at the cap."""
|
||||
tr = _orphan_result(
|
||||
name="scan-summary",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id), "scan_id": str(uuid4())},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count") as mock_count,
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_count.assert_not_called()
|
||||
|
||||
def test_scan_task_is_skipped_entirely(self, tenants_fixture):
|
||||
"""Scan tasks are excluded from recovery: the watchdog never touches them."""
|
||||
tr = _orphan_result(
|
||||
name="scan-perform",
|
||||
kwargs={
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
"scan_id": str(uuid4()),
|
||||
},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with p_alive, p_revoke, p_app:
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id not in result["recovered"]
|
||||
assert tr.task_id not in result["failed"]
|
||||
assert tr.task_id not in result["skipped"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_jira_integration_task_is_not_reenqueued(self, tenants_fixture):
|
||||
"""integration-jira stays terminal: re-running it would create duplicate Jira
|
||||
issues, so an orphaned send is failed instead of re-enqueued."""
|
||||
tenant = tenants_fixture[0]
|
||||
kwargs = {
|
||||
"tenant_id": str(tenant.id),
|
||||
"integration_id": str(uuid4()),
|
||||
"project_key": "PROWLER",
|
||||
"issue_type": "Task",
|
||||
"finding_ids": [str(uuid4()), str(uuid4())],
|
||||
}
|
||||
tr = _orphan_result(
|
||||
name="integration-jira",
|
||||
kwargs=kwargs,
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
tr.refresh_from_db()
|
||||
assert tr.status == states.REVOKED # stale result cleared (no pending alert)
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_skips_live_worker(self, tenants_fixture):
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="alive@host",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=True)
|
||||
with p_alive, p_revoke, p_app:
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["skipped"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_skips_recently_created(self, tenants_fixture):
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=0,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with p_alive, p_revoke, p_app:
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
# too recent: excluded by the grace window (not even a candidate)
|
||||
assert tr.task_id not in result["recovered"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_denylisted_task_failed_not_reenqueued(self, tenants_fixture):
|
||||
"""A non-allowlisted task is failed, never blind re-run."""
|
||||
tr = _orphan_result(
|
||||
name="some-non-idempotent-task",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=1),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
tr.refresh_from_db()
|
||||
assert tr.status == states.REVOKED
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
def test_recovery_cap_marks_failed(self, tenants_fixture):
|
||||
"""When the recovery counter exceeds the cap, the task is failed not re-run."""
|
||||
tr = _orphan_result(
|
||||
name="tenant-deletion",
|
||||
kwargs={"tenant_id": str(tenants_fixture[0].id)},
|
||||
worker="dead@gone",
|
||||
created_minutes_ago=60,
|
||||
)
|
||||
p_alive, p_revoke, p_app, mock_task = self._patches(alive=False)
|
||||
with (
|
||||
p_alive,
|
||||
p_revoke,
|
||||
p_app,
|
||||
patch("tasks.jobs.orphan_recovery._recovery_attempt_count", return_value=4),
|
||||
):
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=2, max_attempts=3, window_hours=6, dry_run=False
|
||||
)
|
||||
|
||||
assert tr.task_id in result["failed"]
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestOrphanRecoveryHelpers:
|
||||
def test_advisory_lock_acquires_and_releases(self):
|
||||
with advisory_lock() as acquired:
|
||||
assert acquired is True
|
||||
|
||||
def test_is_worker_alive_true_when_responds(self):
|
||||
inspect = MagicMock()
|
||||
inspect.ping.return_value = {"w@h": {"ok": "pong"}}
|
||||
with patch(
|
||||
"tasks.jobs.orphan_recovery.current_app.control.inspect",
|
||||
return_value=inspect,
|
||||
):
|
||||
assert is_worker_alive("w@h") is True
|
||||
|
||||
def test_is_worker_alive_false_when_silent(self):
|
||||
inspect = MagicMock()
|
||||
inspect.ping.return_value = None
|
||||
with patch(
|
||||
"tasks.jobs.orphan_recovery.current_app.control.inspect",
|
||||
return_value=inspect,
|
||||
):
|
||||
assert is_worker_alive("w@h") is False
|
||||
|
||||
def test_recovery_attempt_count_increments(self):
|
||||
# Unique signature so the Valkey counter starts fresh for this test.
|
||||
kwargs_repr = repr({"probe": str(uuid4())})
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.side_effect = [1, 2]
|
||||
with patch("redis.from_url", return_value=redis_client):
|
||||
assert _recovery_attempt_count("probe-task", kwargs_repr, 6) == 1
|
||||
assert _recovery_attempt_count("probe-task", kwargs_repr, 6) == 2
|
||||
|
||||
|
||||
class TestRecoveryFeatureFlags:
|
||||
def test_all_groups_enabled_by_default(self):
|
||||
tasks = reenqueueable_tasks()
|
||||
assert "scan-summary" in tasks
|
||||
assert {"provider-deletion", "tenant-deletion"} <= tasks
|
||||
|
||||
@override_settings(TASK_RECOVERY_SUMMARIES_ENABLED=False)
|
||||
def test_summaries_group_flag_excludes_summary_tasks(self):
|
||||
tasks = reenqueueable_tasks()
|
||||
assert "scan-summary" not in tasks
|
||||
assert "scan-compliance-overviews" not in tasks
|
||||
assert "provider-deletion" in tasks
|
||||
|
||||
@override_settings(TASK_RECOVERY_DELETIONS_ENABLED=False)
|
||||
def test_deletions_group_flag_excludes_deletion_tasks(self):
|
||||
tasks = reenqueueable_tasks()
|
||||
assert "provider-deletion" not in tasks
|
||||
assert "tenant-deletion" not in tasks
|
||||
assert "scan-summary" in tasks
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestRecoveryMasterFlag:
|
||||
@override_settings(TASK_RECOVERY_ENABLED=False)
|
||||
def test_master_flag_disables_task_recovery(self):
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.orphan_recovery._reconcile_task_results"
|
||||
) as mock_reconcile,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.cleanup.cleanup_stale_attack_paths_scans",
|
||||
return_value={},
|
||||
),
|
||||
):
|
||||
result = reconcile_orphans(grace_minutes=2, max_attempts=3, dry_run=False)
|
||||
|
||||
mock_reconcile.assert_not_called()
|
||||
assert result["acquired"] is True
|
||||
assert result["enabled"] is False
|
||||
|
||||
@override_settings(TASK_RECOVERY_ENABLED=True)
|
||||
def test_master_flag_enabled_runs_task_recovery(self):
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.orphan_recovery._reconcile_task_results",
|
||||
return_value={"recovered": [], "failed": [], "skipped": []},
|
||||
) as mock_reconcile,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.cleanup.cleanup_stale_attack_paths_scans",
|
||||
return_value={},
|
||||
),
|
||||
):
|
||||
reconcile_orphans(grace_minutes=2, max_attempts=3, dry_run=False)
|
||||
|
||||
mock_reconcile.assert_called_once()
|
||||
@@ -80,7 +80,7 @@ def basic_csa_compliance_data():
|
||||
tenant_id="tenant-123",
|
||||
scan_id="scan-456",
|
||||
provider_id="provider-789",
|
||||
compliance_id="csa_ccm_4.0_aws",
|
||||
compliance_id="csa_ccm_4.0",
|
||||
framework="CSA-CCM",
|
||||
name="CSA Cloud Controls Matrix v4.0",
|
||||
version="4.0",
|
||||
|
||||
@@ -1880,6 +1880,62 @@ class TestCreateComplianceRequirements:
|
||||
|
||||
assert "requirements_created" in result
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_create_compliance_requirements_idempotent_on_rerun(
|
||||
self,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
providers_fixture,
|
||||
findings_fixture,
|
||||
):
|
||||
"""Re-running compliance materialization must not raise nor duplicate rows.
|
||||
|
||||
Uses transaction=True because the COPY path commits on its own connection,
|
||||
so the test must use real commits (mirroring production) rather than the
|
||||
default rollback wrapper.
|
||||
"""
|
||||
from api.models import ComplianceRequirementOverview
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE"
|
||||
) as mock_compliance_template:
|
||||
tenant_id = str(tenants_fixture[0].id)
|
||||
scan_id = str(scans_fixture[0].id)
|
||||
|
||||
mock_compliance_template.__getitem__.return_value = {
|
||||
"test_compliance": {
|
||||
"framework": "Test Framework",
|
||||
"version": "1.0",
|
||||
"requirements": {
|
||||
"req_1": {
|
||||
"description": "Test Requirement 1",
|
||||
"checks": {"test_check_id": None},
|
||||
"checks_status": {
|
||||
"pass": 2,
|
||||
"fail": 1,
|
||||
"manual": 0,
|
||||
"total": 3,
|
||||
},
|
||||
"status": "FAIL",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
create_compliance_requirements(tenant_id, scan_id)
|
||||
count_after_first = ComplianceRequirementOverview.objects.filter(
|
||||
scan_id=scan_id
|
||||
).count()
|
||||
|
||||
# Second run must not raise and must not duplicate rows.
|
||||
create_compliance_requirements(tenant_id, scan_id)
|
||||
count_after_second = ComplianceRequirementOverview.objects.filter(
|
||||
scan_id=scan_id
|
||||
).count()
|
||||
|
||||
assert count_after_first > 0
|
||||
assert count_after_second == count_after_first
|
||||
|
||||
def test_create_compliance_requirements_kubernetes_provider(
|
||||
self,
|
||||
tenants_fixture,
|
||||
|
||||
@@ -15,8 +15,10 @@ from tasks.jobs.lighthouse_providers import (
|
||||
from tasks.tasks import (
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY,
|
||||
STALE_TMP_OUTPUT_MAX_AGE_HOURS,
|
||||
ScanReportRLSTask,
|
||||
_cleanup_orphan_scheduled_scans,
|
||||
_perform_scan_complete_tasks,
|
||||
_scan_tmp_output_directory,
|
||||
check_integrations_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
generate_outputs_task,
|
||||
@@ -321,6 +323,7 @@ class TestGenerateOutputs:
|
||||
|
||||
mock_transformed_stats = {"some": "stats"}
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch(
|
||||
"tasks.tasks.FindingOutput._transform_findings_stats",
|
||||
return_value=mock_transformed_stats,
|
||||
@@ -439,6 +442,7 @@ class TestGenerateOutputs:
|
||||
mock_provider.uid = "test-provider-uid"
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter,
|
||||
patch("tasks.tasks.Provider.objects.get", return_value=mock_provider),
|
||||
patch("tasks.tasks.initialize_prowler_provider"),
|
||||
@@ -594,6 +598,7 @@ class TestGenerateOutputs:
|
||||
]
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch("tasks.tasks.ScanSummary.objects.filter") as mock_summary,
|
||||
patch(
|
||||
"tasks.tasks.Provider.objects.get",
|
||||
@@ -668,6 +673,7 @@ class TestGenerateOutputs:
|
||||
mock_provider.uid = "test-provider-uid"
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.get_prowler_provider_compliance", return_value={}),
|
||||
patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter,
|
||||
patch("tasks.tasks.Provider.objects.get", return_value=mock_provider),
|
||||
patch("tasks.tasks.initialize_prowler_provider"),
|
||||
@@ -771,6 +777,38 @@ class TestGenerateOutputs:
|
||||
mock_s3_task.assert_called_once()
|
||||
|
||||
|
||||
class TestScanReportRLSTaskOnFailure:
|
||||
def test_on_failure_removes_scan_tmp_directory(self):
|
||||
task = ScanReportRLSTask()
|
||||
|
||||
with patch("tasks.tasks.rmtree") as mock_rmtree:
|
||||
task.on_failure(
|
||||
exc=OSError("No space left on device"),
|
||||
task_id="task-abc",
|
||||
args=(),
|
||||
kwargs={"tenant_id": "t-1", "scan_id": "s-1"},
|
||||
_einfo=None,
|
||||
)
|
||||
|
||||
mock_rmtree.assert_called_once_with(
|
||||
_scan_tmp_output_directory("t-1", "s-1"), ignore_errors=True
|
||||
)
|
||||
|
||||
def test_on_failure_skips_when_missing_kwargs(self):
|
||||
task = ScanReportRLSTask()
|
||||
|
||||
with patch("tasks.tasks.rmtree") as mock_rmtree:
|
||||
task.on_failure(
|
||||
exc=OSError("No space left on device"),
|
||||
task_id="task-abc",
|
||||
args=(),
|
||||
kwargs={},
|
||||
_einfo=None,
|
||||
)
|
||||
|
||||
mock_rmtree.assert_not_called()
|
||||
|
||||
|
||||
class TestScanCompleteTasks:
|
||||
@patch("tasks.tasks.aggregate_attack_surface_task.apply_async")
|
||||
@patch("tasks.tasks.chain")
|
||||
@@ -1079,6 +1117,7 @@ class TestCheckIntegrationsTask:
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
@patch("tasks.tasks.get_prowler_provider_compliance", return_value={})
|
||||
@patch("tasks.tasks.s3_integration_task")
|
||||
@patch("tasks.tasks.Integration.objects.filter")
|
||||
@patch("tasks.tasks.ScanSummary.objects.filter")
|
||||
@@ -1111,6 +1150,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_scan_summary,
|
||||
mock_integration_filter,
|
||||
mock_s3_task,
|
||||
mock_get_prowler_compliance,
|
||||
):
|
||||
"""Test that ASFF output is generated for AWS providers with SecurityHub integration."""
|
||||
# Setup
|
||||
@@ -1207,6 +1247,7 @@ class TestCheckIntegrationsTask:
|
||||
|
||||
assert result == {"upload": True}
|
||||
|
||||
@patch("tasks.tasks.get_prowler_provider_compliance", return_value={})
|
||||
@patch("tasks.tasks.s3_integration_task")
|
||||
@patch("tasks.tasks.Integration.objects.filter")
|
||||
@patch("tasks.tasks.ScanSummary.objects.filter")
|
||||
@@ -1239,6 +1280,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_scan_summary,
|
||||
mock_integration_filter,
|
||||
mock_s3_task,
|
||||
mock_get_prowler_compliance,
|
||||
):
|
||||
"""Test that ASFF output is NOT generated for AWS providers without SecurityHub integration."""
|
||||
# Setup
|
||||
@@ -1332,6 +1374,7 @@ class TestCheckIntegrationsTask:
|
||||
|
||||
assert result == {"upload": True}
|
||||
|
||||
@patch("tasks.tasks.get_prowler_provider_compliance", return_value={})
|
||||
@patch("tasks.tasks.ScanSummary.objects.filter")
|
||||
@patch("tasks.tasks.Provider.objects.get")
|
||||
@patch("tasks.tasks.initialize_prowler_provider")
|
||||
@@ -1360,6 +1403,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_initialize_provider,
|
||||
mock_provider_get,
|
||||
mock_scan_summary,
|
||||
mock_get_prowler_compliance,
|
||||
):
|
||||
"""Test that ASFF output is NOT generated for non-AWS providers (e.g., Azure, GCP)."""
|
||||
# Setup
|
||||
@@ -2672,3 +2716,36 @@ class TestReaggregateAllFindingGroupSummaries:
|
||||
assert result == {"scans_reaggregated": 0}
|
||||
mock_group.assert_not_called()
|
||||
mock_chain.assert_not_called()
|
||||
|
||||
|
||||
class TestTaskTimeLimits:
|
||||
"""The per-task limits in task_annotations must actually take effect.
|
||||
|
||||
Celery applies a "*" annotation after the per-task one, so a "*" entry would
|
||||
silently overwrite every specific limit and cap long scans at the default. The
|
||||
default is set as the global limit instead, and these per-task limits must win.
|
||||
"""
|
||||
|
||||
def test_long_running_tasks_exceed_the_default_limit(self):
|
||||
from config.celery import celery_app
|
||||
|
||||
default = celery_app.conf.task_time_limit
|
||||
for name in (
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"provider-deletion",
|
||||
"tenant-deletion",
|
||||
):
|
||||
assert celery_app.tasks[name].time_limit > default
|
||||
|
||||
def test_connection_checks_stay_below_the_default_limit(self):
|
||||
from config.celery import celery_app
|
||||
|
||||
default = celery_app.conf.task_time_limit
|
||||
for name in (
|
||||
"provider-connection-check",
|
||||
"integration-connection-check",
|
||||
"lighthouse-connection-check",
|
||||
"lighthouse-provider-connection-check",
|
||||
):
|
||||
assert celery_app.tasks[name].time_limit < default
|
||||
|
||||
Generated
+29
-24
@@ -163,7 +163,7 @@ constraints = [
|
||||
{ name = "drf-simple-apikey", specifier = "==2.2.1" },
|
||||
{ name = "drf-spectacular", specifier = "==0.27.2" },
|
||||
{ name = "drf-spectacular-jsonapi", specifier = "==0.5.1" },
|
||||
{ name = "dulwich", specifier = "==0.23.0" },
|
||||
{ name = "dulwich", specifier = "==1.2.5" },
|
||||
{ name = "duo-client", specifier = "==5.5.0" },
|
||||
{ name = "durationpy", specifier = "==0.10" },
|
||||
{ name = "email-validator", specifier = "==2.2.0" },
|
||||
@@ -291,7 +291,7 @@ constraints = [
|
||||
{ name = "pydantic-core", specifier = "==2.41.5" },
|
||||
{ name = "pygithub", specifier = "==2.8.0" },
|
||||
{ name = "pygments", specifier = "==2.20.0" },
|
||||
{ name = "pyjwt", specifier = "==2.12.1" },
|
||||
{ name = "pyjwt", specifier = "==2.13.0" },
|
||||
{ name = "pylint", specifier = "==3.2.5" },
|
||||
{ name = "pymsalruntime", specifier = "==0.18.1" },
|
||||
{ name = "pynacl", specifier = "==1.6.2" },
|
||||
@@ -374,8 +374,10 @@ constraints = [
|
||||
{ name = "zstd", specifier = "==1.5.7.3" },
|
||||
]
|
||||
overrides = [
|
||||
{ name = "dulwich", specifier = "==1.2.5" },
|
||||
{ name = "microsoft-kiota-abstractions", specifier = "==1.9.9" },
|
||||
{ name = "okta", specifier = "==3.4.2" },
|
||||
{ name = "pyjwt", extras = ["crypto"], specifier = "==2.13.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -393,7 +395,7 @@ version = "1.2.7"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
@@ -1074,7 +1076,7 @@ dependencies = [
|
||||
{ name = "pkginfo" },
|
||||
{ name = "psutil", marker = "sys_platform != 'cygwin'" },
|
||||
{ name = "py-deviceid" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "pyopenssl" },
|
||||
{ name = "requests", extra = ["socks"] },
|
||||
]
|
||||
@@ -2457,7 +2459,7 @@ source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "django" },
|
||||
{ name = "djangorestframework" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a8/27/2874a325c11112066139769f7794afae238a07ce6adf96259f08fd37a9d7/djangorestframework_simplejwt-5.5.1.tar.gz", hash = "sha256:e72c5572f51d7803021288e2057afcbd03f17fe11d484096f40a460abc76e87f", size = 101265, upload-time = "2025-07-21T16:52:25.026Z" }
|
||||
wheels = [
|
||||
@@ -2576,24 +2578,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "dulwich"
|
||||
version = "0.23.0"
|
||||
version = "1.2.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4b/ac/ba58cf420640c7bc77ae8e1b31e174d83c9117750c63cf9ea3b5e202e5c4/dulwich-0.23.0.tar.gz", hash = "sha256:0aa6c2489dd5e978b27e9b75983b7331a66c999f0efc54ebe37cab808ed322ae", size = 575116, upload-time = "2025-06-21T17:56:47.494Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7f/85/ceb8ecff5cdeee4ceeebb86b599476dee559041dacc6c2c50cc0d4711549/dulwich-1.2.5.tar.gz", hash = "sha256:0395b2c8924c3424bafe2d9c1edd5348cc4b21ce9c1d6655bf01f9a5c47164c8", size = 1253230, upload-time = "2026-05-28T22:27:55.17Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/11/f6bbba8583f69cf19ef4bd7f5fde1a6b5ccaf8b6951781cec8db247116f4/dulwich-0.23.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d68498fdda13ab00791b483daab3bcfe9f9721c037aa458695e6ad81640c57cc", size = 972658, upload-time = "2025-06-21T17:56:13.505Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/9d/2720e0ab58666378a33c752a61543f936cd6b06dfe5d84a2215ddc0914b0/dulwich-0.23.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cb7bb930b12471a1cfcea4b3d25a671dc0ad32573f0ad25684684298959a1527", size = 1049813, upload-time = "2025-06-21T17:56:14.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/f3/81d8075141dfcc0a0449c2093596e58d3e11444e3af54e819eca63b84dd0/dulwich-0.23.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2abbce32fd2bc7902bcc5f69b10bf22576810de21651baaa864b78fd7aec261", size = 1051639, upload-time = "2025-06-21T17:56:16.437Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/0d/c06ccb227b096aef5906142fe78b5c79f9070a0ea6152fc219941186d540/dulwich-0.23.0-cp311-cp311-win32.whl", hash = "sha256:9e3151f10ce2a9ff91bca64c74345217f53bdd947dc958032343822009832f7a", size = 642918, upload-time = "2025-06-21T17:56:18.373Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/1c/1e99aa34c9aead9e641b2d9934f0a3d00257f75027cf5cdecc8a1a6c18ae/dulwich-0.23.0-cp311-cp311-win_amd64.whl", hash = "sha256:3ae9f1d9dc92d4e9a3f89ba2c55221f7b6442c5dd93b3f6f539a3c9eb3f37bdd", size = 659010, upload-time = "2025-06-21T17:56:19.947Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/d7/1e6fba0235babe912e8467b036062e37d11672cbbeb0d8074f9d4559057b/dulwich-0.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:52cdef66a7994d29528ca79ca59452518bbba3fd56a9c61c61f6c467c1c7956e", size = 960292, upload-time = "2025-06-21T17:56:21.308Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/6a/23f0c487ec03f2752600cab4a8e0dedb38186246c475bf3fa90a8db830d5/dulwich-0.23.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d473888a6ab9ed5d4a4c3f053cbe5b77f72d54b6efdf5688fed76094316e571e", size = 1047892, upload-time = "2025-06-21T17:56:22.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/e2/8f3d216be5fd0ee1180d917b59b34b54b9896384cf139f319b5d3a8f16b4/dulwich-0.23.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:19fcf20224c641a61c774da92f098fbaae9938c7e17a52841e64092adf7e78f9", size = 1048699, upload-time = "2025-06-21T17:56:24.602Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/c4/18e6223cd4ad1ae9334eb4e6aa5952fd8f5c3d75762918eb90c209fec4ba/dulwich-0.23.0-cp312-cp312-win32.whl", hash = "sha256:7fc8b76b704ef35cd001e993e3aa4e1d666a2064bf467c07c560f12b2959dcaf", size = 641268, upload-time = "2025-06-21T17:56:26.18Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/9c/65bfbbac62d8a2967e13f6a1512371c5eb6b906a61fb6dead992669cad0e/dulwich-0.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:cb0566b888b578325350b4d67c61a0de35d417e9877560e3a6df88cae4576a59", size = 657837, upload-time = "2025-06-21T17:56:27.821Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/31/49318ee9db4b402e6d8b9b01bd4cae9298f59e1bb9bd56cf4a94e48fa069/dulwich-0.23.0-py3-none-any.whl", hash = "sha256:d8da6694ca332bb48775e35ee2215aa4673821164a91b83062f699c69f7cd135", size = 313776, upload-time = "2025-06-21T17:56:46.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/4a/654ae1671610fdf6b65a64586ad67ddd8550d4d08a632b2a4b9614754b6d/dulwich-1.2.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:556593fd11637f80f6018bee1916b1a84f5b420423b470ebb3f1a782ad6ef081", size = 1399277, upload-time = "2026-05-28T22:27:00.801Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/d8/06ee3bc8eded4bd7adf8adf0c9ea5f19bf96f7e5e626bfaf7311cde4208a/dulwich-1.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a70477c991e96cfe8fdd7c866e7251faf71b38bfeb51d6f27554c9cce1caabf3", size = 1382310, upload-time = "2026-05-28T22:27:02.216Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/17/a03adf50b9095f9f5d863393f21d585dea39bdc4fdf60788ff3a9407a512/dulwich-1.2.5-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9008ef25cabd379cda4fa86000fc38ca14b72afe17db798a8c85c0b2b7ce4d1e", size = 1470993, upload-time = "2026-05-28T22:27:04.075Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/58/1dc352d2a5e80befe4338af7208febb44bcfd7496b0dde5ac6dacb07b031/dulwich-1.2.5-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a5549f4afc973e0a15ea6b0244d57f848d3f3ee13dac557eb311024aebebf128", size = 1497820, upload-time = "2026-05-28T22:27:05.549Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/a8/e058959a87e7df7753b112ef66a43ccbc57338c1bbdc23a0edf3833396df/dulwich-1.2.5-cp311-cp311-win32.whl", hash = "sha256:5108acead814d1de8b6262d6d8fb90af7e82f5a4d83788b6b48e39d01800a92f", size = 1066549, upload-time = "2026-05-28T22:27:06.832Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/91/ff0b444f686718635348986bd73dfce42e947912417893de35de399b878b/dulwich-1.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:5e067b7feceb7034bc99e7c7143a704f1d97d4be7027d9a0aa5a83c0657ff091", size = 1079481, upload-time = "2026-05-28T22:27:08.33Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/22/4f75770bbe5521cac61c4820ef46d4fbf8c2175d3519ba3d0378d4ba798e/dulwich-1.2.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:701a9ecf7a8a44f5e2459e46befa93530cf36a8b1ae3140aefc007db1d7d0207", size = 1396522, upload-time = "2026-05-28T22:27:09.997Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/b1/c07c347681c0cf6acd4b189bf6e8d6207c71a1347b7a1e865eb40faa46b9/dulwich-1.2.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f90d68bfa97c4ca71de7507984365aefe27b6d248cb28dc99644d0f3ae8c60b", size = 1334826, upload-time = "2026-05-28T22:27:11.582Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/80/6818eb7ce492e18ab2efa92ab901d173b4b0b159e5681c1424f329600c40/dulwich-1.2.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:00b54a1d56ddbacdd8eadd6d4787a51b3a05fefa30eadbf9165fd283a00b90ed", size = 1416616, upload-time = "2026-05-28T22:27:13.195Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/a7/9790e60d19870f6554f7583722bb324c1355784316f20aeda1c0b5b1491a/dulwich-1.2.5-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d8f7ea8f47e38e5b0de3fab97e07e9c9161ffddc90b3964512cab2b7749df4e6", size = 1441354, upload-time = "2026-05-28T22:27:14.683Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/44/0ea8a69c24aa1254ff5996d682eae2eab287d471b937dcdb26d9ea9720b4/dulwich-1.2.5-cp312-cp312-win32.whl", hash = "sha256:8929134acf4ff967203df7600b38535f9b5b590462067a7e30dbce01acb97af9", size = 1017058, upload-time = "2026-05-28T22:27:16.121Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/9c/2fcddda7faec3bae52db7c64bfcb5dc756f597f33fae90e8d4e4b4d3b39b/dulwich-1.2.5-cp312-cp312-win_amd64.whl", hash = "sha256:9693d2c9e226b2ea855c1dc3a87e2f4d972f7523fc0f7924e5997e9f4c23d97f", size = 1031731, upload-time = "2026-05-28T22:27:17.633Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/4b/4a18a59ad230581cd0ef460e96001f90762e566dc2dfdba22aa358eb5a0e/dulwich-1.2.5-py3-none-any.whl", hash = "sha256:1679b376433a0fc7f36586afda1d4ed7427afa7a79d4bf17e5014474eea69fa4", size = 686745, upload-time = "2026-05-28T22:27:53.695Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4031,7 +4036,7 @@ dependencies = [
|
||||
{ name = "pycryptodomex" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydash" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
@@ -4494,7 +4499,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.29.0"
|
||||
version = "1.31.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
@@ -4873,11 +4878,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.12.1"
|
||||
version = "2.13.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -5785,7 +5790,7 @@ source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3c/2f/99fb8718274116c5c146c745755620fd5c5943f78ca52ca9b17e94348286/workos-6.0.4.tar.gz", hash = "sha256:b0bfe8fd212b8567422c4ea3732eb33608794033eb3a69900c6b04db183c32d6", size = 172217, upload-time = "2026-04-16T03:09:28.583Z" }
|
||||
wheels = [
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import warnings
|
||||
|
||||
from dashboard.common_methods import get_section_containers_3_levels
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def get_table(data):
|
||||
aux = data[
|
||||
[
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"NAME",
|
||||
"CHECKID",
|
||||
"STATUS",
|
||||
"REGION",
|
||||
"ACCOUNTID",
|
||||
"RESOURCEID",
|
||||
]
|
||||
]
|
||||
|
||||
return get_section_containers_3_levels(
|
||||
aux,
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"NAME",
|
||||
)
|
||||
@@ -139,6 +139,8 @@ services:
|
||||
|
||||
worker-dev:
|
||||
image: prowler-api-dev
|
||||
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
|
||||
stop_grace_period: 120s
|
||||
build:
|
||||
context: ./api
|
||||
dockerfile: Dockerfile
|
||||
|
||||
@@ -129,6 +129,8 @@ services:
|
||||
|
||||
worker:
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
# Give Celery soft shutdown time to drain/re-queue in-flight tasks on stop.
|
||||
stop_grace_period: 120s
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
|
||||
+2
-2
@@ -134,7 +134,7 @@ Example 1 is vague and even potentially ambiguous. Verbs state your purpose and
|
||||
|
||||
Explicit use of second-person pronouns (you) and possessives (your) should be minimized whenever possible. Those constructions are best reserved for cases when instructions are directly given in an imperative form:
|
||||
|
||||
**Example of Improvement Through Avoiding Second Person Pronouns**
|
||||
### Example of Improvement Through Avoiding Second Person Pronouns
|
||||
|
||||
**Original:**
|
||||
Prowler App can be installed in different ways, depending on your environment:
|
||||
@@ -236,7 +236,7 @@ The use of bullet points is highly recommended when:
|
||||
* Information can be logically divided into multiple categories, each sharing characteristics, features, or other relevant classifications.
|
||||
* Items are significant enough as standalone concepts to deserve their own bullet point.
|
||||
|
||||
**Example of Improvement Through Bullet Points**
|
||||
#### Example of Improvement Through Bullet Points
|
||||
|
||||
**Original:**
|
||||
It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMS, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme), and your custom security frameworks.
|
||||
|
||||
@@ -8,7 +8,77 @@ This guide explains the AI Skills system that provides on-demand context and pat
|
||||
**What are AI Skills?** Skills are structured instructions that help AI agents (Claude Code, Cursor, Copilot, etc.) understand Prowler's conventions, patterns, and best practices.
|
||||
</Info>
|
||||
|
||||
## Architecture Overview
|
||||
Skills live in the [`skills/`](https://github.com/prowler-cloud/prowler/tree/master/skills) directory of the Prowler OSS repository. Each skill is a folder containing a `SKILL.md` file with its patterns and metadata.
|
||||
|
||||
## Installation
|
||||
|
||||
To enable skills for the supported AI coding assistants, run the setup script from the repository root:
|
||||
|
||||
```bash
|
||||
./skills/setup.sh
|
||||
```
|
||||
|
||||
The script creates symlinks so each tool finds the skills in its expected location:
|
||||
|
||||
| Tool | Created by setup |
|
||||
|------|------------------|
|
||||
| Claude Code | `.claude/skills/` symlink and `CLAUDE.md` |
|
||||
| Gemini CLI | `.gemini/skills/` symlink and `GEMINI.md` |
|
||||
| Codex (OpenAI) | `.codex/skills/` symlink (uses `AGENTS.md` natively) |
|
||||
| GitHub Copilot | `.github/copilot-instructions.md` symlink to `AGENTS.md` |
|
||||
|
||||
After running the setup, restart the AI coding assistant to load the skills.
|
||||
|
||||
## Using Skills
|
||||
|
||||
AI agents discover skills automatically and load them when a request matches a skill trigger. To load a skill manually during a session, point the agent to the skill's `SKILL.md` file:
|
||||
|
||||
```text
|
||||
Read skills/{skill-name}/SKILL.md
|
||||
```
|
||||
|
||||
For the full list of available skills, their triggers, and the Auto-invoke mappings, see the [`skills/README.md`](https://github.com/prowler-cloud/prowler/blob/master/skills/README.md) and [`AGENTS.md`](https://github.com/prowler-cloud/prowler/blob/master/AGENTS.md) in the repository.
|
||||
|
||||
## Available Skills
|
||||
|
||||
| Type | Skills |
|
||||
|------|--------|
|
||||
| **Generic** | typescript, react-19, nextjs-16, tailwind-4, pytest, playwright, django-drf, zod-4, zustand-5, ai-sdk-5, vitest, tdd |
|
||||
| **Prowler** | prowler, prowler-sdk-check, prowler-api, prowler-ui, prowler-mcp, prowler-provider, prowler-compliance, prowler-compliance-review, prowler-docs, prowler-pr, prowler-ci, prowler-attack-paths-query |
|
||||
| **Testing** | prowler-test-sdk, prowler-test-api, prowler-test-ui |
|
||||
| **Meta** | skill-creator, skill-sync |
|
||||
|
||||
<Note>
|
||||
This table is a snapshot. The repository is the source of truth: see [`skills/README.md`](https://github.com/prowler-cloud/prowler/blob/master/skills/README.md) for the current, complete list.
|
||||
</Note>
|
||||
|
||||
## Skill Structure
|
||||
|
||||
Each skill follows the [Agent Skills spec](https://agentskills.io):
|
||||
|
||||
```text
|
||||
skills/{skill-name}/
|
||||
├── SKILL.md # Patterns, rules, decision trees
|
||||
├── assets/ # Code templates, schemas
|
||||
└── references/ # Links to local docs (single source of truth)
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Self-contained skills** - Critical patterns inline for fast loading
|
||||
2. **Local doc references** - No web URLs, points to `docs/developer-guide/*.mdx`
|
||||
3. **Single source of truth** - Skills reference docs, no duplication
|
||||
4. **On-demand loading** - AI loads only what's needed for the task
|
||||
|
||||
## Creating New Skills
|
||||
|
||||
Use the `skill-creator` meta-skill to create new skills that follow the Agent Skills spec. See [`AGENTS.md`](https://github.com/prowler-cloud/prowler/blob/master/AGENTS.md) for the full list of available skills and their triggers.
|
||||
|
||||
## How Skills Work
|
||||
|
||||
The diagrams below explain the internals of the skill system. They are useful for understanding the design, but are not required to install or use skills.
|
||||
|
||||
### Architecture Overview
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
@@ -28,7 +98,7 @@ graph LR
|
||||
style F fill:#1a4d2e,stroke:#66bb6a,color:#fff
|
||||
```
|
||||
|
||||
## How It Works
|
||||
### Request Lifecycle
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
@@ -68,7 +138,7 @@ sequenceDiagram
|
||||
A->>U: Creates check with correct patterns
|
||||
```
|
||||
|
||||
## Before vs After
|
||||
### With and Without Skills
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
@@ -96,7 +166,7 @@ graph TD
|
||||
style AFTER fill:#1a4d1a,stroke:#66bb6a,color:#fff
|
||||
```
|
||||
|
||||
## Complete Architecture
|
||||
### Full Component Map
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
@@ -110,7 +180,7 @@ flowchart TB
|
||||
subgraph GENERIC["Generic Skills"]
|
||||
G1["typescript"]
|
||||
G2["react-19"]
|
||||
G3["nextjs-15"]
|
||||
G3["nextjs-16"]
|
||||
G4["tailwind-4"]
|
||||
G5["pytest"]
|
||||
G6["playwright"]
|
||||
@@ -186,34 +256,3 @@ flowchart TB
|
||||
style STRUCTURE fill:#5c3d1a,stroke:#ffb74d,color:#fff
|
||||
style DOCS fill:#1a3d4d,stroke:#4dd0e1,color:#fff
|
||||
```
|
||||
|
||||
## Skills Included
|
||||
|
||||
| Type | Skills |
|
||||
|------|--------|
|
||||
| **Generic** | typescript, react-19, nextjs-15, tailwind-4, pytest, playwright, django-drf, zod-4, zustand-5, ai-sdk-5 |
|
||||
| **Prowler** | prowler, prowler-sdk-check, prowler-api, prowler-ui, prowler-mcp, prowler-provider, prowler-compliance, prowler-compliance-review, prowler-docs, prowler-pr, prowler-ci |
|
||||
| **Testing** | prowler-test-sdk, prowler-test-api, prowler-test-ui |
|
||||
| **Meta** | skill-creator, skill-sync |
|
||||
|
||||
## Skill Structure
|
||||
|
||||
Each skill follows the [Agent Skills spec](https://agentskills.io):
|
||||
|
||||
```
|
||||
skills/{skill-name}/
|
||||
├── SKILL.md # Patterns, rules, decision trees
|
||||
├── assets/ # Code templates, schemas
|
||||
└── references/ # Links to local docs (single source of truth)
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Self-contained skills** - Critical patterns inline for fast loading
|
||||
2. **Local doc references** - No web URLs, points to `docs/developer-guide/*.mdx`
|
||||
3. **Single source of truth** - Skills reference docs, no duplication
|
||||
4. **On-demand loading** - AI loads only what's needed for the task
|
||||
|
||||
## Creating New Skills
|
||||
|
||||
Use the `skill-creator` meta-skill to create new skills that follow the Agent Skills spec. See `AGENTS.md` for the full list of available skills and their triggers.
|
||||
|
||||
@@ -2,40 +2,228 @@
|
||||
title: 'Creating a New Security Compliance Framework in Prowler'
|
||||
---
|
||||
|
||||
This guide explains how to add a new security compliance framework to Prowler, end to end. It covers directory layout, the JSON schema, check mapping conventions, the Pydantic models that validate each framework, the CSV output formatter, local validation, testing, and the pull request process.
|
||||
This guide explains how to add a new security compliance framework to Prowler, end to end. It covers directory layout, the two supported JSON schemas (universal and legacy), the Pydantic models that validate each framework, check mapping conventions, output formatting, local validation, testing, and the pull request process.
|
||||
|
||||
## Introduction
|
||||
|
||||
A compliance framework in Prowler maps a public or custom control catalog (for example CIS, NIST 800-53, PCI DSS, HIPAA, ENS, CCC) to the security checks that Prowler already runs. Each requirement links to zero, one or more Prowler checks. When a scan executes, findings are aggregated per requirement to produce the compliance report rendered by Prowler CLI and Prowler Cloud.
|
||||
A compliance framework in Prowler maps a public or custom control catalog (for example CIS, NIST 800-53, PCI DSS, HIPAA, ENS, CCC, DORA) to the security checks that Prowler already runs. Each requirement links to zero, one or more Prowler checks. When a scan executes, findings are aggregated per requirement to produce the compliance report rendered by Prowler CLI and Prowler Cloud.
|
||||
|
||||
Prowler ships with 85+ compliance frameworks across All Providers. The catalog lives under `prowler/compliance/<provider>/` (or `prowler/compliance/` for universal compliance frameworks)
|
||||
Prowler ships 85+ compliance frameworks across all providers. The catalog lives under `prowler/compliance/<provider>/` (legacy, per-provider) or `prowler/compliance/` (universal, multi-provider).
|
||||
|
||||
<Warning>
|
||||
A compliance framework must represent the **complete state** of the source catalog. Every requirement defined by the framework has to be present in the JSON file, even when none of the existing Prowler checks can automate it. In that case, leave `Checks` as an empty array, but do not omit the requirement.
|
||||
A compliance framework must represent the **complete state** of the source catalog. Every requirement defined by the framework has to be present in the JSON file, even when no Prowler check can automate it. In that case, leave the requirement's check list empty, but do not omit the requirement.
|
||||
|
||||
Requirement coverage feeds the compliance percentage calculations and the metadata surfaces (dashboards, widgets, exports). Missing requirements skew those metrics and break the report as a faithful snapshot of the framework.
|
||||
</Warning>
|
||||
|
||||
### Two supported schemas
|
||||
|
||||
| Schema | When to use | File location | Discovered as |
|
||||
| --- | --- | --- | --- |
|
||||
| **Universal (recommended for new frameworks)** | Multi-provider frameworks, or single-provider frameworks that benefit from declarative table/PDF rendering | `prowler/compliance/<framework>.json` (top-level) | Available for **every** provider whose key appears in any `requirement.checks` dict |
|
||||
| **Legacy provider-specific** | Single-provider frameworks with framework-specific attribute classes already declared in the codebase (CIS, ENS, ISO 27001, etc.) | `prowler/compliance/<provider>/<framework>_<version>_<provider>.json` | Available only under that provider |
|
||||
|
||||
Auto-discovery happens in `get_bulk_compliance_frameworks_universal(provider)` (`prowler/lib/check/compliance_models.py:915`), which scans **both** the top-level `prowler/compliance/` directory and every per-provider sub-directory. Legacy frameworks are transparently converted to the universal `ComplianceFramework` model via `adapt_legacy_to_universal()` before being returned, so the rest of Prowler — CLI table rendering, CSV/OCSF outputs, PDF generation — works the same regardless of the source schema.
|
||||
|
||||
> The legacy entry-point `Compliance.get_bulk(provider)` (used by older code paths) only scans per-provider sub-directories. Universal top-level files are picked up exclusively via the universal loader; this matters if you are wiring a new code path against the legacy API.
|
||||
|
||||
For **new** frameworks, prefer the universal schema: it requires no Python code changes, supports multiple providers in a single file, and table/PDF rendering is driven entirely from declarative configuration inside the JSON.
|
||||
|
||||
> All Pydantic models in `compliance_models.py` are imported from `pydantic.v1`. Subclasses you add for the legacy schema must use `from pydantic.v1 import BaseModel`.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Before adding a new framework, complete the following checks:
|
||||
|
||||
- **Verify the framework is not already supported.** Inspect `prowler/compliance/<provider>/` for an existing JSON file matching the name and version.
|
||||
- **Verify the framework is not already supported.** Inspect `prowler/compliance/` and every `prowler/compliance/<provider>/` for an existing JSON file matching the name and version.
|
||||
- **Confirm the required checks exist.** Every requirement that can be automated must point to one or more existing Prowler checks. For each missing check, implement it first by following the [Prowler Checks](/developer-guide/checks) guide.
|
||||
- **Review a reference framework.** Use an existing framework with a similar structure as your template. `cis_2.0_aws.json` is the canonical reference for CIS-style frameworks. `ccc_aws.json`, `ens_rd2022_aws.json`, and `nist_800_53_revision_5_aws.json` illustrate other attribute shapes.
|
||||
- **Review a reference framework.** Use an existing framework with a similar structure as your template:
|
||||
- Universal: `prowler/compliance/dora.json`, `prowler/compliance/csa_ccm_4.0.json`.
|
||||
- Legacy: `prowler/compliance/aws/cis_2.0_aws.json` (canonical CIS shape), `prowler/compliance/aws/ccc_aws.json`, `prowler/compliance/aws/ens_rd2022_aws.json`, `prowler/compliance/aws/nist_800_53_revision_5_aws.json`.
|
||||
|
||||
## Four-Layer Architecture
|
||||
## Universal Compliance Framework
|
||||
|
||||
A compliance framework spans four layers. A complete contribution must touch each layer that applies.
|
||||
### Where the file lives
|
||||
|
||||
- **Layer 1 – Schema validation:** The Pydantic models in `prowler/lib/check/compliance_models.py` define the canonical schema for each attribute shape (CIS, ENS, Mitre, CCC, C5, CSA CCM, ISO 27001, KISA ISMS-P, AWS Well-Architected, Prowler ThreatScore, and a generic fallback).
|
||||
- **Layer 2 – JSON catalog:** The framework JSON file in `prowler/compliance/<provider>/` lists every requirement and maps it to checks.
|
||||
- **Layer 3 – Output formatter:** The Python module in `prowler/lib/outputs/compliance/<framework>/` builds the CSV row model, the per-provider transformer, and the CLI summary table.
|
||||
- **Layer 4 – Output dispatchers:** The dispatchers in `prowler/lib/outputs/compliance/compliance.py` and `prowler/lib/outputs/compliance/compliance_output.py` route findings to the right formatter based on the framework identifier.
|
||||
Place the file at the top level of the compliance directory:
|
||||
|
||||
The rest of this guide walks each layer in order.
|
||||
```
|
||||
prowler/compliance/<framework_name>.json
|
||||
```
|
||||
|
||||
## Directory Structure and File Naming
|
||||
Examples in the repository: `prowler/compliance/csa_ccm_4.0.json`, `prowler/compliance/dora.json`.
|
||||
|
||||
The file is auto-discovered — there is **no** need to register it in any `__init__.py`, modify `prowler/lib/outputs/`, or update any other Python module. The framework key Prowler CLI accepts via `--compliance` is the basename of the JSON file without `.json` (`dora.json` → `dora`).
|
||||
|
||||
### Top-level structure
|
||||
|
||||
```json
|
||||
{
|
||||
"framework": "<short identifier, e.g. \"DORA\" or \"CSA-CCM\">",
|
||||
"name": "<human-readable full name>",
|
||||
"version": "<framework version>",
|
||||
"description": "<one-paragraph description shown in --list-compliance and PDF reports>",
|
||||
"icon": "<short icon slug, optional>",
|
||||
"attributes_metadata": [ /* see below */ ],
|
||||
"outputs": { /* see below — optional */ },
|
||||
"requirements": [ /* see below */ ]
|
||||
}
|
||||
```
|
||||
|
||||
A `provider` field at the top level is **optional**. The framework's effective provider list is derived by `ComplianceFramework.get_providers()` (`compliance_models.py:739`) from the union of all keys appearing in `requirement.checks` across all requirements; the explicit `provider` field is used **only as a fallback** when no requirement carries any `checks` key. This is what enables a single file (e.g. `dora.json`) to cover AWS today and add Azure / GCP / etc. tomorrow without restructuring.
|
||||
|
||||
Provider keys inside `requirement.checks` must match the directory names under `prowler/providers/`. The valid keys at present are: `aws`, `azure`, `gcp`, `m365`, `kubernetes`, `iac`, `github`, `googleworkspace`, `alibabacloud`, `cloudflare`, `mongodbatlas`, `nhn`, `openstack`, `oraclecloud`, `llm`. Comparison in `supports_provider()` is case-insensitive, but lowercase is the convention used everywhere in the repository.
|
||||
|
||||
### `attributes_metadata`
|
||||
|
||||
Declares the shape of the per-requirement `attributes` dict. When this field is present, the root validator `validate_attributes_against_metadata` (`compliance_models.py:669`) enforces the schema at load time and rejects:
|
||||
|
||||
- Missing keys marked `required: true`.
|
||||
- Keys present in `attributes` but not declared in `attributes_metadata` (typo / drift guard).
|
||||
- Values that violate a declared `enum`.
|
||||
- Values whose Python type does not match a declared `int`, `float` or `bool`.
|
||||
|
||||
The runtime type check **only** covers `int`, `float` and `bool`. For `str`, `list_str` and `list_dict` the type is documentation-only — non-conforming values won't fail validation. If `attributes_metadata` is omitted, **no per-requirement validation runs at all**.
|
||||
|
||||
```json
|
||||
"attributes_metadata": [
|
||||
{
|
||||
"key": "Pillar",
|
||||
"label": "Pillar",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"enum": [
|
||||
"ICT Risk Management",
|
||||
"ICT-Related Incident Reporting",
|
||||
"Digital Operational Resilience Testing",
|
||||
"ICT Third-Party Risk Management",
|
||||
"Information Sharing"
|
||||
],
|
||||
"output_formats": { "csv": true, "ocsf": true }
|
||||
},
|
||||
{
|
||||
"key": "Article",
|
||||
"label": "Article",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"output_formats": { "csv": true, "ocsf": true }
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Per attribute:
|
||||
|
||||
- `key` (required): attribute name as it will appear in `requirement.attributes`.
|
||||
- `label`: human-readable label used in CSV headers and PDF.
|
||||
- `type`: one of `str`, `int`, `float`, `bool`, `list_str`, `list_dict`. Defaults to `str`.
|
||||
- `enum`: optional list of allowed values; non-conforming values are rejected at load time.
|
||||
- `required`: if `true`, every requirement must include this key with a non-null value.
|
||||
- `enum_display` / `enum_order`: optional per-enum-value visual metadata (label, abbreviation, color, icon) and explicit ordering for PDF rendering.
|
||||
- `output_formats`: `{ "csv": <bool>, "ocsf": <bool> }` — toggles inclusion in each output format. Both default to `true`.
|
||||
|
||||
### `outputs`
|
||||
|
||||
Optional. Controls how the framework is rendered in the console table and in the generated PDF report. Skipping it falls back to sensible defaults.
|
||||
|
||||
```json
|
||||
"outputs": {
|
||||
"table_config": {
|
||||
"group_by": "Pillar"
|
||||
},
|
||||
"pdf_config": {
|
||||
"language": "en",
|
||||
"primary_color": "#003399",
|
||||
"secondary_color": "#0055A5",
|
||||
"bg_color": "#F0F4FA",
|
||||
"group_by_field": "Pillar",
|
||||
"sections": [ "ICT Risk Management", "ICT-Related Incident Reporting", "..." ],
|
||||
"section_short_names": { "ICT Risk Management": "ICT Risk Mgmt" },
|
||||
"charts": [
|
||||
{
|
||||
"id": "pillar_compliance",
|
||||
"type": "horizontal_bar",
|
||||
"group_by": "Pillar",
|
||||
"title": "Compliance Score by Pillar",
|
||||
"y_label": "Pillar",
|
||||
"x_label": "Compliance %",
|
||||
"value_source": "compliance_percent",
|
||||
"color_mode": "by_value"
|
||||
}
|
||||
],
|
||||
"filter": { "only_failed": true, "include_manual": false }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`table_config.group_by` must reference an attribute key declared in `attributes_metadata`. The same applies to `pdf_config.group_by_field` and to every `charts[].group_by`.
|
||||
|
||||
For frameworks with weighted scoring (e.g. ThreatScore) declare `pdf_config.scoring` with `risk_field` / `weight_field` / `risk_boost_factor`. For column splitting (e.g. CIS Level 1 vs Level 2) use `table_config.split_by`.
|
||||
|
||||
### `requirements`
|
||||
|
||||
```json
|
||||
"requirements": [
|
||||
{
|
||||
"id": "DORA-Art5",
|
||||
"name": "Governance and organisation",
|
||||
"description": "Financial entities shall have a sound, comprehensive and well-documented ICT internal governance and control framework. ...",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 5",
|
||||
"ArticleTitle": "Governance and organisation"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"iam_avoid_root_usage",
|
||||
"iam_no_root_access_key",
|
||||
"iam_root_mfa_enabled"
|
||||
],
|
||||
"azure": [],
|
||||
"gcp": []
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Per requirement:
|
||||
|
||||
- `id` (required): unique identifier within the framework.
|
||||
- `description` (required): the requirement text as authored by the framework.
|
||||
- `name`: short title shown alongside the id.
|
||||
- `attributes`: flat dict; keys must conform to `attributes_metadata`.
|
||||
- `checks`: dict keyed by provider name (the same lowercase keys listed in the previous section). Each value is a list of Prowler check names that evidence this requirement for that provider. The list **may be empty** and the dict itself defaults to `{}` if omitted; either way the requirement is still loaded and listed by `--list-compliance-requirements`, it just has zero checks to execute. Note: there is **no automatic check-existence validation** at load time — referencing a non-existent check name will silently produce a requirement with no findings. Validate this yourself (see "Validating Your Framework" below).
|
||||
|
||||
For MITRE-style frameworks, additional optional fields are available on the requirement: `tactics`, `sub_techniques`, `platforms`, `technique_url` (these are populated automatically when adapting a legacy MITRE JSON to the universal model).
|
||||
|
||||
### Multi-provider frameworks
|
||||
|
||||
A single universal file can cover any number of providers. The framework appears under each provider's `--list-compliance` output as long as **at least one** requirement has that provider key in its `checks` dict.
|
||||
|
||||
When extending an existing universal framework with a new provider, the only change required is editing `requirement.checks`:
|
||||
|
||||
```diff
|
||||
"checks": {
|
||||
"aws": ["iam_avoid_root_usage", "iam_no_root_access_key"],
|
||||
+ "azure": ["entra_policy_ensure_mfa_for_admin_roles"]
|
||||
}
|
||||
```
|
||||
|
||||
No code changes, no new file, no registration step.
|
||||
|
||||
## Legacy Provider-Specific Compliance Framework
|
||||
|
||||
The legacy schema is still fully supported and remains the format used by most frameworks shipped today (CIS, NIST, ISO 27001, FedRAMP, PCI DSS, GDPR, HIPAA, ENS, etc.). It binds a framework to a single provider and validates each requirement against a framework-specific Pydantic attribute class.
|
||||
|
||||
The legacy schema spans **four layers** — a complete contribution must touch every layer that applies:
|
||||
|
||||
- **Layer 1 — Schema validation:** the Pydantic models in `prowler/lib/check/compliance_models.py` define the canonical schema for each attribute shape.
|
||||
- **Layer 2 — JSON catalog:** the framework JSON file in `prowler/compliance/<provider>/` lists every requirement and maps it to checks.
|
||||
- **Layer 3 — Output formatter:** the Python module in `prowler/lib/outputs/compliance/<framework>/` builds the CSV row model, the per-provider transformer, and the CLI summary table.
|
||||
- **Layer 4 — Output dispatchers:** the dispatchers in `prowler/lib/outputs/compliance/compliance.py` and `prowler/lib/outputs/compliance/compliance_output.py` route findings to the right formatter based on the framework identifier.
|
||||
|
||||
The universal schema collapses Layers 3 and 4 into declarative configuration inside the JSON — that is the main reason it is preferred for new contributions.
|
||||
|
||||
### Directory structure and file naming
|
||||
|
||||
Compliance frameworks live at:
|
||||
|
||||
@@ -46,8 +234,8 @@ prowler/compliance/<provider>/<framework>_<version>_<provider>.json
|
||||
The filename conventions are:
|
||||
|
||||
- All lowercase, words separated with underscores.
|
||||
- `<provider>` is a supported provider identifier: `aws`, `azure`, `gcp`, `kubernetes`, `m365`, `github`, `googleworkspace`, `alibabacloud`, `oraclecloud`, `cloudflare`, `mongodbatlas`, `nhn`, `openstack`, `iac`, `llm`.
|
||||
- `<version>` is optional. Omit it when the framework has no versioning, as in `ccc_aws.json`.
|
||||
- `<provider>` is a supported provider identifier (same lowercase list as the universal section above).
|
||||
- `<version>` is optional but recommended. Omit only when the framework has no versioning (e.g. `ccc_aws.json`).
|
||||
- The file basename (without `.json`) is the framework key that Prowler CLI accepts via `--compliance`.
|
||||
|
||||
Examples:
|
||||
@@ -62,48 +250,50 @@ The output formatter directory mirrors the framework name:
|
||||
|
||||
```
|
||||
prowler/lib/outputs/compliance/<framework>/
|
||||
├── <framework>.py # CLI summary-table dispatcher
|
||||
├── <framework>.py # CLI summary-table dispatcher
|
||||
├── <framework>_<provider>.py # Per-provider transformer class
|
||||
├── models.py # Pydantic CSV row model
|
||||
└── __init__.py
|
||||
```
|
||||
|
||||
## JSON Schema Reference
|
||||
### JSON schema reference
|
||||
|
||||
Every compliance file is a JSON document with the following top-level keys.
|
||||
Every legacy compliance file is a JSON document with the following top-level keys. `Framework`, `Name` and `Provider` are validated non-empty by the root validator `framework_and_provider_must_not_be_empty` (`compliance_models.py:329`).
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `Framework` | string | Yes | Canonical framework identifier, for example `CIS`, `NIST-800-53-Revision-5`, `ENS`, `CCC`. |
|
||||
| `Name` | string | Yes | Human-readable framework name displayed by Prowler App. |
|
||||
| `Version` | string | Yes | Framework version, for example `2.0`. Use an empty string only for frameworks without versioning. See [Version Handling](#version-handling). |
|
||||
| `Version` | string | Yes (recommended) | Framework version, e.g. `2.0`. See [Version Handling](#version-handling). |
|
||||
| `Provider` | string | Yes | Upper-cased provider identifier: `AWS`, `AZURE`, `GCP`, `KUBERNETES`, `M365`, `GITHUB`, `GOOGLEWORKSPACE`, and so on. |
|
||||
| `Description` | string | Yes | Short description of the framework's scope and purpose. |
|
||||
| `Requirements` | array | Yes | List of [requirement objects](#requirement-object). |
|
||||
|
||||
### Requirement Object
|
||||
#### Requirement Object
|
||||
|
||||
Each entry in `Requirements` describes one control or requirement.
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `Id` | string | Yes | Unique identifier within the framework, for example `1.10` or `CCC.Core.CN01.AR01`. |
|
||||
| `Name` | string | No | Optional human-readable name used by frameworks that distinguish control name from description, such as NIST. |
|
||||
| `Name` | string | No | Optional human-readable name (frameworks like NIST distinguish control name from description). |
|
||||
| `Description` | string | Yes | Verbatim description from the source framework. |
|
||||
| `Attributes` | array | Yes | List of [attribute objects](#attribute-objects). The shape depends on the framework. |
|
||||
| `Checks` | array of strings | Yes | Prowler check identifiers that automate the requirement. Leave the list empty when the control cannot be automated. |
|
||||
|
||||
### Attribute Objects
|
||||
#### Attribute Objects
|
||||
|
||||
Attributes carry the metadata that Prowler App and the CSV output display for each requirement. The object shape is framework-specific and is validated by a dedicated Pydantic model in `prowler/lib/check/compliance_models.py`. The most common shapes are summarized below.
|
||||
`Attributes` is parsed against the union declared in `Compliance_Requirement.Attributes` (`compliance_models.py:293`). Pydantic v1 tries each member of the union in declaration order and falls back to `Generic_Compliance_Requirement_Attribute` (the last entry) when nothing else matches — so a brand-new shape that doesn't match any existing class will silently be accepted as Generic, losing its specific fields.
|
||||
|
||||
#### CIS_Requirement_Attribute
|
||||
As of today, the registered attribute classes are: `CIS_Requirement_Attribute`, `ENS_Requirement_Attribute`, `ASDEssentialEight_Requirement_Attribute`, `ISO27001_2013_Requirement_Attribute`, `AWS_Well_Architected_Requirement_Attribute`, `KISA_ISMSP_Requirement_Attribute`, `Prowler_ThreatScore_Requirement_Attribute`, `CCC_Requirement_Attribute`, `C5Germany_Requirement_Attribute`, `CSA_CCM_Requirement_Attribute`, and `Generic_Compliance_Requirement_Attribute` (fallback). MITRE-style frameworks use the separate `Mitre_Requirement` model with `Tactics` / `SubTechniques` / `Platforms` / `TechniqueURL` at the requirement top level. The most common shapes are summarized below.
|
||||
|
||||
##### CIS_Requirement_Attribute
|
||||
|
||||
Used by every CIS benchmark.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|---|---|
|
||||
| `Section` | string | Yes | Top-level section, for example `1 Identity and Access Management`. |
|
||||
| `Section` | string | Yes | Top-level section, e.g. `1 Identity and Access Management`. |
|
||||
| `SubSection` | string | No | Optional second-level grouping. |
|
||||
| `Profile` | enum | Yes | One of `Level 1`, `Level 2`, `E3 Level 1`, `E3 Level 2`, `E5 Level 1`, `E5 Level 2`. |
|
||||
| `AssessmentStatus` | enum | Yes | `Manual` or `Automated`. |
|
||||
@@ -116,7 +306,7 @@ Used by every CIS benchmark.
|
||||
| `DefaultValue` | string | No | Default configuration value, when relevant. |
|
||||
| `References` | string | Yes | Colon-separated list of reference URLs. |
|
||||
|
||||
#### ENS_Requirement_Attribute
|
||||
##### ENS_Requirement_Attribute
|
||||
|
||||
Used by the Spanish ENS (Esquema Nacional de Seguridad) frameworks.
|
||||
|
||||
@@ -132,13 +322,13 @@ Used by the Spanish ENS (Esquema Nacional de Seguridad) frameworks.
|
||||
| `ModoEjecucion` | string | Yes | Execution mode (`manual`, `automático`, `híbrido`). |
|
||||
| `Dependencias` | array of strings | Yes | Ids of prerequisite controls. Empty list when none. |
|
||||
|
||||
#### CCC_Requirement_Attribute
|
||||
##### CCC_Requirement_Attribute
|
||||
|
||||
Used by the Common Cloud Controls Catalog.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|---|---|
|
||||
| `FamilyName` | string | Yes | Control family, for example `Data`. |
|
||||
| `FamilyName` | string | Yes | Control family, e.g. `Data`. |
|
||||
| `FamilyDescription` | string | Yes | Description of the family. |
|
||||
| `Section` | string | Yes | Section title. |
|
||||
| `SubSection` | string | Yes | Subsection title, or empty string. |
|
||||
@@ -148,9 +338,9 @@ Used by the Common Cloud Controls Catalog.
|
||||
| `SectionThreatMappings` | array of objects | Yes | Each entry has `ReferenceId` and `Identifiers`. |
|
||||
| `SectionGuidelineMappings` | array of objects | Yes | Each entry has `ReferenceId` and `Identifiers`. |
|
||||
|
||||
#### Generic_Compliance_Requirement_Attribute
|
||||
##### Generic_Compliance_Requirement_Attribute
|
||||
|
||||
The fallback attribute model used when no framework-specific schema applies (for example NIST 800-53, PCI DSS, GDPR, HIPAA).
|
||||
The fallback attribute model used when no framework-specific schema applies (e.g. NIST 800-53, PCI DSS, GDPR, HIPAA). It is **always the last** element of the `Compliance_Requirement.Attributes` Union; that ordering is load-bearing.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|---|---|
|
||||
@@ -158,17 +348,17 @@ The fallback attribute model used when no framework-specific schema applies (for
|
||||
| `Section` | string | No | Section name. |
|
||||
| `SubSection` | string | No | Subsection name. |
|
||||
| `SubGroup` | string | No | Subgroup name. |
|
||||
| `Service` | string | No | Affected service, for example `aws`, `iam`. |
|
||||
| `Service` | string | No | Affected service, e.g. `iam`. |
|
||||
| `Type` | string | No | Control type. |
|
||||
| `Comment` | string | No | Free-form comment. |
|
||||
|
||||
Additional per-framework attribute models exist for `AWS_Well_Architected_Requirement_Attribute`, `ISO27001_2013_Requirement_Attribute`, `Mitre_Requirement_Attribute_<Provider>`, `KISA_ISMSP_Requirement_Attribute`, `Prowler_ThreatScore_Requirement_Attribute`, `C5Germany_Requirement_Attribute`, and `CSA_CCM_Requirement_Attribute`. Consult `prowler/lib/check/compliance_models.py` for their full field sets.
|
||||
For the remaining attribute classes (`AWS_Well_Architected_Requirement_Attribute`, `ISO27001_2013_Requirement_Attribute`, `Mitre_Requirement_Attribute_<Provider>`, `KISA_ISMSP_Requirement_Attribute`, `Prowler_ThreatScore_Requirement_Attribute`, `C5Germany_Requirement_Attribute`, `CSA_CCM_Requirement_Attribute`) consult `prowler/lib/check/compliance_models.py` for the full field sets.
|
||||
|
||||
<Note>
|
||||
The `Attributes` field is a Pydantic `Union`. The generic attribute model must remain the last element of that Union, otherwise Pydantic v1 silently coerces every framework into the generic shape and your specialized fields are dropped.
|
||||
The `Attributes` field is a Pydantic `Union`. The generic attribute model **must** remain the last element of that Union — otherwise Pydantic v1 silently coerces every framework into the generic shape and your specialized fields are dropped. Adding a brand-new attribute shape requires inserting the Pydantic class **before** `Generic_Compliance_Requirement_Attribute`.
|
||||
</Note>
|
||||
|
||||
## Minimal Working Example
|
||||
#### Minimal working example
|
||||
|
||||
The following snippet is a complete, valid framework file named `my_framework_1.0_aws.json`, saved at `prowler/compliance/aws/my_framework_1.0_aws.json`. It uses the generic attribute shape for simplicity.
|
||||
|
||||
@@ -214,26 +404,26 @@ The following snippet is a complete, valid framework file named `my_framework_1.
|
||||
}
|
||||
```
|
||||
|
||||
## Mapping Checks to Requirements
|
||||
### Mapping checks to requirements
|
||||
|
||||
Each requirement links to the Prowler checks that, together, produce a PASS or FAIL verdict for that control.
|
||||
|
||||
- **Include every requirement from the source catalog.** The framework file must mirror the full control list, one-to-one. Compliance percentages, dashboards, and exported metadata are computed against the total requirement count, so omitting an unmappable control inflates coverage and misrepresents the framework.
|
||||
- List every check by its canonical identifier, the value of `CheckID` inside the check's `.metadata.json` file.
|
||||
- **Include every requirement from the source catalog.** The framework file must mirror the full control list, one-to-one. Compliance percentages, dashboards, and exported metadata are computed against the total requirement count.
|
||||
- List every check by its canonical identifier — the value of `CheckID` inside the check's `.metadata.json` file.
|
||||
- One requirement can reference multiple checks. The requirement is evaluated as FAIL when any referenced check produces a FAIL finding for a resource in scope.
|
||||
- Leave `Checks` as an empty array when the requirement cannot be automated. The requirement still appears in the report, contributes to the total, and resolves to `MANUAL`. An empty mapping is valid; a missing requirement is not.
|
||||
- Leave `Checks` (legacy) or `checks.<provider>` (universal) as an empty array when the requirement cannot be automated. The requirement still appears in the report and contributes to the total.
|
||||
- Reuse checks across requirements when the same control applies in multiple places. Do not duplicate check logic to match framework structure.
|
||||
- Avoid referencing checks from a different provider. A compliance file is bound to one provider, and cross-provider checks will never match findings in the scan.
|
||||
- Avoid referencing checks from a different provider. A legacy compliance file is bound to one provider, and cross-provider checks will never match findings in the scan.
|
||||
|
||||
To discover available checks, run:
|
||||
To discover available checks:
|
||||
|
||||
```bash
|
||||
uv run python prowler-cli.py <provider> --list-checks
|
||||
```
|
||||
|
||||
## Supporting Multiple Providers
|
||||
### Supporting multiple providers (legacy)
|
||||
|
||||
Each compliance file targets a single provider. To cover several providers with the same framework (for example CIS across AWS, Azure, and GCP), ship one JSON file per provider:
|
||||
The legacy schema binds each file to a single provider. To cover several providers with the same framework, ship one JSON file per provider:
|
||||
|
||||
```
|
||||
prowler/compliance/aws/cis_2.0_aws.json
|
||||
@@ -241,15 +431,15 @@ prowler/compliance/azure/cis_2.0_azure.json
|
||||
prowler/compliance/gcp/cis_2.0_gcp.json
|
||||
```
|
||||
|
||||
Keep the `Framework` and `Version` values identical across the files so the dispatcher matches them, and change only the `Provider`, `Checks`, and provider-specific metadata.
|
||||
Keep the `Framework` and `Version` values identical across the files so the dispatcher matches them; change only the `Provider`, `Checks`, and provider-specific metadata. The CIS output formatter already supports every provider listed above.
|
||||
|
||||
The CIS output formatter already supports every provider listed above. For a brand-new framework that spans several providers, add one transformer per provider in `prowler/lib/outputs/compliance/<framework>/` and extend the summary-table dispatcher accordingly. See [Output Formatter](#output-formatter).
|
||||
For a brand-new framework that spans several providers, **prefer the universal schema** — it covers every provider from a single file. If you must use the legacy schema, add one transformer per provider in `prowler/lib/outputs/compliance/<framework>/` and extend the summary-table dispatcher accordingly. See [Output Formatter](#output-formatter).
|
||||
|
||||
## Output Formatter
|
||||
### Output formatter
|
||||
|
||||
Prowler renders every compliance framework in two forms: a detailed CSV report written to disk, and a summary table printed in the CLI. Both are produced by the output formatter package for the framework.
|
||||
Legacy frameworks render in two forms: a detailed CSV report written to disk, and a summary table printed in the CLI. Both are produced by the output formatter package for the framework. Universal frameworks do **not** need a Python output formatter — the `outputs` config inside the JSON drives rendering — so this section applies only to the legacy schema.
|
||||
|
||||
For a new framework named `my_framework`, create:
|
||||
For a new legacy framework named `my_framework`, create:
|
||||
|
||||
```
|
||||
prowler/lib/outputs/compliance/my_framework/
|
||||
@@ -259,19 +449,19 @@ prowler/lib/outputs/compliance/my_framework/
|
||||
└── models.py # CSV row Pydantic model
|
||||
```
|
||||
|
||||
### Step 1 – Define the CSV Row Model
|
||||
#### Step 1 — Define the CSV row model
|
||||
|
||||
In `models.py`, declare a Pydantic v1 model with one field per CSV column. Use existing models such as `AWSCISModel` in `prowler/lib/outputs/compliance/cis/models.py` as the reference. Fields typically include `Provider`, `Description`, `AccountId`, `Region`, `AssessmentDate`, `Requirements_Id`, `Requirements_Description`, one `Requirements_Attributes_*` field per attribute key, plus the finding fields `Status`, `StatusExtended`, `ResourceId`, `ResourceName`, `CheckId`, `Muted`, `Framework`, `Name`.
|
||||
|
||||
### Step 2 – Implement the Transformer Class
|
||||
#### Step 2 — Implement the transformer
|
||||
|
||||
In `my_framework_aws.py`, subclass `ComplianceOutput` from `prowler.lib.outputs.compliance.compliance_output` and implement `transform(findings, compliance, compliance_name)`. Iterate over `findings`, match each finding to the requirements it satisfies through `finding.compliance.get(compliance_name, [])`, and append one row per attribute to `self._data`.
|
||||
|
||||
### Step 3 – Add the Summary-Table Dispatcher
|
||||
#### Step 3 — Add the summary-table dispatcher
|
||||
|
||||
In `my_framework.py`, implement `get_my_framework_table(findings, bulk_checks_metadata, compliance_framework, output_filename, output_directory, compliance_overview)` following the pattern in `prowler/lib/outputs/compliance/cis/cis.py`.
|
||||
|
||||
### Step 4 – Register the Framework in the Dispatchers
|
||||
#### Step 4 — Register the framework in the dispatchers
|
||||
|
||||
- Add the dispatcher call in `prowler/lib/outputs/compliance/compliance.py`, inside `display_compliance_table`, with a branch such as `elif "my_framework" in compliance_framework:`.
|
||||
- Register the CSV model and transformer in `prowler/lib/outputs/compliance/compliance_output.py` so the CSV file is emitted during the scan.
|
||||
@@ -280,49 +470,94 @@ In `my_framework.py`, implement `get_my_framework_table(findings, bulk_checks_me
|
||||
For NIST-style catalogs that use `Generic_Compliance_Requirement_Attribute`, no custom formatter is needed. The generic formatter in `prowler/lib/outputs/compliance/generic/` handles them automatically, provided the JSON validates against the generic attribute schema.
|
||||
</Note>
|
||||
|
||||
## Version Handling
|
||||
### Legacy-to-universal adapter
|
||||
|
||||
At load time, every legacy file is transparently adapted to a `ComplianceFramework` via `adapt_legacy_to_universal()` (`compliance_models.py:819`), which: (a) flattens the first element of `Attributes` into a flat `attributes` dict, (b) wraps `Checks` as `{provider_lower: [...]}`, (c) infers `attributes_metadata` from the matched Pydantic class via `_infer_attribute_metadata()`. The rest of Prowler (CSV/OCSF/PDF output, CLI table) then treats both formats identically.
|
||||
|
||||
Loader-error behaviour differs between the two entry points:
|
||||
|
||||
- `load_compliance_framework()` (legacy) is **fail-fast**: it calls `sys.exit(1)` on any `ValidationError` (`compliance_models.py:464`).
|
||||
- `load_compliance_framework_universal()` is more lenient — it logs the error and returns `None`, so `get_bulk_compliance_frameworks_universal()` simply skips the broken file and keeps loading the rest.
|
||||
|
||||
## Version handling
|
||||
|
||||
Prowler matches frameworks by concatenating `Framework` and `Version`. A missing or empty `Version` collapses several frameworks to the same key and breaks CLI filtering with `--compliance`.
|
||||
|
||||
- Always set `Version` to a non-empty string, even for frameworks that rename editions rather than version them. Use the edition identifier (for example `RD2022`, `v2025.10`, `4.0`).
|
||||
- Always set `Version` (or `version` for universal frameworks) to a non-empty string, even for frameworks that rename editions rather than version them. Use the edition identifier (for example `RD2022`, `v2025.10`, `4.0`, `2022/2554`).
|
||||
- When the source catalog has no version, use the first year of adoption or the release date.
|
||||
- Make sure the version substring embedded in the filename matches `Version`, because the CLI dispatcher reads `compliance_framework.split("_")[1]` to select the correct version.
|
||||
- For **legacy** files, make sure the version substring embedded in the filename matches `Version`, because the CLI dispatcher reads `compliance_framework.split("_")[1]` to select the correct version.
|
||||
|
||||
## Validating the Framework Locally
|
||||
## Validating Your Framework
|
||||
|
||||
Follow the steps below before opening a pull request.
|
||||
Before opening a PR, validate the JSON loads cleanly against the model and that every referenced check actually exists.
|
||||
|
||||
### 1. Run the Compliance Model Validator
|
||||
### 1. Schema validation
|
||||
|
||||
For **universal** frameworks, load the file and inspect what was parsed. The framework key inside `bulk` is the **basename of the JSON file** (without `.json`); for `prowler/compliance/dora.json` that key is `dora`, for `prowler/compliance/aws/cis_5.0_aws.json` it is `cis_5.0_aws`.
|
||||
|
||||
```python
|
||||
from prowler.lib.check.compliance_models import (
|
||||
load_compliance_framework_universal,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
|
||||
fw = load_compliance_framework_universal("prowler/compliance/<your_framework>.json")
|
||||
assert fw is not None, "load returned None — check the logs for the validation error"
|
||||
print(fw.framework, len(fw.requirements), fw.get_providers())
|
||||
|
||||
bulk = get_bulk_compliance_frameworks_universal("aws")
|
||||
assert "<your_framework_filename_without_json>" in bulk
|
||||
```
|
||||
|
||||
### 2. Check existence cross-check
|
||||
|
||||
There is **no automatic check-existence validation** at load time. Cross-check that every check name in your framework maps to a real check directory:
|
||||
|
||||
```python
|
||||
import os
|
||||
real = set()
|
||||
for svc in os.listdir("prowler/providers/aws/services"):
|
||||
svc_path = f"prowler/providers/aws/services/{svc}"
|
||||
if not os.path.isdir(svc_path):
|
||||
continue
|
||||
for entry in os.listdir(svc_path):
|
||||
if os.path.isfile(f"{svc_path}/{entry}/{entry}.metadata.json"):
|
||||
real.add(entry)
|
||||
|
||||
referenced = {c for r in fw.requirements for c in r.checks.get("aws", [])}
|
||||
missing = referenced - real
|
||||
assert not missing, f"checks referenced in framework but not found in repo: {sorted(missing)}"
|
||||
```
|
||||
|
||||
### 3. CLI smoke test
|
||||
|
||||
```bash
|
||||
uv run python prowler-cli.py <provider> --list-compliance
|
||||
```
|
||||
|
||||
The framework must appear in the output. A validation error indicates a schema mismatch between the JSON file and the attribute model.
|
||||
|
||||
### 2. Run a Scan Filtered by the Framework
|
||||
The framework must appear in the output. A validation error indicates a schema mismatch.
|
||||
|
||||
```bash
|
||||
uv run python prowler-cli.py <provider> \
|
||||
--compliance <framework>_<version>_<provider> \
|
||||
--compliance <framework_key> \
|
||||
--log-level ERROR
|
||||
```
|
||||
|
||||
Verify that:
|
||||
|
||||
- Prowler produces a CSV file under `output/compliance/` with the expected name.
|
||||
- The CLI summary table lists every section in the framework.
|
||||
- The CLI summary table lists every section / pillar of the framework.
|
||||
- Findings roll up under the expected requirements.
|
||||
|
||||
### 3. Inspect the CSV Output
|
||||
### 4. Inspect the CSV output
|
||||
|
||||
Open the generated CSV and confirm:
|
||||
|
||||
- All columns defined in `models.py` appear.
|
||||
- Every requirement has at least one row per scanned resource.
|
||||
- Values such as `Requirements_Attributes_Section` reflect the JSON content.
|
||||
- All columns defined in `models.py` (legacy) or in `attributes_metadata` (universal) appear.
|
||||
- Every requirement has at least one row per scanned resource (when there are findings).
|
||||
- Attribute values such as `Requirements_Attributes_Section` reflect the JSON content.
|
||||
|
||||
### 4. Verify the Framework in Prowler App
|
||||
### 5. Verify the framework in Prowler App
|
||||
|
||||
Launch Prowler App locally (`docker compose up` from the repository root) and run a scan with the new compliance framework. Confirm the compliance page renders the requirements, sections, and status widgets correctly.
|
||||
|
||||
@@ -331,7 +566,7 @@ Launch Prowler App locally (`docker compose up` from the repository root) and ru
|
||||
Compliance contributions require two layers of tests.
|
||||
|
||||
- **Schema tests** exercise the Pydantic models. Extend `tests/lib/check/universal_compliance_models_test.py` with a case that loads the new JSON file and asserts the attribute type matches the expected model.
|
||||
- **Output tests** exercise the transformer. Mirror the structure under `tests/lib/outputs/compliance/<framework>/` with fixtures that feed synthetic findings through the transformer and assert the resulting CSV rows.
|
||||
- **Output tests** (legacy frameworks only) exercise the transformer. Mirror the structure under `tests/lib/outputs/compliance/<framework>/` with fixtures that feed synthetic findings through the transformer and assert the resulting CSV rows.
|
||||
|
||||
Run the suite with:
|
||||
|
||||
@@ -342,7 +577,20 @@ uv run pytest -n auto tests/lib/check/universal_compliance_models_test.py \
|
||||
|
||||
For guidance on writing Prowler SDK tests, refer to [Unit Testing](/developer-guide/unit-testing).
|
||||
|
||||
## Submitting the Pull Request
|
||||
## Running and listing your framework
|
||||
|
||||
Once the file is in place, the CLI auto-discovers it:
|
||||
|
||||
```sh
|
||||
prowler <provider> --list-compliance # framework appears in the list
|
||||
prowler <provider> --compliance <framework_key> --list-checks
|
||||
prowler <provider> --compliance <framework_key> # full scan + compliance report
|
||||
prowler <provider> --compliance <framework_key> --list-compliance-requirements <framework_key>
|
||||
```
|
||||
|
||||
For end-user-facing tutorials (recommended for high-profile frameworks), add a dedicated page under `docs/user-guide/compliance/tutorials/` and register it in the `"Compliance"` group of `docs/docs.json`. See `docs/user-guide/compliance/tutorials/threatscore.mdx` as a reference.
|
||||
|
||||
## Submitting the pull request
|
||||
|
||||
Before opening the pull request:
|
||||
|
||||
@@ -352,28 +600,31 @@ Before opening the pull request:
|
||||
uv run pytest -n auto
|
||||
```
|
||||
2. Add a changelog entry under the `### 🚀 Added` section of `prowler/CHANGELOG.md`, describing the new framework and the providers it covers.
|
||||
3. Follow the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and set the PR title using Conventional Commits, for example `feat(compliance): add My Framework 1.0 for AWS`.
|
||||
3. Follow the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and set the PR title using Conventional Commits, e.g. `feat(compliance): add My Framework 1.0 for AWS`.
|
||||
4. Request review from the compliance codeowners listed in `.github/CODEOWNERS`.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
The following issues are the most common when contributing a compliance framework.
|
||||
|
||||
- **`ValidationError: field required` during scan.** The JSON is missing a required attribute field. Re-check the matching Pydantic model in `prowler/lib/check/compliance_models.py`.
|
||||
- **All attributes collapse to `Generic_Compliance_Requirement_Attribute` values.** The Pydantic `Union` is ordered incorrectly, or the JSON matches only the generic shape. Move the generic model to the last Union position and ensure every required field is present in the JSON.
|
||||
- **`--compliance` filter does not find the framework.** The filename does not match the expected pattern `<framework>_<version>_<provider>.json`, the version is empty, or the file lives outside `prowler/compliance/<provider>/`.
|
||||
- **CLI summary table is empty but the CSV is populated.** The dispatcher branch in `prowler/lib/outputs/compliance/compliance.py` is missing or its substring match does not catch the framework key.
|
||||
- **CSV file is missing after the scan.** The transformer class is not registered in `prowler/lib/outputs/compliance/compliance_output.py`, or `transform()` raises silently. Run the scan with `--log-level DEBUG`.
|
||||
- **Findings do not roll up under a requirement.** A check listed in `Checks` either does not exist for that provider or is spelled incorrectly. Run `--list-checks | grep <check_name>` to confirm.
|
||||
- **`ValidationError: field required` during scan (legacy).** The JSON is missing a required attribute field. Re-check the matching Pydantic model in `prowler/lib/check/compliance_models.py`.
|
||||
- **All attributes collapse to `Generic_Compliance_Requirement_Attribute` values (legacy).** The Pydantic `Union` is ordered incorrectly, or the JSON matches only the generic shape. Keep the generic model in the last Union position and ensure every required field is present in the JSON.
|
||||
- **`attributes_metadata validation failed` (universal).** The root validator in `compliance_models.py:669` rejected the file. The error message lists each offending requirement; common causes are unknown attribute keys (typo or missing entry in `attributes_metadata`), enum violations, or missing required keys.
|
||||
- **`--compliance` filter does not find the framework.** For legacy: the filename does not match `<framework>_<version>_<provider>.json`, the version is empty, or the file lives outside `prowler/compliance/<provider>/`. For universal: the file is not at the top level of `prowler/compliance/` or it loaded as `None` (check logs for the validation error).
|
||||
- **CLI summary table is empty but the CSV is populated (legacy).** The dispatcher branch in `prowler/lib/outputs/compliance/compliance.py` is missing or its substring match does not catch the framework key.
|
||||
- **CSV file is missing after the scan (legacy).** The transformer class is not registered in `prowler/lib/outputs/compliance/compliance_output.py`, or `transform()` raises silently. Run the scan with `--log-level DEBUG`.
|
||||
- **Findings do not roll up under a requirement.** A check listed in `Checks` either does not exist for that provider or is spelled incorrectly. Run `--list-checks | grep <check_name>` to confirm, or run the check-existence cross-check from "Validating Your Framework".
|
||||
|
||||
## Reference Examples
|
||||
## Reference examples
|
||||
|
||||
Use the following files as templates when modeling a new contribution.
|
||||
|
||||
- `prowler/compliance/aws/cis_2.0_aws.json` – CIS attribute shape.
|
||||
- `prowler/compliance/aws/nist_800_53_revision_5_aws.json` – Generic attribute shape.
|
||||
- `prowler/compliance/aws/ccc_aws.json` – CCC attribute shape.
|
||||
- `prowler/compliance/azure/ens_rd2022_azure.json` – ENS attribute shape.
|
||||
- `prowler/lib/check/compliance_models.py` – Canonical Pydantic schemas.
|
||||
- `prowler/lib/outputs/compliance/cis/` – Reference implementation of a multi-provider output formatter.
|
||||
- `prowler/lib/outputs/compliance/generic/` – Reference implementation of a generic output formatter.
|
||||
- `prowler/compliance/dora.json` — universal schema, single-provider populated (AWS), ready to extend with more providers.
|
||||
- `prowler/compliance/csa_ccm_4.0.json` — universal schema, multi-provider populated (AWS, Azure, GCP, AlibabaCloud, OracleCloud).
|
||||
- `prowler/compliance/aws/cis_2.0_aws.json` — legacy CIS attribute shape.
|
||||
- `prowler/compliance/aws/nist_800_53_revision_5_aws.json` — legacy generic attribute shape.
|
||||
- `prowler/compliance/aws/ccc_aws.json` — legacy CCC attribute shape.
|
||||
- `prowler/compliance/azure/ens_rd2022_azure.json` — legacy ENS attribute shape.
|
||||
- `prowler/lib/check/compliance_models.py` — canonical Pydantic schemas for both formats.
|
||||
- `prowler/lib/outputs/compliance/cis/` — reference implementation of a multi-provider legacy output formatter.
|
||||
- `prowler/lib/outputs/compliance/generic/` — reference implementation of a legacy generic output formatter.
|
||||
|
||||
@@ -0,0 +1,730 @@
|
||||
---
|
||||
title: 'StackIT Provider'
|
||||
---
|
||||
|
||||
This page details the [StackIT Cloud](https://www.stackit.de/) provider implementation in Prowler.
|
||||
|
||||
By default, Prowler audits a single StackIT project per scan. To configure it, provide the project ID and either a service account key file path or inline service account key JSON.
|
||||
|
||||
## StackIT Provider Classes Architecture
|
||||
|
||||
The StackIT provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the StackIT-specific implementation, highlighting how the generic provider concepts are realized for StackIT in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider).
|
||||
|
||||
### `StackitProvider` (Main Class)
|
||||
|
||||
- **Location:** [`prowler/providers/stackit/stackit_provider.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/stackit/stackit_provider.py)
|
||||
- **Base Class:** Inherits from `Provider` (see [base class details](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/common/provider.py)).
|
||||
- **Purpose:** Central orchestrator for StackIT-specific logic, API authentication, credential validation, and configuration.
|
||||
- **Key StackIT Responsibilities:**
|
||||
- Initializes StackIT SDK authentication via a service account key file or inline service account key JSON. The SDK mints and refreshes access tokens internally.
|
||||
- Validates the service account credentials and project ID (UUID format validation).
|
||||
- Loads and manages configuration, mutelist, and fixer settings.
|
||||
- Provides properties and methods for downstream StackIT service classes to access credentials, identity, and configuration data.
|
||||
|
||||
### Data Models
|
||||
|
||||
- **Location:** [`prowler/providers/stackit/models.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/stackit/models.py)
|
||||
- **Purpose:** Define structured data for StackIT identity and output configuration.
|
||||
- **Key StackIT Models:**
|
||||
- `StackITIdentityInfo`: Holds StackIT identity metadata, including project ID and project name (fetched automatically from Resource Manager API).
|
||||
- `StackITOutputOptions`: Customizes default output filenames so StackIT reports include the audited project ID.
|
||||
- IaaS resource models such as `SecurityGroup` and `SecurityGroupRule` are defined in the IaaS service module.
|
||||
|
||||
### StackIT Services
|
||||
|
||||
- **Location:** [`prowler/providers/stackit/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/stackit/services)
|
||||
- **Purpose:** Implement StackIT service clients and resource collection logic following the generic [service pattern](/developer-guide/services#service-base-class).
|
||||
- **Current Implementation:** The `IaaSService` collects security groups, rules, and network interface usage across supported StackIT regions.
|
||||
|
||||
### Exception Handling
|
||||
|
||||
- **Location:** [`prowler/providers/stackit/exceptions/exceptions.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/stackit/exceptions/exceptions.py)
|
||||
- **Purpose:** Custom exception classes for StackIT-specific error handling, such as credential validation, API connection, and configuration errors.
|
||||
- **Key Exception Classes:**
|
||||
- `StackITBaseException`: Base exception for all StackIT provider errors.
|
||||
- `StackITCredentialsError`: Raised when credentials are invalid or missing.
|
||||
- `StackITInvalidProjectIdError`: Raised when project ID is invalid or not in UUID format.
|
||||
- `StackITAPIError`: Raised when StackIT API calls fail.
|
||||
|
||||
## Authentication
|
||||
|
||||
### Service Account Creation and Key Generation
|
||||
|
||||
StackIT uses service account keys for API authentication. Service account keys are RSA key-pair based and provide secure, short-lived access tokens.
|
||||
|
||||
### Creating a Service Account Key
|
||||
|
||||
#### Method 1: Via StackIT Portal
|
||||
|
||||
1. **Navigate to Service Accounts**
|
||||
- Go to the [StackIT Portal](https://portal.stackit.cloud/)
|
||||
- Select your project
|
||||
- Click on **Service Accounts** in the left sidebar
|
||||
|
||||
2. **Create or Select Service Account**
|
||||
- If you don't have a service account, click **Create Service Account**
|
||||
- Provide a name and description
|
||||
- Assign necessary permissions:
|
||||
- For IaaS security checks: `iaas.viewer` or `project.owner`
|
||||
- For comprehensive audits: `project.owner`
|
||||
|
||||
3. **Generate Service Account Key**
|
||||
- Select your service account
|
||||
- Navigate to **Service Account Keys**
|
||||
- Click **Create key**
|
||||
- Choose one of the following options:
|
||||
- **STACKIT-generated key pair** (Recommended): Let STACKIT automatically generate an RSA key-pair
|
||||
- **User-provided key pair**: Upload your own RSA 2048 public key
|
||||
|
||||
4. **Download and Save the Key**
|
||||
- Download the generated service account key file (JSON format)
|
||||
- **Important**: Save the key securely - it contains your private key and will only be available once
|
||||
- Store the key file in a secure location (e.g., `~/.stackit/sa_key.json`)
|
||||
|
||||
#### Method 2: Via StackIT CLI
|
||||
|
||||
```bash
|
||||
# Install STACKIT CLI (if not already installed)
|
||||
# Follow instructions at: https://github.com/stackitcloud/stackit-cli
|
||||
|
||||
# Create service account key (STACKIT-generated)
|
||||
stackit service-account key create --email my-service-account@example.com
|
||||
|
||||
# Or create with your own RSA 2048 public key
|
||||
# First, generate your RSA key pair:
|
||||
openssl genrsa -out private-key.pem 2048
|
||||
openssl rsa -in private-key.pem -pubout -out public-key.pem
|
||||
|
||||
# Then create the key with your public key:
|
||||
stackit service-account key create \
|
||||
--email my-service-account@example.com \
|
||||
--public-key "$(cat public-key.pem)"
|
||||
```
|
||||
|
||||
### Finding Your Project ID
|
||||
|
||||
Your StackIT project ID is a UUID that can be found:
|
||||
|
||||
1. In the StackIT Portal URL when viewing your project: `https://portal.stackit.cloud/projects/{PROJECT_ID}/...`
|
||||
2. In the project settings page
|
||||
3. Using the StackIT CLI: `stackit project list`
|
||||
|
||||
### Passing the Service Account Key to Prowler
|
||||
|
||||
Prowler accepts the service account credentials in two equivalent forms; both go through the same StackIT SDK flow and refresh access tokens internally.
|
||||
|
||||
#### Option 1: Key File Path (key persisted on disk)
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
export STACKIT_PROJECT_ID="12345678-1234-1234-1234-123456789abc"
|
||||
|
||||
prowler stackit
|
||||
```
|
||||
|
||||
Or as CLI flags:
|
||||
|
||||
```bash
|
||||
prowler stackit \
|
||||
--stackit-service-account-key-path ~/.stackit/sa-key.json \
|
||||
--stackit-project-id 12345678-1234-1234-1234-123456789abc
|
||||
```
|
||||
|
||||
#### Option 2: Inline Key Content (CI/CD, secret managers)
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY="$(vault kv get -field=key stackit/sa)"
|
||||
export STACKIT_PROJECT_ID="12345678-1234-1234-1234-123456789abc"
|
||||
|
||||
prowler stackit
|
||||
```
|
||||
|
||||
Prefer the environment variable over the matching `--stackit-service-account-key` CLI flag; passing the secret on the command line leaks it through process listings and shell history.
|
||||
|
||||
### Credential Lookup Order
|
||||
|
||||
Prowler resolves credentials in this order:
|
||||
|
||||
1. **Command-line arguments**:
|
||||
- `--stackit-service-account-key`
|
||||
- `--stackit-service-account-key-path`
|
||||
- `--stackit-project-id`
|
||||
2. **Environment variables**:
|
||||
- `STACKIT_SERVICE_ACCOUNT_KEY`
|
||||
- `STACKIT_SERVICE_ACCOUNT_KEY_PATH`
|
||||
- `STACKIT_PROJECT_ID`
|
||||
|
||||
When both the inline key and the key file path are set, the inline content takes precedence.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Command-Line Arguments
|
||||
|
||||
StackIT-specific command-line arguments:
|
||||
|
||||
| Argument | Description | Required | Default |
|
||||
|----------|-------------|----------|---------|
|
||||
| `--stackit-service-account-key-path` | Path to a StackIT service account key JSON file | Yes* | `$STACKIT_SERVICE_ACCOUNT_KEY_PATH` |
|
||||
| `--stackit-service-account-key` | Inline JSON content of a StackIT service account key (preferred env var: `STACKIT_SERVICE_ACCOUNT_KEY`) | Yes* | `$STACKIT_SERVICE_ACCOUNT_KEY` |
|
||||
| `--stackit-project-id` | StackIT project ID (UUID format) | Yes* | `$STACKIT_PROJECT_ID` |
|
||||
| `--stackit-region` | StackIT region(s) to scan | No | All available regions |
|
||||
|
||||
\* Required unless provided via environment variables.
|
||||
|
||||
### Input Validation
|
||||
|
||||
The StackIT provider performs comprehensive input validation:
|
||||
|
||||
- **Service Account Credentials**:
|
||||
- At least one of `service_account_key_path` (file path) or `service_account_key` (inline JSON) must be supplied; both empty raises `StackITNonExistentTokenError`
|
||||
- When both are provided the inline content takes precedence
|
||||
- The key file path is logged as-is; the inline content is redacted in the credentials box
|
||||
|
||||
- **Project ID**:
|
||||
- Must not be empty
|
||||
- Must be a valid UUID format (e.g., `12345678-1234-1234-1234-123456789abc`)
|
||||
- Validated using Python's UUID constructor
|
||||
|
||||
Invalid credentials will result in clear error messages before any API calls are made.
|
||||
|
||||
## Available Services
|
||||
|
||||
### IaaS (Infrastructure as a Service)
|
||||
|
||||
- **Service Class:** `IaaSService`
|
||||
- **Location:** [`prowler/providers/stackit/services/iaas/iaas_service.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/stackit/services/iaas/iaas_service.py)
|
||||
- **SDK:** Uses the [stackit-iaas](https://pypi.org/project/stackit-iaas/) Python SDK
|
||||
- **Purpose:** Manages IaaS resources including security groups, servers, and network interfaces.
|
||||
|
||||
**Supported Resources:**
|
||||
- Security Groups and Rules
|
||||
- Servers (Virtual Machines)
|
||||
- Network Interfaces (NICs)
|
||||
|
||||
**Key Features:**
|
||||
- Automatic discovery of all security groups in the project
|
||||
- Security rule parsing with support for unrestricted access detection
|
||||
- Network interface analysis to determine whether security groups are in use
|
||||
- By default, reports only security groups attached to at least one NIC; `--scan-unused-services` includes unused security groups too
|
||||
|
||||
## Available Checks
|
||||
|
||||
The StackIT provider currently implements 4 security checks focused on network security:
|
||||
|
||||
### 1. iaas_security_group_ssh_unrestricted
|
||||
|
||||
- **Severity:** High
|
||||
- **Description:** Detects security groups that allow unrestricted SSH access (port 22) from the internet.
|
||||
- **Risk:** Unrestricted SSH access increases the attack surface and risk of brute-force attacks.
|
||||
- **Detection Logic:**
|
||||
- Checks for ingress rules allowing TCP port 22
|
||||
- Flags rules with `ip_range=None` or `ip_range="0.0.0.0/0"` or `ip_range="::/0"`
|
||||
- Reports security groups attached to NICs by default, or all security groups when `--scan-unused-services` is enabled
|
||||
|
||||
### 2. iaas_security_group_rdp_unrestricted
|
||||
|
||||
- **Severity:** High
|
||||
- **Description:** Detects security groups that allow unrestricted RDP access (port 3389) from the internet.
|
||||
- **Risk:** Unrestricted RDP access enables potential unauthorized remote desktop access.
|
||||
- **Detection Logic:**
|
||||
- Checks for ingress rules allowing TCP port 3389
|
||||
- Flags unrestricted IP ranges (None, 0.0.0.0/0, ::/0)
|
||||
- Reports security groups attached to NICs by default, or all security groups when `--scan-unused-services` is enabled
|
||||
|
||||
### 3. iaas_security_group_database_unrestricted
|
||||
|
||||
- **Severity:** High
|
||||
- **Description:** Detects security groups that allow unrestricted access to common database ports.
|
||||
- **Monitored Ports:**
|
||||
- MySQL: 3306
|
||||
- PostgreSQL: 5432
|
||||
- MongoDB: 27017
|
||||
- Redis: 6379
|
||||
- SQL Server: 1433
|
||||
- CouchDB: 5984
|
||||
- **Risk:** Unrestricted database access can lead to data breaches and unauthorized data access.
|
||||
|
||||
### 4. iaas_security_group_all_traffic_unrestricted
|
||||
|
||||
- **Severity:** Critical
|
||||
- **Description:** Detects security groups that allow all traffic from the internet.
|
||||
- **Detection Logic:**
|
||||
- Checks for rules with `port_range=None` (all ports)
|
||||
- Checks for rules with port range covering 0-65535 or 1-65535
|
||||
- Flags unrestricted IP ranges
|
||||
- Critical security misconfiguration requiring immediate remediation
|
||||
|
||||
### Important Implementation Notes
|
||||
|
||||
**Self-Referencing Security Group Rules:**
|
||||
|
||||
Security group rules with `remoteSecurityGroupId` set are automatically filtered out from unrestricted access checks. These rules only allow traffic from instances within the same security group (self-referencing), not from the internet, and are therefore not flagged as security risks.
|
||||
|
||||
**Rule Display Names:**
|
||||
|
||||
All findings include user-friendly rule descriptions when available. If a security group rule has a description field set (the name shown in the StackIT UI), it will be displayed in the finding message along with the rule ID:
|
||||
- With description: `'Allow SSH from office' (sgr-abc123)`
|
||||
- Without description: `'sgr-abc123'`
|
||||
|
||||
**Network Interface (NIC) Usage Filtering:**
|
||||
|
||||
The IaaS service lists project NICs and records the security group IDs attached to them. Checks use that signal to decide whether a security group is in use:
|
||||
|
||||
1. **Default behavior:** Report security groups attached to at least one NIC.
|
||||
2. **`--scan-unused-services`:** Report every security group, including unused ones.
|
||||
3. **FAIL logic:** Internet exposure is driven by security group rules that allow unrestricted source ranges, not by the presence of a public IP on the NIC.
|
||||
|
||||
**Unrestricted IP Ranges:**
|
||||
|
||||
The StackIT API represents "unrestricted" in two ways:
|
||||
- **`ip_range=null`**: No IP restriction specified (implicit unrestricted)
|
||||
- **`ip_range="0.0.0.0/0"` or `"::/0"`**: Explicitly configured to allow all IPs
|
||||
|
||||
Both are flagged as unrestricted. A `null` value is **more permissive** than an explicit range and applies to all protocols/ports if other fields are also `null`.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Python Version
|
||||
|
||||
- **Minimum:** Python 3.10+
|
||||
- **Reason:** The StackIT SDK requires Python 3.10 or higher
|
||||
|
||||
### Dependencies
|
||||
|
||||
The StackIT provider requires the following Python packages (automatically installed with Prowler):
|
||||
|
||||
- **stackit-core** (v0.2.0): Core SDK for StackIT API authentication and configuration
|
||||
- **stackit-iaas** (v1.4.0): IaaS service SDK for managing compute resources
|
||||
- **stackit-resourcemanager** (v0.8.0): Resource Manager SDK for fetching project metadata (e.g., project names)
|
||||
|
||||
These dependencies are defined in `pyproject.toml` and installed automatically with:
|
||||
|
||||
```bash
|
||||
poetry install
|
||||
```
|
||||
|
||||
**Note:** The `stackit-resourcemanager` package enables automatic retrieval of project names for display in reports. If this package is not available, Prowler will still function normally but project names will be empty in the output.
|
||||
|
||||
## Region Support
|
||||
|
||||
### Supported Regions
|
||||
|
||||
- **Available Regions:** `eu01` (Germany South) and `eu02` (Austria West)
|
||||
- **Default:** All scans use both `eu01` and `eu02` regions by default.
|
||||
|
||||
### Multi-Region Scanning
|
||||
|
||||
Prowler supports scanning multiple StackIT regions in a single execution. By default, it will scan all regions defined in the `stackit_regions_by_service.json` configuration file.
|
||||
|
||||
### CLI Argument
|
||||
|
||||
You can specify which regions to scan using the `--stackit-region` argument:
|
||||
|
||||
```bash
|
||||
# Scan only eu01
|
||||
prowler stackit --stackit-region eu01
|
||||
|
||||
# Scan both eu01 and eu02
|
||||
prowler stackit --stackit-region eu01 eu02
|
||||
```
|
||||
|
||||
### Implementation Details
|
||||
|
||||
- **Regional Clients:** Prowler generates a separate API client for each audited region.
|
||||
- **Service Iteration:** Each service (e.g., IaaS) iterates through the regional clients to fetch and audit resources.
|
||||
- **Identity Tracking:** The `audited_regions` are stored in the identity model for reporting.
|
||||
|
||||
### Future Enhancements
|
||||
|
||||
As StackIT adds more regions, they can be easily added to Prowler by updating the `prowler/providers/stackit/stackit_regions_by_service.json` file without requiring code changes.
|
||||
|
||||
## Command Examples
|
||||
|
||||
### Scan Specific Regions
|
||||
|
||||
Scan only the `eu01` region:
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--stackit-region eu01
|
||||
```
|
||||
|
||||
Scan multiple regions:
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--stackit-region eu01 eu02
|
||||
```
|
||||
|
||||
### Scan Specific Checks
|
||||
|
||||
Run only SSH unrestricted check:
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--checks iaas_security_group_ssh_unrestricted
|
||||
```
|
||||
|
||||
### Scan All Security Group Checks
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--services iaas
|
||||
```
|
||||
|
||||
### Output Formats
|
||||
|
||||
Generate JSON output:
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--output-formats json
|
||||
```
|
||||
|
||||
Generate HTML report:
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--output-formats html
|
||||
```
|
||||
|
||||
## Known Limitations
|
||||
|
||||
### Current Limitations
|
||||
|
||||
1. **Single Project Scope**: Only one project can be scanned at a time
|
||||
2. **Service Coverage**: Only the IaaS service is currently implemented
|
||||
3. **Check Coverage**: Limited to security group network security checks (4 checks total)
|
||||
4. **No Compliance Frameworks**: Compliance framework mappings are not yet implemented
|
||||
|
||||
### Planned Enhancements
|
||||
|
||||
- Multi-project scanning capability
|
||||
- Additional IaaS checks (volume encryption, server public IP exposure, backup status)
|
||||
- Compliance framework mappings (CIS, custom StackIT best practices)
|
||||
- StackIT CLI remediation examples in metadata
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Authentication Errors
|
||||
|
||||
**Error:** `StackIT service account key was rejected`
|
||||
|
||||
**Solutions:**
|
||||
1. Re-issue the service account key in the StackIT Portal
|
||||
2. Verify the service account key file or inline JSON content is complete
|
||||
3. Check that the service account has the necessary permissions (`iaas.viewer` or `project.owner`)
|
||||
4. Ensure the service account key is provided through `STACKIT_SERVICE_ACCOUNT_KEY_PATH`, `STACKIT_SERVICE_ACCOUNT_KEY`, or the matching CLI arguments
|
||||
|
||||
**Error:** `StackIT credentials not found or are invalid`
|
||||
|
||||
**Solutions:**
|
||||
1. Ensure the project ID and one service account credential source are provided
|
||||
2. Check that credentials are set via environment variables or command-line arguments
|
||||
3. Verify there are no extra spaces or newlines in the credentials
|
||||
|
||||
**Error:** `Invalid StackIT project ID format`
|
||||
|
||||
**Solutions:**
|
||||
1. Verify the project ID is a valid UUID format: `12345678-1234-1234-1234-123456789abc`
|
||||
2. Copy the project ID directly from the StackIT Portal
|
||||
3. Ensure there are no extra spaces or quotes around the UUID
|
||||
|
||||
### API Connection Errors
|
||||
|
||||
**Error:** `Failed to connect to StackIT API`
|
||||
|
||||
**Solutions:**
|
||||
1. Check your internet connection
|
||||
2. Verify the StackIT API endpoint is accessible from your network
|
||||
3. Check if there are any firewall rules blocking HTTPS connections
|
||||
4. Review the full error message for specific API error codes
|
||||
|
||||
**Error:** `HTTP 403 Forbidden`
|
||||
|
||||
**Solutions:**
|
||||
1. Verify the service account has the correct permissions
|
||||
2. Ensure the project ID is correct and you have access to it
|
||||
3. Check that the service account is enabled (not disabled or expired)
|
||||
4. Verify the service account key has not been revoked
|
||||
|
||||
**Error:** `HTTP 404 Not Found`
|
||||
|
||||
**Solutions:**
|
||||
1. Verify the project ID exists and is correct
|
||||
2. Check that the IaaS service is enabled in your project
|
||||
3. Ensure you're using the correct region (eu01)
|
||||
|
||||
### Empty Results
|
||||
|
||||
**Issue:** No security groups or findings reported
|
||||
|
||||
**Solutions:**
|
||||
1. Verify that security groups exist in your project
|
||||
2. Check that the IaaS service is properly configured
|
||||
3. Ensure the service account has `iaas.viewer` permission
|
||||
4. Check Prowler logs for any API errors (use `--log-level DEBUG`)
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Enable debug logging for detailed troubleshooting:
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
|
||||
prowler stackit \
|
||||
--stackit-project-id "your-project-id" \
|
||||
--log-level DEBUG
|
||||
```
|
||||
|
||||
This will show:
|
||||
- API authentication details (with inline service account keys redacted)
|
||||
- Resource discovery progress
|
||||
- Security rule parsing details
|
||||
- Any API errors or warnings
|
||||
|
||||
## Specific Patterns in StackIT Services
|
||||
|
||||
The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations:
|
||||
|
||||
- Directly in the code, in location [`prowler/providers/stackit/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/stackit/services)
|
||||
- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view.
|
||||
|
||||
The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other StackIT services as reference.
|
||||
|
||||
### StackIT Service Common Patterns
|
||||
|
||||
- Services communicate with StackIT using the StackIT Python SDK, you can find the documentation [here](https://github.com/stackitcloud/stackit-sdk-python).
|
||||
- Service constructors receive a `StackitProvider` instance and use it to access credentials, identity, and configuration.
|
||||
- The provider builds StackIT SDK `Configuration` objects from the service account key path or inline key content.
|
||||
- Resource containers **must** be initialized in the constructor, typically as lists or dictionaries.
|
||||
- Do not manipulate `os.environ` for credentials inside services. Use the provider session and SDK configuration helpers.
|
||||
- All StackIT resources are represented as Pydantic `BaseModel` classes, providing type safety and structured access to resource attributes.
|
||||
- StackIT SDK calls are wrapped in try/except blocks, with specific handling for API errors, always logging errors.
|
||||
- **Centralized Error Handling**: Use `provider.handle_api_error(exception)` for consistent authentication error detection across all services.
|
||||
- **SDK Warning Suppression**: StackIT SDK prints deprecation warnings to stderr - use the `suppress_stderr()` context manager during SDK initialization and API calls.
|
||||
- **Unrestricted Access Detection**: In StackIT API, `None` values mean "allow all" (more permissive than explicit 0.0.0.0/0).
|
||||
- `protocol=None` → All protocols allowed
|
||||
- `ip_range=None` → All source IPs allowed (unrestricted!)
|
||||
- `port_range=None` → All ports allowed
|
||||
- `remote_security_group_id` set → Only allows traffic from the same security group (not unrestricted!)
|
||||
|
||||
### IaaS Service Specific Patterns
|
||||
|
||||
**Security Group Discovery:**
|
||||
```python
|
||||
# List all security groups
|
||||
security_groups = client.list_security_groups(
|
||||
project_id=self.project_id,
|
||||
region=region,
|
||||
)
|
||||
|
||||
# List network interfaces to determine security group usage
|
||||
nics = client.list_project_nics(
|
||||
project_id=self.project_id,
|
||||
region=region,
|
||||
)
|
||||
|
||||
# Checks report in-use security groups by default. Use --scan-unused-services
|
||||
# to include security groups that are not attached to any NIC.
|
||||
```
|
||||
|
||||
**Centralized Authentication Error Handling:**
|
||||
```python
|
||||
def _handle_api_call(self, api_function, *args, **kwargs):
|
||||
"""Wrapper for API calls with centralized error handling."""
|
||||
try:
|
||||
with suppress_stderr(): # Suppress SDK warnings
|
||||
return api_function(*args, **kwargs)
|
||||
except Exception as e:
|
||||
# Use centralized error handler from provider
|
||||
self.provider.handle_api_error(e) # Detects 401 and raises StackITInvalidTokenError
|
||||
```
|
||||
|
||||
**Unrestricted Access Detection:**
|
||||
```python
|
||||
def is_unrestricted(rule):
|
||||
"""Check if a rule allows unrestricted access."""
|
||||
# Filter out self-referencing rules
|
||||
if rule.remote_security_group_id is not None:
|
||||
return False
|
||||
# Check for unrestricted IP ranges
|
||||
return rule.ip_range is None or rule.ip_range in ["0.0.0.0/0", "::/0"]
|
||||
|
||||
def is_tcp(rule):
|
||||
"""Check if a rule applies to TCP protocol."""
|
||||
# None means all protocols (including TCP)
|
||||
return rule.protocol is None or rule.protocol.lower() in ["tcp", "all"]
|
||||
|
||||
def includes_port(rule, port):
|
||||
"""Check if a rule includes a specific port."""
|
||||
# None means all ports
|
||||
if rule.port_range is None:
|
||||
return True
|
||||
return rule.port_range.min <= port <= rule.port_range.max
|
||||
```
|
||||
|
||||
## Specific Patterns in StackIT Checks
|
||||
|
||||
The StackIT checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks:
|
||||
|
||||
- Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/stackit/services/iaas/iaas_security_group_ssh_unrestricted/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/stackit/services/iaas/iaas_security_group_ssh_unrestricted))
|
||||
- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view.
|
||||
|
||||
The best reference to understand how to implement a new check is following the [check creation documentation](/developer-guide/checks#creating-a-check) and taking other similar StackIT checks as reference.
|
||||
|
||||
### Check Report Class
|
||||
|
||||
The `CheckReportStackIT` class models a single finding for a StackIT resource in a check report. It is defined in [`prowler/lib/check/models.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/lib/check/models.py) and inherits from the generic `Check_Report` base class.
|
||||
|
||||
#### Purpose
|
||||
|
||||
`CheckReportStackIT` extends the base report structure with StackIT-specific fields, enabling detailed tracking of the resource, project, and location associated with each finding.
|
||||
|
||||
#### Constructor and Attribute Population
|
||||
|
||||
When you instantiate `CheckReportStackIT`, you must provide the check metadata and a resource object. The class will attempt to automatically populate its StackIT-specific attributes from the resource, using the following logic:
|
||||
|
||||
- **`resource_id`**:
|
||||
- Uses `resource.id` if present.
|
||||
- Otherwise, uses `resource.resource_id` if present.
|
||||
- Defaults to an empty string if none are available.
|
||||
|
||||
- **`resource_name`**:
|
||||
- Uses `resource.name` if present.
|
||||
- Defaults to an empty string if not available.
|
||||
|
||||
- **`project_id`**:
|
||||
- Uses `resource.project_id` if present.
|
||||
- Defaults to an empty string if not available (should be set in check logic).
|
||||
|
||||
- **`location`**:
|
||||
- Uses `resource.region` if present.
|
||||
- Otherwise, uses `resource.location` if present.
|
||||
- Defaults to an empty string if not available.
|
||||
|
||||
If the resource object does not contain the required attributes, you must set them manually in the check logic.
|
||||
|
||||
Other attributes are inherited from the `Check_Report` class, from which you **always** have to set the `status` and `status_extended` attributes in the check logic.
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```python
|
||||
from prowler.lib.check.models import CheckReportStackIT
|
||||
|
||||
report = CheckReportStackIT(
|
||||
metadata=self.metadata(),
|
||||
resource=security_group
|
||||
)
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Security group {security_group.name} allows unrestricted SSH access from the internet."
|
||||
report.resource_id = security_group.id
|
||||
report.resource_name = security_group.name
|
||||
report.project_id = security_group.project_id
|
||||
report.location = security_group.region
|
||||
```
|
||||
|
||||
### Common Check Pattern
|
||||
|
||||
```python
|
||||
from prowler.lib.check.models import Check, CheckReportStackIT
|
||||
from prowler.providers.stackit.services.iaas.iaas_client import iaas_client
|
||||
|
||||
class iaas_security_group_ssh_unrestricted(Check):
|
||||
"""Check if IaaS security groups allow unrestricted SSH access."""
|
||||
|
||||
def execute(self):
|
||||
findings = []
|
||||
|
||||
for security_group in iaas_client.security_groups:
|
||||
if not (iaas_client.scan_unused_services or security_group.in_use):
|
||||
continue
|
||||
|
||||
report = CheckReportStackIT(
|
||||
metadata=self.metadata(),
|
||||
resource=security_group
|
||||
)
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"Security group {security_group.name} does not allow unrestricted SSH access."
|
||||
|
||||
# Check each rule
|
||||
for rule in security_group.rules:
|
||||
if (rule.is_ingress() and
|
||||
rule.is_tcp() and
|
||||
rule.includes_port(22) and
|
||||
rule.is_unrestricted()):
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Security group {security_group.name} allows unrestricted SSH access from the internet."
|
||||
break
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
### Official StackIT Documentation
|
||||
|
||||
- **StackIT Portal**: [https://portal.stackit.cloud/](https://portal.stackit.cloud/)
|
||||
- **StackIT Documentation**: [https://docs.stackit.cloud/](https://docs.stackit.cloud/)
|
||||
- **StackIT API Documentation**: [https://docs.api.eu01.stackit.cloud/](https://docs.api.eu01.stackit.cloud/)
|
||||
|
||||
### Python SDK
|
||||
|
||||
- **StackIT Python SDK (GitHub)**: [https://github.com/stackitcloud/stackit-sdk-python](https://github.com/stackitcloud/stackit-sdk-python)
|
||||
- **stackit-core (PyPI)**: [https://pypi.org/project/stackit-core/](https://pypi.org/project/stackit-core/)
|
||||
- **stackit-iaas (PyPI)**: [https://pypi.org/project/stackit-iaas/](https://pypi.org/project/stackit-iaas/)
|
||||
- **IaaS Models**: [https://github.com/stackitcloud/stackit-sdk-python/tree/main/services/iaas/src/stackit/iaas/models](https://github.com/stackitcloud/stackit-sdk-python/tree/main/services/iaas/src/stackit/iaas/models)
|
||||
|
||||
### Prowler Resources
|
||||
|
||||
- **Provider Implementation**: [`prowler/providers/stackit/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/stackit/)
|
||||
- **IaaS Service**: [`prowler/providers/stackit/services/iaas/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/stackit/services/iaas/)
|
||||
- **Prowler Hub**: [https://hub.prowler.com/](https://hub.prowler.com/)
|
||||
- **GitHub Issues**: [https://github.com/prowler-cloud/prowler/issues](https://github.com/prowler-cloud/prowler/issues)
|
||||
|
||||
## Contributing
|
||||
|
||||
If you'd like to contribute to the StackIT provider:
|
||||
|
||||
1. **Add New Checks**: Follow the [check creation guide](/developer-guide/checks#creating-a-check) and use existing StackIT checks as templates
|
||||
2. **Enhance Services**: Implement additional IaaS resource discovery or add new services
|
||||
3. **Improve Documentation**: Add metadata enhancements, CLI remediation examples, or Terraform code samples
|
||||
4. **Report Issues**: Submit bug reports or feature requests on [GitHub](https://github.com/prowler-cloud/prowler/issues)
|
||||
|
||||
### Quick Start for Contributors
|
||||
|
||||
1. **Install dependencies**: `poetry install` (includes stackit-core and stackit-iaas)
|
||||
2. **Set credentials**: Export `STACKIT_SERVICE_ACCOUNT_KEY_PATH` and `STACKIT_PROJECT_ID`
|
||||
3. **Run checks**: `prowler stackit`
|
||||
4. **View code**: Start in `prowler/providers/stackit/`
|
||||
5. **Add checks**: Create new check directories under `services/iaas/`
|
||||
6. **Run tests**: `poetry run pytest tests/providers/stackit/ -v`
|
||||
|
||||
### Code Quality Standards
|
||||
|
||||
The StackIT provider should follow the same quality expectations as the rest of the Prowler SDK:
|
||||
|
||||
- Keep service and check logic covered by unit tests.
|
||||
- Redact inline service account keys from generated output.
|
||||
- Keep documentation aligned with the implemented services and checks.
|
||||
- Follow existing provider, service, and check patterns before adding StackIT-specific abstractions.
|
||||
+19
-3
@@ -124,8 +124,8 @@
|
||||
"user-guide/tutorials/prowler-app-rbac",
|
||||
"user-guide/tutorials/prowler-app-multi-tenant",
|
||||
"user-guide/tutorials/prowler-app-api-keys",
|
||||
"user-guide/tutorials/prowler-app-import-findings",
|
||||
"user-guide/tutorials/prowler-app-alerts",
|
||||
"user-guide/tutorials/prowler-import-findings",
|
||||
"user-guide/tutorials/prowler-alerts",
|
||||
{
|
||||
"group": "Mutelist",
|
||||
"expanded": true,
|
||||
@@ -339,6 +339,13 @@
|
||||
"user-guide/providers/scaleway/authentication"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "StackIT",
|
||||
"pages": [
|
||||
"user-guide/providers/stackit/getting-started-stackit",
|
||||
"user-guide/providers/stackit/authentication"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Vercel",
|
||||
"pages": [
|
||||
@@ -401,7 +408,8 @@
|
||||
"developer-guide/kubernetes-details",
|
||||
"developer-guide/m365-details",
|
||||
"developer-guide/github-details",
|
||||
"developer-guide/llm-details"
|
||||
"developer-guide/llm-details",
|
||||
"developer-guide/stackit-details"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -576,6 +584,14 @@
|
||||
{
|
||||
"source": "/contact",
|
||||
"destination": "/support"
|
||||
},
|
||||
{
|
||||
"source": "/user-guide/tutorials/prowler-app-import-findings",
|
||||
"destination": "/user-guide/tutorials/prowler-import-findings"
|
||||
},
|
||||
{
|
||||
"source": "/user-guide/tutorials/prowler-app-alerts",
|
||||
"destination": "/user-guide/tutorials/prowler-alerts"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -20,7 +20,8 @@ Refer to the [Prowler App Tutorial](/user-guide/tutorials/prowler-app) for detai
|
||||
|
||||
_Commands_:
|
||||
|
||||
```bash
|
||||
<CodeGroup>
|
||||
```bash macOS/Linux
|
||||
VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name)
|
||||
curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
@@ -28,6 +29,15 @@ Refer to the [Prowler App Tutorial](/user-guide/tutorials/prowler-app) for detai
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
```powershell Windows PowerShell
|
||||
$VERSION = (Invoke-RestMethod -Uri "https://api.github.com/repos/prowler-cloud/prowler/releases/latest").tag_name
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/docker-compose.yml" -OutFile "docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/.env" -OutFile ".env"
|
||||
docker compose up -d
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
<Callout icon="lock" iconType="regular" color="#e74c3c">
|
||||
For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
</Callout>
|
||||
@@ -118,8 +128,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.27.0"
|
||||
PROWLER_API_VERSION="5.27.0"
|
||||
PROWLER_UI_VERSION="5.29.0"
|
||||
PROWLER_API_VERSION="5.29.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -40,12 +40,6 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
pip install prowler
|
||||
prowler -v
|
||||
```
|
||||
|
||||
To upgrade Prowler to the latest version:
|
||||
|
||||
``` bash
|
||||
pip install --upgrade prowler
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="Docker">
|
||||
_Requirements_:
|
||||
@@ -170,6 +164,68 @@ To install Prowler as a Python package, use `Python >= 3.10, <= 3.12`. Prowler i
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Updating Prowler CLI
|
||||
|
||||
Upgrade Prowler CLI to the latest release using the same method chosen for installation:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="pipx">
|
||||
```bash
|
||||
pipx upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="pip">
|
||||
```bash
|
||||
pip install --upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="Docker">
|
||||
Pull the desired image tag to fetch the latest version:
|
||||
|
||||
```bash
|
||||
docker pull toniblyx/prowler:latest
|
||||
```
|
||||
|
||||
<Note>
|
||||
Replace `latest` with a specific release tag (for example, `stable` or `<x.y.z>`) to pin a version. Refer to the [Container Versions](#container-versions) section for the full list of available tags.
|
||||
</Note>
|
||||
</Tab>
|
||||
<Tab title="GitHub">
|
||||
Pull the latest changes and sync the environment:
|
||||
|
||||
```bash
|
||||
cd prowler
|
||||
git pull
|
||||
uv sync
|
||||
uv run python prowler-cli.py -v
|
||||
```
|
||||
|
||||
<Note>
|
||||
To upgrade to a specific release, check out the corresponding tag before syncing: `git checkout <x.y.z>`.
|
||||
</Note>
|
||||
</Tab>
|
||||
<Tab title="Brew">
|
||||
```bash
|
||||
brew upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="CloudShell">
|
||||
Both AWS CloudShell and Azure CloudShell install Prowler with `pipx`, so the upgrade command is the same:
|
||||
|
||||
```bash
|
||||
pipx upgrade prowler
|
||||
prowler -v
|
||||
```
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
<Note>
|
||||
To install a specific version instead of the latest release, pin it explicitly. For example, with `pipx`: `pipx install prowler==<x.y.z>`, or with `pip`: `pip install prowler==<x.y.z>`. The available releases are listed in the [Releases GitHub section](https://github.com/prowler-cloud/prowler/releases).
|
||||
</Note>
|
||||
|
||||
## Container Versions
|
||||
|
||||
The available versions of Prowler CLI are the following:
|
||||
|
||||
@@ -141,6 +141,45 @@ Choose one of the following installation methods:
|
||||
|
||||
---
|
||||
|
||||
## Updating Prowler MCP Server
|
||||
|
||||
When running Prowler MCP Server locally ("Option 2: Run Locally"), upgrade to the latest version using the same method chosen for installation. The hosted server (`https://mcp.prowler.com/mcp`) is always kept up to date by Prowler and requires no action.
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Docker">
|
||||
Pull the latest image and restart the container:
|
||||
|
||||
```bash
|
||||
docker pull prowlercloud/prowler-mcp
|
||||
```
|
||||
|
||||
<Note>
|
||||
Recreate any running container after pulling the new image so the updated version takes effect.
|
||||
</Note>
|
||||
</Tab>
|
||||
<Tab title="From Source">
|
||||
Pull the latest changes and sync the dependencies:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
git pull
|
||||
uv sync
|
||||
uv run prowler-mcp --help
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="Build Docker Image">
|
||||
Pull the latest source and rebuild the image:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
git pull
|
||||
docker build -t prowler-mcp .
|
||||
```
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Command Line Options
|
||||
|
||||
The Prowler MCP Server supports the following command-line arguments:
|
||||
|
||||
@@ -36,6 +36,7 @@ Prowler supports a wide range of providers organized by category:
|
||||
| [OpenStack](/user-guide/providers/openstack/getting-started-openstack) | Official | Projects | UI, API, CLI |
|
||||
| [Oracle Cloud](/user-guide/providers/oci/getting-started-oci) | Official | Tenancies / Compartments | UI, API, CLI |
|
||||
| [Scaleway](/user-guide/providers/scaleway/getting-started-scaleway) | [Contact us](https://prowler.com/contact) | Organizations | CLI |
|
||||
| [StackIT](/user-guide/providers/stackit/getting-started-stackit) | [Contact us](https://prowler.com/contact) | Projects | CLI |
|
||||
|
||||
### Infrastructure as Code Providers
|
||||
|
||||
|
||||
@@ -91,6 +91,7 @@ The following list includes all the Azure checks with configurable variables tha
|
||||
| `sqlserver_recommended_minimal_tls_version` | `recommended_minimal_tls_versions` | List of Strings |
|
||||
| `vm_sufficient_daily_backup_retention_period` | `vm_backup_min_daily_retention_days` | Integer |
|
||||
| `vm_desired_sku_size` | `desired_vm_sku_sizes` | List of Strings |
|
||||
| `storage_smb_channel_encryption_with_secure_algorithm` | `recommended_smb_channel_encryption_algorithms` | List of Strings |
|
||||
| `defender_attack_path_notifications_properly_configured` | `defender_attack_path_minimal_risk_level` | String |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_threshold` | Float |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_minutes` | Integer |
|
||||
@@ -165,6 +166,7 @@ The following list includes all the Okta checks with configurable variables that
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|------------------------------------|---------|
|
||||
| `application_admin_console_session_idle_timeout_15min` | `okta_admin_console_idle_timeout_max_minutes` | Integer |
|
||||
| `signon_global_session_idle_timeout_15min` | `okta_max_session_idle_minutes` | Integer |
|
||||
|
||||
## Config YAML File Structure
|
||||
@@ -534,6 +536,18 @@ azure:
|
||||
"1.3"
|
||||
]
|
||||
|
||||
# Azure Storage
|
||||
# azure.storage_smb_channel_encryption_with_secure_algorithm
|
||||
# List of SMB channel encryption algorithms allowed on file shares. A storage
|
||||
# account passes only if every enabled algorithm is in this list. Defaults to
|
||||
# the value required by CIS (AES-256-GCM only, excluding weaker AES-128 ciphers).
|
||||
recommended_smb_channel_encryption_algorithms:
|
||||
[
|
||||
"AES-256-GCM",
|
||||
# "AES-128-CCM",
|
||||
# "AES-128-GCM",
|
||||
]
|
||||
|
||||
# Azure Virtual Machines
|
||||
# azure.vm_desired_sku_size
|
||||
# List of desired VM SKU sizes that are allowed in the organization
|
||||
|
||||
@@ -6,7 +6,7 @@ title: 'Run Prowler in CI/CD and Send Findings to Prowler Cloud'
|
||||
For new projects, use the official [Prowler GitHub Action](/user-guide/tutorials/prowler-app-github-action) — a Docker-based reusable action that runs scans, optionally pushes findings to Prowler Cloud, and uploads SARIF results to GitHub Code Scanning. The GitHub Actions examples below document the legacy pip-based flow.
|
||||
</Warning>
|
||||
|
||||
This cookbook demonstrates how to integrate Prowler into CI/CD pipelines so that security scans run automatically and findings are sent to Prowler Cloud via [Import Findings](/user-guide/tutorials/prowler-app-import-findings). Examples cover GitHub Actions and GitLab CI.
|
||||
This cookbook demonstrates how to integrate Prowler into CI/CD pipelines so that security scans run automatically and findings are sent to Prowler Cloud via [Import Findings](/user-guide/tutorials/prowler-import-findings). Examples cover GitHub Actions and GitLab CI.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -19,7 +19,7 @@ This cookbook demonstrates how to integrate Prowler into CI/CD pipelines so that
|
||||
|
||||
Prowler CLI provides the `--push-to-cloud` flag, which uploads scan results directly to Prowler Cloud after a scan completes. Combined with the `PROWLER_CLOUD_API_KEY` environment variable, this enables fully automated ingestion without manual file uploads.
|
||||
|
||||
For full details on the flag and API, refer to the [Import Findings](/user-guide/tutorials/prowler-app-import-findings) documentation.
|
||||
For full details on the flag and API, refer to the [Import Findings](/user-guide/tutorials/prowler-import-findings) documentation.
|
||||
|
||||
<Note>
|
||||
The examples in this guide use AWS as the target provider, but the same approach applies to any provider supported by Prowler (Azure, GCP, Kubernetes, and others). Replace `prowler aws` with the desired provider command (e.g., `prowler gcp`, `prowler azure`) and configure the corresponding credentials in the CI/CD environment.
|
||||
@@ -195,7 +195,7 @@ By default, Prowler exits with a non-zero code when it finds failing checks. Thi
|
||||
* **GitLab CI**: Add `allow_failure: true` to the job
|
||||
|
||||
<Note>
|
||||
Ingestion failures (e.g., network issues reaching Prowler Cloud) do not affect the Prowler exit code. The scan completes normally and only a warning is emitted. See [Import Findings troubleshooting](/user-guide/tutorials/prowler-app-import-findings#troubleshooting) for details.
|
||||
Ingestion failures (e.g., network issues reaching Prowler Cloud) do not affect the Prowler exit code. The scan completes normally and only a warning is emitted. See [Import Findings troubleshooting](/user-guide/tutorials/prowler-import-findings#troubleshooting) for details.
|
||||
</Note>
|
||||
|
||||
### Caching Prowler Installation
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
title: 'Run Kubernetes In-Cluster and Send Findings to Prowler Cloud'
|
||||
---
|
||||
|
||||
This cookbook walks through deploying Prowler inside a Kubernetes cluster on a recurring schedule and automatically sending findings to Prowler Cloud via [Import Findings](/user-guide/tutorials/prowler-app-import-findings). By the end, security scan results from the cluster appear in Prowler Cloud without any manual file uploads.
|
||||
This cookbook walks through deploying Prowler inside a Kubernetes cluster on a recurring schedule and automatically sending findings to Prowler Cloud via [Import Findings](/user-guide/tutorials/prowler-import-findings). By the end, security scan results from the cluster appear in Prowler Cloud without any manual file uploads.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -181,7 +181,7 @@ Once the job completes and findings are pushed:
|
||||
2. Open the "Scans" section to verify the ingestion job status
|
||||
3. Browse findings under the Kubernetes provider
|
||||
|
||||
For details on the ingestion workflow and status tracking, refer to the [Import Findings](/user-guide/tutorials/prowler-app-import-findings) documentation.
|
||||
For details on the ingestion workflow and status tracking, refer to the [Import Findings](/user-guide/tutorials/prowler-import-findings) documentation.
|
||||
|
||||
## Tips and Troubleshooting
|
||||
|
||||
@@ -204,4 +204,4 @@ For details on the ingestion workflow and status tracking, refer to the [Import
|
||||
--namespace prowler-ns
|
||||
```
|
||||
|
||||
* **Failed uploads**: If the push to Prowler Cloud fails, the scan still completes and findings are saved locally in the container. Check the [Import Findings troubleshooting section](/user-guide/tutorials/prowler-app-import-findings#troubleshooting) for common error messages.
|
||||
* **Failed uploads**: If the push to Prowler Cloud fails, the scan still completes and findings are saved locally in the container. Check the [Import Findings troubleshooting section](/user-guide/tutorials/prowler-import-findings#troubleshooting) for common error messages.
|
||||
|
||||
@@ -18,7 +18,7 @@ Prowler requests the following read-only OAuth 2.0 scopes:
|
||||
| `https://www.googleapis.com/auth/admin.directory.domain.readonly` | Read access to domain information |
|
||||
| `https://www.googleapis.com/auth/admin.directory.customer.readonly` | Read access to customer information (Customer ID) |
|
||||
| `https://www.googleapis.com/auth/admin.directory.orgunit.readonly` | Read access to organizational unit hierarchy (identifies the root OU for policy filtering) |
|
||||
| `https://www.googleapis.com/auth/cloud-identity.policies.readonly` | Read access to domain-level application policies (required for Calendar, Gmail, Chat, and Drive service checks) |
|
||||
| `https://www.googleapis.com/auth/cloud-identity.policies.readonly` | Read access to domain-level application policies (required for Calendar, Chat, Drive, Gmail, Groups, Marketplace, Security, and Sites service checks) |
|
||||
| `https://www.googleapis.com/auth/admin.directory.rolemanagement.readonly` | Read access to admin roles and role assignments |
|
||||
|
||||
<Warning>
|
||||
@@ -40,7 +40,7 @@ In the [Google Cloud Console](https://console.cloud.google.com), select the targ
|
||||
| API | Required For |
|
||||
|-----|--------------|
|
||||
| **Admin SDK API** | Directory service checks (users, roles, domains) |
|
||||
| **Cloud Identity API** | Calendar, Gmail, Chat, and Drive service checks (domain-level application policies) |
|
||||
| **Cloud Identity API** | All service checks except Directory (domain-level application policies) |
|
||||
|
||||
For each API:
|
||||
|
||||
@@ -49,7 +49,7 @@ For each API:
|
||||
3. Click **Enable**
|
||||
|
||||
<Note>
|
||||
Both APIs must be enabled in the same GCP project that hosts the Service Account. Calendar, Gmail, Chat, and Drive checks will return no findings if the Cloud Identity API is not enabled.
|
||||
Both APIs must be enabled in the same GCP project that hosts the Service Account. All service checks except Directory will return no findings if the Cloud Identity API is not enabled.
|
||||
</Note>
|
||||
|
||||
### Step 3: Create a Service Account
|
||||
@@ -178,7 +178,7 @@ If Prowler connects but returns empty results or permission errors for specific
|
||||
|
||||
### Policy API Checks Return No Findings
|
||||
|
||||
If the Directory checks run successfully but the Calendar, Gmail, Chat, or Drive checks return no findings, the Cloud Identity Policy API is not reachable for this Service Account. Verify:
|
||||
If the Directory checks run successfully but other service checks (Calendar, Chat, Drive, Gmail, Groups, Marketplace, Security, Sites) return no findings, the Cloud Identity Policy API is not reachable for this Service Account. Verify:
|
||||
|
||||
- The **Cloud Identity API** is enabled in the GCP project hosting the Service Account (Step 2)
|
||||
- The scope `https://www.googleapis.com/auth/cloud-identity.policies.readonly` is included in the Domain-Wide Delegation OAuth scopes list in the Admin Console (Step 5)
|
||||
|
||||
@@ -30,25 +30,65 @@ If a different authentication method is needed (SSWS API token, OAuth with user
|
||||
|
||||
### Required OAuth Scopes
|
||||
|
||||
The bundled signon checks require the following read-only scopes:
|
||||
The bundled checks require the following read-only scopes:
|
||||
|
||||
- `okta.policies.read`
|
||||
- `okta.brands.read`
|
||||
- `okta.apps.read`
|
||||
- `okta.authenticators.read`
|
||||
- `okta.networkZones.read`
|
||||
- `okta.apiTokens.read`
|
||||
- `okta.roles.read`
|
||||
- `okta.groups.read`
|
||||
- `okta.logStreams.read`
|
||||
- `okta.idps.read`
|
||||
|
||||
Additional scopes will be needed as more services and checks are added. These are the current ones needed:
|
||||
|
||||
| Scope | Used by |
|
||||
|---|---|
|
||||
| `okta.policies.read` | Sign-on / password / authentication policies |
|
||||
| `okta.policies.read` | Sign-on, password, authentication, and `USER_LIFECYCLE` (Workflow > Automations) policies |
|
||||
| `okta.brands.read` | Sign-in page customizations (DOD Notice and Consent Banner check) |
|
||||
| `okta.apps.read` | First-party app settings (Okta Admin Console session), integrated app inventory, and the Authentication Policies bound to Okta applications |
|
||||
| `okta.authenticators.read` | Okta authenticator configuration, including Okta Verify and Smart Card IdP |
|
||||
| `okta.networkZones.read` | Network Zone inventory, anonymized-proxy blocklist checks, and API token Network Zone validation |
|
||||
| `okta.apiTokens.read` | API token metadata and token network conditions |
|
||||
| `okta.roles.read` | Admin role assignments for API token owners (both direct and group-inherited) |
|
||||
| `okta.groups.read` | Group memberships of API token owners, used to resolve admin roles inherited via group assignment (e.g. Super Admin granted through the default admin group) |
|
||||
| `okta.logStreams.read` | Log Stream configuration (`/api/v1/logStreams`) |
|
||||
| `okta.idps.read` | Identity Providers, including Smart Card (X509) IdPs (`/api/v1/idps`) |
|
||||
|
||||
### Required Admin Role
|
||||
|
||||
The service application must be assigned the built-in **Read-Only Administrator** role.
|
||||
The service application must be assigned **one** of the following Okta admin roles:
|
||||
|
||||
Okta's Management API enforces a two-layer authorization model: an OAuth **scope** decides which API endpoints the token can call, and an **admin role** decides whether the call returns data. With only a scope granted, the token mint succeeds but every read returns `403 Forbidden`. The Read-Only Administrator role is the minimum that lets the granted `okta.*.read` scopes actually return configuration data to Prowler's checks — without it, the credential probe at provider startup fails and the scan never gets to evaluate any check.
|
||||
- **Read-Only Administrator** — covers every `signon` check and runs `application_authentication_policy_network_zone_enforced` against the apps it can see. **Visibility caveat:** under Read-Only Administrator the `/api/v1/apps` endpoint returns only the apps the service application is itself assigned to — typically just the service app's own row (for example, `Prowler Scanner`). The check still produces a finding for that app, but the rest of the org's app inventory is invisible at this role level.
|
||||
- **Super Administrator** — required additionally to evaluate five application-service checks that target Okta's first-party apps (Okta Admin Console, Okta Dashboard). With Super Administrator, `application_authentication_policy_network_zone_enforced` also evaluates the full org-wide app inventory instead of the service-app-only slice.
|
||||
|
||||
Read-Only Administrator is intentionally the narrowest role that satisfies this requirement and aligns with the least-privilege guidance in DISA STIG.
|
||||
Okta's Management API enforces a two-layer authorization model: an OAuth **scope** decides which API endpoints the token can call, and an **admin role** decides whether the call returns data. With only a scope granted, the token mint succeeds but every read returns `403 Forbidden`. Read-Only Administrator is the minimum role that lets the granted `okta.*.read` scopes return configuration data to Prowler's checks; without it, the credential probe at provider startup fails and the scan never gets to evaluate any check.
|
||||
|
||||
#### When Super Administrator is required
|
||||
|
||||
Four checks need to resolve the Authentication Policy bound to Okta's first-party apps (Okta Admin Console, Okta Dashboard) and depend on `/api/v1/apps` returning those system apps — which Okta restricts to Super Administrator:
|
||||
|
||||
| Check | STIG |
|
||||
|---|---|
|
||||
| `application_admin_console_mfa_required` | V-273193 |
|
||||
| `application_admin_console_phishing_resistant_authentication` | V-273191 |
|
||||
| `application_dashboard_mfa_required` | V-273194 |
|
||||
| `application_dashboard_phishing_resistant_authentication` | V-273190 |
|
||||
|
||||
Okta filters the first-party apps (`saasure`, `okta_enduser`) out of `/api/v1/apps` for every role below Super Administrator, so `okta.apps.read` alone is not enough. The `okta.apps.manageFirstPartyApps` permission exists only in the paid Okta Identity Governance role `ACCESS_REQUESTS_ADMIN` and cannot be added to custom roles ([Okta Permissions Catalog](https://developer.okta.com/docs/api/openapi/okta-management/guides/permissions)).
|
||||
|
||||
A fifth check — `application_admin_console_session_idle_timeout_15min` (STIG V-273187) — also requires Super Administrator: it calls `GET /api/v1/first-party-app-settings/admin-console`, which returns `403 E0000006` for every role below Super Administrator.
|
||||
|
||||
`user_inactivity_automation_35d_enabled` (STIG V-273188) reads `USER_LIFECYCLE` policies (`list_policies(type='USER_LIFECYCLE')`) using the `okta.policies.read` scope. The Read-Only Administrator role is enough to list them; no Super Administrator requirement.
|
||||
|
||||
When the service app runs with Read-Only Administrator, the checks listed in this section return **MANUAL** instead of PASS/FAIL — the rest of the scan keeps running.
|
||||
|
||||
<Note>
|
||||
Read-Only Administrator stays the recommended default for the least-privilege framing that aligns with DISA STIG. Assign Super Administrator on a separate run when full coverage of the first-party app checks is needed.
|
||||
</Note>
|
||||
|
||||
## Step-by-Step Setup
|
||||
|
||||
@@ -98,19 +138,21 @@ Okta displays the private key **only once**. If you close the modal without copy
|
||||
|
||||
### 5. Grant the required OAuth scopes
|
||||
|
||||
On the app, open the **Okta API Scopes** tab and click **Grant** on every scope Prowler needs. The bundled signon checks require `okta.policies.read` and `okta.brands.read`.
|
||||
On the app, open the **Okta API Scopes** tab and click **Grant** on every scope Prowler needs. The bundled checks require `okta.policies.read`, `okta.brands.read`, `okta.apps.read`, `okta.authenticators.read`, `okta.networkZones.read`, `okta.apiTokens.read`, `okta.roles.read`, `okta.groups.read`, `okta.logStreams.read`, and `okta.idps.read`.
|
||||
|
||||

|
||||
|
||||
### 6. Assign the Read-Only Administrator role
|
||||
### 6. Assign an admin role
|
||||
|
||||
On the app, open the **Admin roles** tab and click **Edit assignments → Add assignment**:
|
||||
|
||||
- **Role:** Read-Only Administrator
|
||||
- **Role:** Read-Only Administrator (default) — covers every `signon` check and runs the per-app network-zone check against the apps the service app can see (typically only the service app's own row).
|
||||
- **Resources:** All resources
|
||||
|
||||
Save the changes.
|
||||
|
||||
To additionally evaluate the first-party application checks (Okta Admin Console / Okta Dashboard idle timeout, MFA, and phishing-resistant authentication) and to widen the per-app network-zone check to the full org-wide app inventory, assign **Super Administrator** instead. Without Super Administrator, the five first-party checks return MANUAL and the network-zone check is limited to the service app's own visibility — the rest of the scan still runs. See [Required Admin Role](#required-admin-role) for the full breakdown.
|
||||
|
||||

|
||||
|
||||
### 7. [Optional] Verify DPoP setting
|
||||
@@ -132,8 +174,8 @@ export OKTA_PRIVATE_KEY_FILE="/secure/path/to/prowler-okta.pem"
|
||||
# or
|
||||
export OKTA_PRIVATE_KEY="$(cat /secure/path/to/prowler-okta.pem)"
|
||||
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read"
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
|
||||
uv run python prowler-cli.py okta
|
||||
```
|
||||
@@ -174,8 +216,12 @@ Prowler validates credentials at startup by listing one sign-on policy. This err
|
||||
|
||||
Raised when the credential probe succeeds at the OAuth layer but the request is rejected because the service app lacks the required scope or admin role:
|
||||
|
||||
- **`invalid_scope`** — one of the requested scopes (`okta.policies.read` or `okta.brands.read`) is not granted on the service app. Grant the missing scope from **Okta API Scopes**.
|
||||
- **`Forbidden` / `not authorized`** — the **Read-Only Administrator** role is not assigned to the service app. Assign it from **Admin roles**.
|
||||
- **`invalid_scope`** — one of the requested scopes (`okta.policies.read`, `okta.brands.read`, `okta.apps.read`, `okta.authenticators.read`, `okta.networkZones.read`, `okta.apiTokens.read`, `okta.roles.read`, `okta.groups.read`, `okta.logStreams.read`, and `okta.idps.read`) is not granted on the service app. Grant the missing scope from **Okta API Scopes**.
|
||||
- **`Forbidden` / `not authorized`** — no admin role is assigned to the service app. Assign **Read-Only Administrator** (or **Super Administrator** for the first-party application checks) from **Admin roles**.
|
||||
|
||||
### Application-service checks return MANUAL on first-party apps
|
||||
|
||||
When the service app runs with Read-Only Administrator, the five application-service checks targeting the Okta Admin Console and Okta Dashboard return MANUAL. This is by design — Okta restricts the underlying endpoints (`/api/v1/first-party-app-settings/{appName}` and `/api/v1/apps` for first-party app `name` values `saasure` / `okta_enduser`) to **Super Administrator**. Assign the Super Administrator role to the service app to evaluate those checks. See [Required Admin Role](#required-admin-role) for the full list.
|
||||
|
||||
### `invalid_dpop_proof`
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ Set up authentication for Okta with the [Okta Authentication](/user-guide/provid
|
||||
|
||||
- An Okta organization. The UI examples below use **Identity Engine** terminology such as **Global Session Policy**; Classic Engine exposes the equivalent sign-on policy concepts under older names.
|
||||
- A **Super Administrator** account on that organization for the one-time service-app setup.
|
||||
- An **API Services** app integration in the Okta Admin Console with the `okta.policies.read` and `okta.brands.read` scopes granted and the **Read-Only Administrator** role assigned.
|
||||
- An **API Services** app integration in the Okta Admin Console with the `okta.policies.read`, `okta.brands.read`, `okta.apps.read`, `okta.authenticators.read`, `okta.networkZones.read`, `okta.apiTokens.read`, `okta.roles.read`, `okta.groups.read`, `okta.logStreams.read`, and `okta.idps.read` scopes granted and an admin role assigned. **Read-Only Administrator** covers the Sign-On, Network, API Token, User, System Log, and Identity Provider checks, and runs the per-app application network-zone check against the apps the service app can see (under Read-Only Administrator that is typically only the service app's own row — the rest of the org's app inventory stays invisible). **Super Administrator** is required additionally to evaluate the five first-party application checks (Okta Admin Console / Okta Dashboard idle timeout, MFA, phishing-resistant authentication) and to widen the application network-zone check to the full app inventory — see [Okta Authentication](/user-guide/providers/okta/authentication#required-admin-role) for the full breakdown.
|
||||
- Python 3.10+ and Prowler 5.27.0 or later installed locally.
|
||||
|
||||
<CardGroup cols={2}>
|
||||
@@ -85,8 +85,8 @@ Follow the [Okta Authentication](/user-guide/providers/okta/authentication) guid
|
||||
export OKTA_ORG_DOMAIN="acme.okta.com"
|
||||
export OKTA_CLIENT_ID="0oa1234567890abcdef"
|
||||
export OKTA_PRIVATE_KEY_FILE="/secure/path/to/prowler-okta.pem"
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read"
|
||||
# Optional — defaults to "okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read"
|
||||
```
|
||||
|
||||
The private key file may contain either a PEM-encoded RSA key or a JWK JSON document.
|
||||
@@ -128,6 +128,9 @@ okta:
|
||||
# okta.signon_global_session_idle_timeout_15min
|
||||
# Defaults to 15 minutes per DISA STIG V-273186.
|
||||
okta_max_session_idle_minutes: 15
|
||||
# okta.application_admin_console_session_idle_timeout_15min
|
||||
# Defaults to 15 minutes per DISA STIG V-273187.
|
||||
okta_admin_console_idle_timeout_max_minutes: 15
|
||||
```
|
||||
|
||||
To use a custom configuration:
|
||||
@@ -140,9 +143,16 @@ prowler okta --config-file /path/to/config.yaml
|
||||
|
||||
Prowler for Okta includes security checks across the following services:
|
||||
|
||||
| Service | Description |
|
||||
| ----------- | ----------------------------------------------------------------------------------- |
|
||||
| **Sign-On** | Global session policy controls (idle timeout, lifetime, rule priority and ordering) |
|
||||
| Service | Description |
|
||||
| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Sign-On** | Global session policy controls (idle timeout, lifetime, rule priority and ordering) |
|
||||
| **Application** | Okta Admin Console sign-on settings plus Authentication Policy controls for Okta applications (session idle, MFA, phishing resistance, network zones) |
|
||||
| **Authenticator** | Password Policy controls plus Okta Verify FIPS and Smart Card IdP authenticator status |
|
||||
| **Network** | Network Zone blocklists for anonymized proxy sources |
|
||||
| **API Token** | API token owner-role validation and Network Zone restrictions |
|
||||
| **User** | User lifecycle automations (inactivity-based deprovisioning) |
|
||||
| **System Log** | Log Stream configuration that off-loads audit records to a central SIEM |
|
||||
| **Identity Provider** | Identity Providers, including Smart Card (X509) IdP status and certificate-chain visibility |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -154,21 +164,29 @@ This is stricter than simply finding the same timeout value somewhere else in th
|
||||
|
||||
### Default Scopes
|
||||
|
||||
Prowler requests a fixed set of OAuth scopes on every token exchange. The defaults cover the bundled signon checks:
|
||||
Prowler requests a fixed set of OAuth scopes on every token exchange. The defaults cover every bundled check across the Sign-On, Application, Authenticator, Network, API Token, User, System Log, and Identity Provider services:
|
||||
|
||||
- `okta.policies.read`
|
||||
- `okta.brands.read`
|
||||
- `okta.apps.read`
|
||||
- `okta.authenticators.read`
|
||||
- `okta.networkZones.read`
|
||||
- `okta.apiTokens.read`
|
||||
- `okta.roles.read`
|
||||
- `okta.groups.read`
|
||||
- `okta.logStreams.read`
|
||||
- `okta.idps.read`
|
||||
|
||||
The service app must have these scopes granted in the **Okta API Scopes** tab. When the granted set is narrower than the requested set, the token request fails with an `invalid_scope` error and the scan stops at provider initialization.
|
||||
The service app must have these scopes granted in the **Okta API Scopes** tab. `okta.groups.read` is required so the API token Super Admin check can resolve admin roles inherited via group membership; without it the check falls back to direct-only role assignments and emits a best-effort caveat. When the granted set is narrower than the requested set, the token request fails with an `invalid_scope` error and the scan stops at provider initialization.
|
||||
|
||||
When additional checks are enabled — or when running against a service app that exposes a different scope set — override the default with `OKTA_SCOPES` (comma-separated string for the env var) or `--okta-scopes` (space-separated list for the CLI):
|
||||
|
||||
```bash
|
||||
# Environment variable — comma-separated
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.users.read"
|
||||
export OKTA_SCOPES="okta.policies.read,okta.brands.read,okta.apps.read,okta.authenticators.read,okta.networkZones.read,okta.apiTokens.read,okta.roles.read,okta.groups.read,okta.logStreams.read,okta.idps.read,okta.users.read"
|
||||
|
||||
# CLI flag — space-separated
|
||||
prowler okta --okta-scopes okta.policies.read okta.brands.read okta.apps.read okta.users.read
|
||||
prowler okta --okta-scopes okta.policies.read okta.brands.read okta.apps.read okta.authenticators.read okta.networkZones.read okta.apiTokens.read okta.roles.read okta.groups.read okta.logStreams.read okta.idps.read okta.users.read
|
||||
```
|
||||
|
||||
For the full catalog of OAuth scopes exposed by the Okta Management API, refer to the [Okta OAuth 2.0 scopes documentation](https://developer.okta.com/docs/api/oauth2/).
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
---
|
||||
title: 'StackIT Authentication'
|
||||
---
|
||||
|
||||
Prowler authenticates with StackIT using a **service account key file**. The StackIT SDK signs the RSA challenge in the key file and mints/refreshes access tokens internally for the life of the scan, so no manual token rotation is needed.
|
||||
|
||||
## Service Account Key
|
||||
|
||||
StackIT uses RSA key-pair based service account keys. They are issued once, must be stored securely, and are read by the SDK on every scan to mint short-lived access tokens transparently.
|
||||
|
||||
### Option 1: Create the Key via the StackIT Portal
|
||||
|
||||
1. Open the [StackIT Portal](https://portal.stackit.cloud/) and select your project.
|
||||
2. In the left sidebar, click **Service Accounts**.
|
||||
3. Create a service account if you do not have one already. Assign:
|
||||
- `iaas.viewer` for the IaaS security group checks currently shipped, or
|
||||
- `project.owner` if you want to cover any future service Prowler adds.
|
||||
4. Open the service account and go to **Service Account Keys**.
|
||||
5. Click **Create key** and choose **STACKIT-generated key pair** (recommended). Download the resulting JSON file and store it securely (for example, `~/.stackit/sa-key.json`). The private material is only shown once.
|
||||
|
||||
### Option 2: Create the Key via the StackIT CLI
|
||||
|
||||
```bash
|
||||
# Install the StackIT CLI from https://github.com/stackitcloud/stackit-cli first
|
||||
stackit service-account key create --email my-service-account@example.com
|
||||
```
|
||||
|
||||
## Project ID
|
||||
|
||||
Your StackIT project ID is a UUID. You can find it in:
|
||||
|
||||
1. The portal URL when viewing the project: `https://portal.stackit.cloud/projects/{PROJECT_ID}/...`
|
||||
2. The project settings page
|
||||
3. `stackit project list`
|
||||
|
||||
## Passing Credentials to Prowler
|
||||
|
||||
You can give Prowler either the **path** to the key file on disk or the **inline JSON content** of the key. Both go through the same StackIT SDK flow and refresh access tokens internally.
|
||||
|
||||
### Option A: Key File Path (workstation, persistent agents)
|
||||
|
||||
Recommended when the key is stored on disk.
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
export STACKIT_PROJECT_ID="12345678-1234-1234-1234-123456789abc"
|
||||
|
||||
prowler stackit
|
||||
```
|
||||
|
||||
Or as CLI flags:
|
||||
|
||||
```bash
|
||||
prowler stackit \
|
||||
--stackit-service-account-key-path ~/.stackit/sa-key.json \
|
||||
--stackit-project-id 12345678-1234-1234-1234-123456789abc
|
||||
```
|
||||
|
||||
<Note>
|
||||
Keep the key file outside of source control and lock it down with `chmod 600 ~/.stackit/sa-key.json`. Anyone with the JSON can mint access tokens for the service account.
|
||||
</Note>
|
||||
|
||||
### Option B: Inline Key Content (CI/CD, secret managers)
|
||||
|
||||
Recommended when the key is fetched at run time from a secret manager (GitHub Actions secret, AWS Secrets Manager, HashiCorp Vault, etc.) and you do not want to write it to disk.
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY="$(vault kv get -field=key stackit/sa)"
|
||||
export STACKIT_PROJECT_ID="12345678-1234-1234-1234-123456789abc"
|
||||
|
||||
prowler stackit
|
||||
```
|
||||
|
||||
<Note>
|
||||
Prefer the `STACKIT_SERVICE_ACCOUNT_KEY` environment variable over the matching CLI flag (`--stackit-service-account-key`); passing the secret on the command line leaks it through process listings and shell history.
|
||||
</Note>
|
||||
|
||||
When both the inline content and a key path are set, the inline content wins.
|
||||
|
||||
## Credential Lookup Order
|
||||
|
||||
Prowler resolves credentials in this order:
|
||||
|
||||
1. CLI arguments: `--stackit-service-account-key`, `--stackit-service-account-key-path`, `--stackit-project-id`
|
||||
2. Environment variables: `STACKIT_SERVICE_ACCOUNT_KEY`, `STACKIT_SERVICE_ACCOUNT_KEY_PATH`, `STACKIT_PROJECT_ID`
|
||||
|
||||
When both the inline key and the key file path are set, the inline content takes precedence.
|
||||
|
||||
## Token Lifetime
|
||||
|
||||
Access tokens are minted on demand by the SDK from the key file and refreshed before they expire. There is nothing to rotate while Prowler is running.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Likely Cause | Fix |
|
||||
|---------|--------------|-----|
|
||||
| `401 Unauthorized` during scan | Key file is missing fields, the public key is no longer registered, or the key was revoked | Re-issue the service account key in the StackIT portal and update `STACKIT_SERVICE_ACCOUNT_KEY_PATH` |
|
||||
| `403 Forbidden` during scan | Service account lacks role on the project | Re-check role assignment in the StackIT portal; `iaas.viewer` is the minimum for the shipped IaaS checks |
|
||||
| `StackIT project ID must be a valid UUID` | The project ID is not in UUID format | Copy the UUID from the portal URL or `stackit project list` |
|
||||
| `StackIT service account credentials are required` | None of the four credential inputs is set | Export `STACKIT_SERVICE_ACCOUNT_KEY_PATH` or `STACKIT_SERVICE_ACCOUNT_KEY` (or use their CLI counterparts) before running Prowler |
|
||||
@@ -0,0 +1,141 @@
|
||||
---
|
||||
title: 'Getting Started With StackIT'
|
||||
---
|
||||
|
||||
Prowler supports [StackIT](https://www.stackit.de/) from the CLI. This guide walks you through the requirements and how to run scans.
|
||||
|
||||
<Note>
|
||||
StackIT support in Prowler is community-maintained. For commercial support or to request additional service coverage, [contact us](https://prowler.com/contact).
|
||||
</Note>
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before running Prowler with the StackIT provider, ensure you have:
|
||||
|
||||
1. A StackIT account with at least one project
|
||||
2. A StackIT service account key file with permissions on the project (`iaas.viewer` is enough for the currently shipped IaaS checks; `project.owner` works for any future service). See the [Authentication guide](/user-guide/providers/stackit/authentication) for the full setup.
|
||||
3. Access to Prowler CLI (see [Installation](/getting-started/installation/prowler-cli))
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
### Step 1: Point Prowler at the Service Account Key
|
||||
|
||||
Prowler authenticates with a StackIT service account key. The SDK signs the RSA challenge in the key and refreshes access tokens internally for the life of the scan, so there is no manual token rotation.
|
||||
|
||||
**On a workstation or persistent agent** (key on disk):
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY_PATH="$HOME/.stackit/sa-key.json"
|
||||
export STACKIT_PROJECT_ID="12345678-1234-1234-1234-123456789abc"
|
||||
```
|
||||
|
||||
**In CI/CD** (key in a secret manager, never written to disk):
|
||||
|
||||
```bash
|
||||
export STACKIT_SERVICE_ACCOUNT_KEY="$(vault kv get -field=key stackit/sa)"
|
||||
export STACKIT_PROJECT_ID="12345678-1234-1234-1234-123456789abc"
|
||||
```
|
||||
|
||||
CLI flags work too:
|
||||
|
||||
```bash
|
||||
prowler stackit \
|
||||
--stackit-service-account-key-path ~/.stackit/sa-key.json \
|
||||
--stackit-project-id 12345678-1234-1234-1234-123456789abc
|
||||
```
|
||||
|
||||
<Note>
|
||||
For the inline key, prefer the `STACKIT_SERVICE_ACCOUNT_KEY` env var over the matching CLI flag; passing the secret on the command line leaks it through process listings and shell history.
|
||||
|
||||
Keep the key file outside of source control and lock it down with `chmod 600 ~/.stackit/sa-key.json`. Anyone with the JSON can mint access tokens for the service account.
|
||||
</Note>
|
||||
|
||||
### Step 2: Run Your First Scan
|
||||
|
||||
```bash
|
||||
prowler stackit
|
||||
```
|
||||
|
||||
Prowler will discover and audit the project's IaaS security groups across the available StackIT regions.
|
||||
|
||||
**Scan specific regions:**
|
||||
|
||||
```bash
|
||||
prowler stackit --stackit-region eu01 eu02
|
||||
```
|
||||
|
||||
**Run specific security checks:**
|
||||
|
||||
```bash
|
||||
prowler stackit --checks iaas_security_group_ssh_unrestricted
|
||||
|
||||
# List all available checks
|
||||
prowler stackit --list-checks
|
||||
```
|
||||
|
||||
**Filter by check severity:**
|
||||
|
||||
```bash
|
||||
prowler stackit --severity critical high
|
||||
```
|
||||
|
||||
**Generate specific output formats:**
|
||||
|
||||
```bash
|
||||
# JSON only
|
||||
prowler stackit --output-modes json
|
||||
|
||||
# CSV and HTML
|
||||
prowler stackit --output-modes csv html
|
||||
|
||||
# Custom output directory
|
||||
prowler stackit --output-directory /path/to/reports/
|
||||
```
|
||||
|
||||
**Use a mutelist to suppress findings:**
|
||||
|
||||
```yaml
|
||||
# mutelist.yaml
|
||||
Mutelist:
|
||||
Accounts:
|
||||
"12345678-1234-1234-1234-123456789abc":
|
||||
Checks:
|
||||
iaas_security_group_ssh_unrestricted:
|
||||
Regions:
|
||||
- "*"
|
||||
Resources:
|
||||
- "test-sg-id"
|
||||
Tags: []
|
||||
```
|
||||
|
||||
```bash
|
||||
prowler stackit --mutelist-file mutelist.yaml
|
||||
```
|
||||
|
||||
### Step 3: Review the Results
|
||||
|
||||
Prowler outputs findings to the console and writes reports to the `output/` directory by default:
|
||||
|
||||
- CSV: `output/prowler-output-stackit-{project_id}-{timestamp}.csv`
|
||||
- JSON: `output/prowler-output-stackit-{project_id}-{timestamp}.json`
|
||||
- HTML: `output/prowler-output-stackit-{project_id}-{timestamp}.html`
|
||||
|
||||
## Supported StackIT Services
|
||||
|
||||
| Service | StackIT API | Description | Example Checks |
|
||||
|---------|-------------|-------------|----------------|
|
||||
| **IaaS** | `iaas` | Virtual machines, network interfaces, security groups | `iaas_security_group_ssh_unrestricted`, `iaas_security_group_rdp_unrestricted`, `iaas_security_group_database_unrestricted`, `iaas_security_group_all_traffic_unrestricted` |
|
||||
|
||||
Additional services will be added in future releases. Track progress in the [Prowler release notes](https://github.com/prowler-cloud/prowler/releases).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Authentication Errors
|
||||
|
||||
If the scan fails with a 401 error, the service account key is no longer valid (revoked, rotated or the key file is incomplete). Re-issue the key in the [StackIT portal](https://portal.stackit.cloud/) and update `STACKIT_SERVICE_ACCOUNT_KEY_PATH`.
|
||||
|
||||
### Permission Errors
|
||||
|
||||
If checks fail with a 403 error, the service account is missing the required role on the project. Re-check the role assignment in the StackIT portal (`iaas.viewer` is the minimum for the shipped IaaS checks).
|
||||
|
||||
For detailed setup steps, see the [Authentication guide](/user-guide/providers/stackit/authentication).
|
||||
+1
-1
@@ -10,7 +10,7 @@ import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
Alerts notify recipients by email when security findings match saved filter conditions. Use Alerts to track high-priority findings, monitor specific providers or services, and keep teams informed about scan results that match defined criteria.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** with a paid subscription.
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [paid subscription](https://prowler.com/pricing).
|
||||
</Note>
|
||||
|
||||
## Prerequisites
|
||||
@@ -18,7 +18,7 @@ Source: [`prowler-cloud/prowler`](https://github.com/prowler-cloud/prowler) · M
|
||||
| `provider` | yes | — | Cloud provider to scan (`aws`, `azure`, `gcp`, `github`, `kubernetes`, `iac`, `cloudflare`, etc.) |
|
||||
| `image-tag` | no | `stable` | Docker image tag — `stable` (latest release), `latest` (master, not stable), or `<x.y.z>` (pinned). See [available tags](https://hub.docker.com/r/prowlercloud/prowler/tags). |
|
||||
| `output-formats` | no | `json-ocsf` | Output format(s) for scan results. Space-separated (e.g. `sarif json-ocsf`) |
|
||||
| `push-to-cloud` | no | `false` | Push findings to [Prowler Cloud](/user-guide/tutorials/prowler-app-import-findings). When `true`, `PROWLER_CLOUD_API_KEY` is auto-forwarded |
|
||||
| `push-to-cloud` | no | `false` | Push findings to [Prowler Cloud](/user-guide/tutorials/prowler-import-findings). When `true`, `PROWLER_CLOUD_API_KEY` is auto-forwarded |
|
||||
| `flags` | no | `""` | Additional CLI flags (e.g. `--severity critical high`). Values with spaces can be quoted: `--resource-tag 'Environment=My Server'` |
|
||||
| `extra-env` | no | `""` | Space-, newline-, or comma-separated list of env var **names** to forward to the container (see [Authentication](#authentication)) |
|
||||
| `upload-sarif` | no | `false` | Upload SARIF results to GitHub Code Scanning |
|
||||
@@ -43,7 +43,7 @@ Source: [`prowler-cloud/prowler`](https://github.com/prowler-cloud/prowler) · M
|
||||
|
||||
### Push findings to Prowler Cloud
|
||||
|
||||
Send scan results directly to [Prowler Cloud](/user-guide/tutorials/prowler-app-import-findings) for centralized visibility, compliance tracking, and team collaboration.
|
||||
Send scan results directly to [Prowler Cloud](/user-guide/tutorials/prowler-import-findings) for centralized visibility, compliance tracking, and team collaboration.
|
||||
|
||||
```yaml
|
||||
- uses: prowler-cloud/prowler@5.25
|
||||
|
||||
@@ -47,7 +47,11 @@ Follow these steps to remove a user of your account:
|
||||
1. Navigate to **Users** from the side menu.
|
||||
2. Click the delete button of your current user.
|
||||
|
||||
> **Note: Each user will be able to delete himself and not others, regardless of his permissions.**
|
||||
> **Note: Each user can only delete their own account, regardless of their permissions. For this reason, the delete button is only shown on your own row and not on other users' rows.**
|
||||
|
||||
Deleting a user removes the **entire user account** from Prowler, not just its membership in your organization. Because a single account can belong to more than one tenant, allowing one administrator to delete it outright could affect organizations they don't manage and irreversibly remove another person's identity. To keep this destructive action under the control of the account owner, the API only permits a user to delete themselves (it rejects any other target with a `400` response), and the UI mirrors this by showing the delete button exclusively on your own row.
|
||||
|
||||
To remove **another** user from your organization, use the [_Expel from organization_](/user-guide/tutorials/prowler-app-multi-tenant#expelling-a-user-from-an-organization) action instead. Expelling removes the user's membership, role grants, and active sessions for your tenant only, and deletes the underlying account just for that user if your organization was their last remaining membership. This action is reserved for tenant **owners**.
|
||||
|
||||
<img src="/images/prowler-app/rbac/user_remove.png" alt="Remove User" width="700" />
|
||||
|
||||
@@ -239,7 +243,7 @@ To grant all administrative permissions, select the **Grant all admin permission
|
||||
|
||||
The following permissions are available exclusively in **Prowler Cloud**:
|
||||
|
||||
**Manage Ingestions:** Submit and manage findings ingestion jobs via the API. Required to upload OCSF scan results using the `--push-to-cloud` CLI flag or the ingestion endpoints. See [Import Findings](/user-guide/tutorials/prowler-app-import-findings) for details.
|
||||
**Manage Ingestions:** Submit and manage findings ingestion jobs via the API. Required to upload OCSF scan results using the `--push-to-cloud` CLI flag or the ingestion endpoints. See [Import Findings](/user-guide/tutorials/prowler-import-findings) for details.
|
||||
|
||||
**Manage Billing:** Access and manage billing settings, subscription plans, and payment methods.
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
Findings Ingestion enables uploading OCSF (Open Cybersecurity Schema Framework) scan results to Prowler Cloud. This feature supports importing findings from Prowler CLI output files that use the [Detection Finding](https://schema.ocsf.io/classes/detection_finding) class.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** with a paid subscription.
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [paid subscription](https://prowler.com/pricing).
|
||||
</Note>
|
||||
|
||||
## OCSF Detection Finding format
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> **Skills Reference**: See [`prowler-mcp`](../skills/prowler-mcp/SKILL.md)
|
||||
|
||||
### Auto-invoke Skills
|
||||
## Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
@@ -68,7 +68,7 @@ Python 3.12+ | FastMCP 2.13.1 | httpx (async) | Pydantic | uv
|
||||
|
||||
## PROJECT STRUCTURE
|
||||
|
||||
```
|
||||
```text
|
||||
mcp_server/prowler_mcp_server/
|
||||
├── server.py # Main orchestration
|
||||
├── prowler_hub/server.py # Hub tools (no auth)
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
|
||||
All notable changes to the **Prowler MCP Server** are documented in this file.
|
||||
|
||||
## [0.7.2] (Prowler v5.28.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Preserve authorization header in HTTP mode [(#11366)](https://github.com/prowler-cloud/prowler/pull/11366)
|
||||
|
||||
---
|
||||
|
||||
## [0.7.1] (Prowler v5.28.0)
|
||||
|
||||
### 🔐 Security
|
||||
@@ -44,6 +52,8 @@ All notable changes to the **Prowler MCP Server** are documented in this file.
|
||||
|
||||
- Attack Path tool to get Neo4j DB schema [(#10321)](https://github.com/prowler-cloud/prowler/pull/10321)
|
||||
|
||||
---
|
||||
|
||||
## [0.4.0] (Prowler v5.19.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
@@ -83,14 +83,14 @@ npm install --save-exact mcp-remote@0.1.38
|
||||
|
||||
### 2. Local STDIO Mode
|
||||
|
||||
**Run the server locally on your machine**
|
||||
Run the server locally on your machine:
|
||||
|
||||
- Runs as a subprocess of your MCP client
|
||||
- Requires Python 3.12+ or Docker
|
||||
|
||||
### 3. Self-Hosted HTTP Mode
|
||||
|
||||
**Deploy your own remote MCP server**
|
||||
Deploy your own remote MCP server:
|
||||
|
||||
- Full control over deployment
|
||||
- Requires Python 3.12+ or Docker
|
||||
@@ -132,7 +132,7 @@ All tools follow a consistent naming pattern with prefixes:
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
```text
|
||||
prowler_mcp_server/
|
||||
├── server.py # Main orchestrator (imports sub-servers with prefixes)
|
||||
├── main.py # CLI entry point
|
||||
@@ -154,17 +154,20 @@ prowler_mcp_server/
|
||||
|
||||
The Prowler MCP Server enables powerful workflows through AI assistants:
|
||||
|
||||
**Security Operations**
|
||||
### Security Operations
|
||||
|
||||
- "Show me all critical findings from my AWS production accounts"
|
||||
- "Register my new AWS account in Prowler and run a scheduled scan every day"
|
||||
- "List all muted findings and detect what findgings are muted by a not enough good reason in relation to their severity"
|
||||
|
||||
**Security Research**
|
||||
### Security Research
|
||||
|
||||
- "Explain what the S3 bucket public access Prowler check does"
|
||||
- "Find all Prowler checks related to encryption at rest"
|
||||
- "What is the latest version of the CIS that Prowler is covering per provider?"
|
||||
|
||||
**Documentation & Learning**
|
||||
### Documentation & Learning
|
||||
|
||||
- "How do I configure Prowler to scan my GCP organization?"
|
||||
- "What authentication methods does Prowler support for Azure?"
|
||||
- "How can I contribute with a new security check to Prowler?"
|
||||
|
||||
@@ -5,6 +5,7 @@ from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
from fastmcp.server.dependencies import get_http_headers
|
||||
|
||||
from prowler_mcp_server import __version__
|
||||
from prowler_mcp_server.lib.logger import logger
|
||||
|
||||
@@ -68,7 +69,7 @@ class ProwlerAppAuth:
|
||||
async def authenticate(self) -> str:
|
||||
"""Authenticate and return token (API key for STDIO, API key or JWT for HTTP)."""
|
||||
if self.mode == "http":
|
||||
headers = get_http_headers()
|
||||
headers = get_http_headers(include={"authorization"})
|
||||
authorization_header = headers.get("authorization", None)
|
||||
|
||||
if not authorization_header:
|
||||
|
||||
+3
-2
@@ -12,8 +12,9 @@ reason = """
|
||||
CVE-2025-45768 is disputed by the pyjwt maintainers. The advisory describes
|
||||
weak encryption, but the underlying issue is that callers may pick a short
|
||||
HMAC secret — key-length enforcement is the application's responsibility, not
|
||||
a defect in the library. We are on pyjwt 2.12.1 (latest at pin time) and
|
||||
enforce key strength in our own auth code, so this advisory does not apply.
|
||||
a defect in the library. We are on pyjwt 2.13.0 (which now also emits an
|
||||
InsecureKeyLengthWarning for short HMAC secrets) and enforce key strength in
|
||||
our own auth code, so this advisory does not apply.
|
||||
Re-evaluate when a non-disputed advisory or upstream fix lands.
|
||||
"""
|
||||
|
||||
|
||||
@@ -28,12 +28,12 @@ This Terraform configuration creates the necessary IAM role and policies to allo
|
||||
|
||||
### Usage Examples
|
||||
|
||||
#### Basic deployment (without S3 integration):
|
||||
#### Basic deployment (without S3 integration)
|
||||
```bash
|
||||
terraform apply -var="external_id=your-external-id-here"
|
||||
```
|
||||
|
||||
#### With S3 integration enabled:
|
||||
#### With S3 integration enabled
|
||||
```bash
|
||||
terraform apply \
|
||||
-var="external_id=your-external-id-here" \
|
||||
@@ -42,14 +42,14 @@ terraform apply \
|
||||
-var="s3_integration_bucket_account_id=123456789012"
|
||||
```
|
||||
|
||||
#### Using terraform.tfvars file (Recommended):
|
||||
#### Using terraform.tfvars file (Recommended)
|
||||
```bash
|
||||
cp terraform.tfvars.example terraform.tfvars
|
||||
# Edit the file with your values
|
||||
terraform apply
|
||||
```
|
||||
|
||||
#### Command line variables (Alternative):
|
||||
#### Command line variables (Alternative)
|
||||
```bash
|
||||
terraform apply -var="external_id=your-external-id-here"
|
||||
```
|
||||
|
||||
+3
-3
@@ -7,7 +7,7 @@
|
||||
> - [`prowler-compliance`](../skills/prowler-compliance/SKILL.md) - Compliance framework structure
|
||||
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
|
||||
|
||||
### Auto-invoke Skills
|
||||
## Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
@@ -44,7 +44,7 @@ The Prowler SDK is the core Python engine powering cloud security assessments ac
|
||||
|
||||
### Provider Architecture
|
||||
|
||||
```
|
||||
```text
|
||||
prowler/providers/{provider}/
|
||||
├── {provider}_provider.py # Main provider class
|
||||
├── models.py # Provider-specific models
|
||||
@@ -91,7 +91,7 @@ Python 3.10+ | uv | pytest | moto (AWS mocking) | Pre-commit hooks (black, flake
|
||||
|
||||
## PROJECT STRUCTURE
|
||||
|
||||
```
|
||||
```text
|
||||
prowler/
|
||||
├── __main__.py # CLI entry point
|
||||
├── config/ # Global configuration
|
||||
|
||||
@@ -2,6 +2,91 @@
|
||||
|
||||
All notable changes to the **Prowler SDK** are documented in this file.
|
||||
|
||||
## [5.30.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `sagemaker_models_monitor_enabled` check for AWS provider, verifying that each SageMaker monitoring schedule is in the `Scheduled` state so data and model drift is actively detected [(#11278)](https://github.com/prowler-cloud/prowler/pull/11278)
|
||||
- DORA (Digital Operational Resilience Act, Regulation (EU) 2022/2554) universal compliance framework with AWS provider coverage across the five DORA pillars [(#11131)](https://github.com/prowler-cloud/prowler/pull/11131)
|
||||
- Okta authenticator and password policy checks for STIG-aligned hardening requirements [(#11465)](https://github.com/prowler-cloud/prowler/pull/11465)
|
||||
- Okta network zone check to detect whether anonymized proxy traffic is blocked [(#11463)](https://github.com/prowler-cloud/prowler/pull/11463)
|
||||
- Okta API token checks for super admin ownership and network zone restrictions [(#11464)](https://github.com/prowler-cloud/prowler/pull/11464)
|
||||
- Support for external/custom providers, checks, and compliance frameworks without modifying core code [(#10700)](https://github.com/prowler-cloud/prowler/pull/10700)
|
||||
- `elbv2_alb_drop_invalid_header_fields_enabled` check for AWS provider, verifying Application Load Balancers have `routing.http.drop_invalid_header_fields.enabled` set to `true` to mitigate HTTP desync attacks (AWS FSBP ELB.4) [(#11471)](https://github.com/prowler-cloud/prowler/pull/11471)
|
||||
- `user`, `systemlog` and `idp` service for Okta provider with `user_inactivity_automation_35d_enabled`, `systemlog_streaming_enabled` and `idp_smart_card_dod_approved_ca` checks [(#11496)](https://github.com/prowler-cloud/prowler/pull/11496)
|
||||
- External multi-provider compliance frameworks can be registered via the `prowler.compliance.universal` entry point group [(#11490)](https://github.com/prowler-cloud/prowler/pull/11490)
|
||||
- AWS AI Security Framework support in the CLI dashboard [(#11475)](https://github.com/prowler-cloud/prowler/pull/11475)
|
||||
- `entra_service_principal_privileged_role_no_owners` check for M365 provider, failing when a service principal with a permanent Tier 0 directory role has owners on the service principal or its parent app registration [(#11070)](https://github.com/prowler-cloud/prowler/issues/11070)
|
||||
- `kms_key_rotation_max_90_days` check for GCP provider, verifying KMS customer-managed keys are rotated every 90 days or less in line with the CIS Benchmark [(#11516)](https://github.com/prowler-cloud/prowler/pull/11516)
|
||||
- `exchange_mailbox_primary_smtp_uses_custom_domain` check for M365 provider [(#11215)](https://github.com/prowler-cloud/prowler/pull/11215)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `load_and_validate_config_file` now unwraps namespaced config for every built-in and external provider, and no longer leaks the full file as the provider's config when the file is namespaced [(#10700)](https://github.com/prowler-cloud/prowler/pull/10700)
|
||||
- `entra_users_mfa_capable` no longer flags pre-provisioned users with future `employeeHireDate`; future-hire date comparisons now tolerate naive datetimes [(#11511)](https://github.com/prowler-cloud/prowler/pull/11511)
|
||||
- M365 Admin Center group enumeration now follows Microsoft Graph pagination so group-scoped checks include groups beyond the first page [(#11510)](https://github.com/prowler-cloud/prowler/pull/11510)
|
||||
- GCP `kms_key_rotation_enabled` check now only verifies that automatic key rotation is enabled (any interval) instead of enforcing a 90-day period, resolving the mismatch between the check and its documentation; the CIS, Prowler ThreatScore, and CCC requirements that mandate a 90-day maximum were remapped to the new `kms_key_rotation_max_90_days` check [(#11516)](https://github.com/prowler-cloud/prowler/pull/11516)
|
||||
|
||||
---
|
||||
|
||||
## [5.29.3] (Prowler v5.29.3)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- GCP `logging_sink_created` now recognizes organization-level aggregated sinks with `includeChildren=True`, avoiding false failures for covered projects [(#11355)](https://github.com/prowler-cloud/prowler/pull/11355)
|
||||
- GCP `logging_log_metric_filter_and_alert_*` checks now recognize organization-level aggregated sinks with `includeChildren=True`, no longer false-failing projects covered by a central bucket-scoped metric + alert [(#11488)](https://github.com/prowler-cloud/prowler/pull/11488)
|
||||
- Jira integration no longer fails with `400 INVALID_INPUT` when a finding has empty fields [(#11474)](https://github.com/prowler-cloud/prowler/pull/11474)
|
||||
- GCP `iam_service_account_unused` now passes disabled service accounts instead of failing them, since a disabled account cannot authenticate or be used [(#11467)](https://github.com/prowler-cloud/prowler/pull/11467)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `dulwich` from 0.23.0 to 1.2.5 and `pyjwt` from 2.12.1 to 2.13.0, patching `GHSA-897w-fcg9-f6xj` (arbitrary file write) and `PYSEC-2026-179` (HMAC/JWK key confusion) flagged by osv-scanner [(#11499)](https://github.com/prowler-cloud/prowler/pull/11499)
|
||||
|
||||
---
|
||||
|
||||
## [5.29.1] (Prowler v5.29.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- OCSF output writer now re-raises I/O errors (e.g. `ENOSPC`) instead of logging them per finding and leaving a truncated file [(#11421)](https://github.com/prowler-cloud/prowler/pull/11421)
|
||||
|
||||
---
|
||||
|
||||
## [5.29.0] (Prowler v5.29.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `application` service for Okta provider with `application_admin_console_session_idle_timeout_15min`, `application_admin_console_mfa_required`, `application_admin_console_phishing_resistant_authentication`, `application_dashboard_mfa_required`, `application_dashboard_phishing_resistant_authentication`, and `application_authentication_policy_network_zone_enforced` checks [(#11358)](https://github.com/prowler-cloud/prowler/pull/11358)
|
||||
- AWS AI Security Framework compliance for AWS provider [(#11353)](https://github.com/prowler-cloud/prowler/pull/11353)
|
||||
- `storage_account_public_network_access_disabled` check for Azure provider and remapped the Azure CIS "Public Network Access is Disabled" requirements to it [(#11334)](https://github.com/prowler-cloud/prowler/pull/11334)
|
||||
- StackIT provider with service account key authentication [(#9237)](https://github.com/prowler-cloud/prowler/pull/9237)
|
||||
- 8 Rules service checks for Google Workspace provider using the Cloud Identity Policy API [(#11379)](https://github.com/prowler-cloud/prowler/pull/11379)
|
||||
- 12 Security service checks for Google Workspace provider using the Cloud Identity Policy API [(#11356)](https://github.com/prowler-cloud/prowler/pull/11356)
|
||||
|
||||
### ⚠️ Deprecated
|
||||
|
||||
- `s3_bucket_default_encryption` check for AWS provider since SSE-S3 is automatically applied to all S3 buckets by AWS as of January 5, 2023 and can no longer be disabled [(#11230)](https://github.com/prowler-cloud/prowler/pull/11230)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Broken documentation URLs in Google Workspace check metadata [(#11405)](https://github.com/prowler-cloud/prowler/pull/11405)
|
||||
- ENS RD 311/2022 (AWS) compliance mapping: `vpc_different_regions` was uncorrectly mapped under the `mp.com.4` family (Network segregation). That check is now mapped to a new `op.cont.2.aws.vpc.1` requirement under the Continuity of Service control [(#11372)](https://github.com/prowler-cloud/prowler/pull/11372)
|
||||
- Compliance CSV row count now matches the UI per requirement by sourcing rows from the framework JSON's `requirement.Checks` instead of the stale `finding.compliance` snapshot [(#11370)](https://github.com/prowler-cloud/prowler/pull/11370)
|
||||
- OpenStack provider exception codes moved from the `10000-10999` range, shared with the AlibabaCloud provider, to the free `17000-17999` range to keep error codes unambiguous [(#11382)](https://github.com/prowler-cloud/prowler/pull/11382)
|
||||
- Azure provider authentication against sovereign clouds (`AzureChinaCloud`, `AzureUSGovernment`) [(#10284)](https://github.com/prowler-cloud/prowler/pull/10284)
|
||||
|
||||
---
|
||||
|
||||
## [5.28.1] (Prowler v5.28.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `compute_project_os_login_enabled` and `compute_project_os_login_2fa_enabled` checks for GCP provider no longer false-FAIL on projects where the `enable-oslogin` / `enable-oslogin-2fa` metadata is not set explicitly but is inherited automatically from the `constraints/compute.requireOsLogin` org policy. The policy controller writes the inherited value in lowercase (`"true"`), but the service-layer parser compared it to the uppercase string literal `"TRUE"`. Comparison is now case-insensitive [(#11341)](https://github.com/prowler-cloud/prowler/pull/11341)
|
||||
- `storage_smb_channel_encryption_with_secure_algorithm` check for Azure provider no longer passes when a storage account allows a weak SMB channel encryption algorithm (e.g. `AES-128-CCM`/`AES-128-GCM`) alongside `AES-256-GCM`; it now requires every enabled algorithm to be in the recommended list, configurable via `azure.recommended_smb_channel_encryption_algorithms` (defaults to `AES-256-GCM` only, as required by CIS) [(#11327)](https://github.com/prowler-cloud/prowler/pull/11327)
|
||||
- Azure and M365 providers crashing with `RuntimeError: There is no current event loop` on Python 3.12 when called from threads without an active event loop (e.g. Celery workers) [(#11360)](https://github.com/prowler-cloud/prowler/pull/11360)
|
||||
|
||||
---
|
||||
|
||||
## [5.28.0] (Prowler v5.28.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
+62
-77
@@ -10,7 +10,6 @@ from colorama import Fore, Style
|
||||
from colorama import init as colorama_init
|
||||
|
||||
from prowler.config.config import (
|
||||
EXTERNAL_TOOL_PROVIDERS,
|
||||
cloud_api_base_url,
|
||||
csv_file_suffix,
|
||||
get_available_compliance_frameworks,
|
||||
@@ -85,11 +84,6 @@ from prowler.lib.outputs.compliance.compliance import (
|
||||
display_compliance_table,
|
||||
process_universal_compliance_frameworks,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.csa.csa_alibabacloud import AlibabaCloudCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_aws import AWSCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_azure import AzureCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_gcp import GCPCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_oraclecloud import OracleCloudCSA
|
||||
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
|
||||
@@ -158,6 +152,7 @@ from prowler.providers.okta.models import OktaOutputOptions
|
||||
from prowler.providers.openstack.models import OpenStackOutputOptions
|
||||
from prowler.providers.oraclecloud.models import OCIOutputOptions
|
||||
from prowler.providers.scaleway.models import ScalewayOutputOptions
|
||||
from prowler.providers.stackit.models import StackITOutputOptions
|
||||
from prowler.providers.vercel.models import VercelOutputOptions
|
||||
|
||||
|
||||
@@ -209,9 +204,10 @@ def prowler():
|
||||
# We treat the compliance framework as another output format
|
||||
if compliance_framework:
|
||||
args.output_formats.extend(compliance_framework)
|
||||
# If no input compliance framework, set all, unless a specific service or check is input
|
||||
# Skip for IAC and LLM providers that don't use compliance frameworks
|
||||
elif default_execution and provider not in ["iac", "llm"]:
|
||||
# If no input compliance framework, set all, unless a specific service or check is input.
|
||||
# Skip for tool-wrapper providers (iac, llm, image, and any external plug-in
|
||||
# declaring `is_external_tool_provider = True`) — they don't use compliance frameworks.
|
||||
elif default_execution and not Provider.is_tool_wrapper_provider(provider):
|
||||
args.output_formats.extend(get_available_compliance_frameworks(provider))
|
||||
|
||||
# Set Logger configuration
|
||||
@@ -249,7 +245,7 @@ def prowler():
|
||||
universal_frameworks = {}
|
||||
|
||||
# Skip compliance frameworks for external-tool providers
|
||||
if provider not in EXTERNAL_TOOL_PROVIDERS:
|
||||
if not Provider.is_tool_wrapper_provider(provider):
|
||||
bulk_compliance_frameworks = Compliance.get_bulk(provider)
|
||||
# Complete checks metadata with the compliance framework specification
|
||||
bulk_checks_metadata = update_checks_metadata_with_compliance(
|
||||
@@ -317,7 +313,7 @@ def prowler():
|
||||
sys.exit()
|
||||
|
||||
# Skip service and check loading for external-tool providers
|
||||
if provider not in EXTERNAL_TOOL_PROVIDERS:
|
||||
if not Provider.is_tool_wrapper_provider(provider):
|
||||
# Import custom checks from folder
|
||||
if checks_folder:
|
||||
custom_checks = parse_checks_from_folder(global_provider, checks_folder)
|
||||
@@ -416,6 +412,10 @@ def prowler():
|
||||
output_options = OCIOutputOptions(
|
||||
args, bulk_checks_metadata, global_provider.identity
|
||||
)
|
||||
elif provider == "stackit":
|
||||
output_options = StackITOutputOptions(
|
||||
args, bulk_checks_metadata, global_provider.identity
|
||||
)
|
||||
elif provider == "alibabacloud":
|
||||
output_options = AlibabaCloudOutputOptions(
|
||||
args, bulk_checks_metadata, global_provider.identity
|
||||
@@ -436,6 +436,20 @@ def prowler():
|
||||
output_options = ScalewayOutputOptions(
|
||||
args, bulk_checks_metadata, global_provider.identity
|
||||
)
|
||||
else:
|
||||
# Dynamic fallback: any external/custom provider
|
||||
try:
|
||||
output_options = global_provider.get_output_options(
|
||||
args, bulk_checks_metadata
|
||||
)
|
||||
except NotImplementedError:
|
||||
# No provider-specific OutputOptions: use the generic default so the
|
||||
# run still produces output instead of aborting.
|
||||
from prowler.providers.common.models import default_output_options
|
||||
|
||||
output_options = default_output_options(
|
||||
global_provider, args, bulk_checks_metadata
|
||||
)
|
||||
|
||||
# Run the quick inventory for the provider if available
|
||||
if hasattr(args, "quick_inventory") and args.quick_inventory:
|
||||
@@ -445,7 +459,7 @@ def prowler():
|
||||
# Execute checks
|
||||
findings = []
|
||||
|
||||
if provider in EXTERNAL_TOOL_PROVIDERS:
|
||||
if Provider.is_tool_wrapper_provider(provider):
|
||||
# For external-tool providers, run the scan directly
|
||||
if provider == "llm":
|
||||
|
||||
@@ -455,12 +469,19 @@ def prowler():
|
||||
|
||||
findings = global_provider.run_scan(streaming_callback=streaming_callback)
|
||||
else:
|
||||
# Original behavior for IAC and Image
|
||||
try:
|
||||
if provider == "image":
|
||||
try:
|
||||
findings = global_provider.run()
|
||||
except ImageBaseException as error:
|
||||
logger.critical(f"{error}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
# IAC and external tool-wrapper providers registered via entry
|
||||
# points. Unexpected failures propagate to the outer except
|
||||
# Exception backstop further down in this file — keeping the
|
||||
# branch free of an Image-specific catch that would otherwise
|
||||
# mislead plug-in authors reading this code.
|
||||
findings = global_provider.run()
|
||||
except ImageBaseException as error:
|
||||
logger.critical(f"{error}")
|
||||
sys.exit(1)
|
||||
# Note: External tool providers don't support granular progress tracking since
|
||||
# they run external tools as a black box and return all findings at once.
|
||||
# Progress tracking would just be 0% → 100%.
|
||||
@@ -801,18 +822,6 @@ def prowler():
|
||||
)
|
||||
generated_outputs["compliance"].append(c5)
|
||||
c5.batch_write_data_to_file()
|
||||
elif compliance_name == "csa_ccm_4.0_aws":
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
f"{output_options.output_filename}_{compliance_name}.csv"
|
||||
)
|
||||
csa_ccm_4_0_aws = AWSCSA(
|
||||
findings=finding_outputs,
|
||||
compliance=bulk_compliance_frameworks[compliance_name],
|
||||
file_path=filename,
|
||||
)
|
||||
generated_outputs["compliance"].append(csa_ccm_4_0_aws)
|
||||
csa_ccm_4_0_aws.batch_write_data_to_file()
|
||||
else:
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
@@ -916,18 +925,6 @@ def prowler():
|
||||
)
|
||||
generated_outputs["compliance"].append(c5_azure)
|
||||
c5_azure.batch_write_data_to_file()
|
||||
elif compliance_name == "csa_ccm_4.0_azure":
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
f"{output_options.output_filename}_{compliance_name}.csv"
|
||||
)
|
||||
csa_ccm_4_0_azure = AzureCSA(
|
||||
findings=finding_outputs,
|
||||
compliance=bulk_compliance_frameworks[compliance_name],
|
||||
file_path=filename,
|
||||
)
|
||||
generated_outputs["compliance"].append(csa_ccm_4_0_azure)
|
||||
csa_ccm_4_0_azure.batch_write_data_to_file()
|
||||
else:
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
@@ -1031,18 +1028,6 @@ def prowler():
|
||||
)
|
||||
generated_outputs["compliance"].append(c5_gcp)
|
||||
c5_gcp.batch_write_data_to_file()
|
||||
elif compliance_name == "csa_ccm_4.0_gcp":
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
f"{output_options.output_filename}_{compliance_name}.csv"
|
||||
)
|
||||
csa_ccm_4_0_gcp = GCPCSA(
|
||||
findings=finding_outputs,
|
||||
compliance=bulk_compliance_frameworks[compliance_name],
|
||||
file_path=filename,
|
||||
)
|
||||
generated_outputs["compliance"].append(csa_ccm_4_0_gcp)
|
||||
csa_ccm_4_0_gcp.batch_write_data_to_file()
|
||||
else:
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
@@ -1277,18 +1262,6 @@ def prowler():
|
||||
)
|
||||
generated_outputs["compliance"].append(cis)
|
||||
cis.batch_write_data_to_file()
|
||||
elif compliance_name == "csa_ccm_4.0_oraclecloud":
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
f"{output_options.output_filename}_{compliance_name}.csv"
|
||||
)
|
||||
csa_ccm_4_0_oraclecloud = OracleCloudCSA(
|
||||
findings=finding_outputs,
|
||||
compliance=bulk_compliance_frameworks[compliance_name],
|
||||
file_path=filename,
|
||||
)
|
||||
generated_outputs["compliance"].append(csa_ccm_4_0_oraclecloud)
|
||||
csa_ccm_4_0_oraclecloud.batch_write_data_to_file()
|
||||
else:
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
@@ -1317,18 +1290,6 @@ def prowler():
|
||||
)
|
||||
generated_outputs["compliance"].append(cis)
|
||||
cis.batch_write_data_to_file()
|
||||
elif compliance_name == "csa_ccm_4.0_alibabacloud":
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
f"{output_options.output_filename}_{compliance_name}.csv"
|
||||
)
|
||||
csa_ccm_4_0_alibabacloud = AlibabaCloudCSA(
|
||||
findings=finding_outputs,
|
||||
compliance=bulk_compliance_frameworks[compliance_name],
|
||||
file_path=filename,
|
||||
)
|
||||
generated_outputs["compliance"].append(csa_ccm_4_0_alibabacloud)
|
||||
csa_ccm_4_0_alibabacloud.batch_write_data_to_file()
|
||||
elif compliance_name == "prowler_threatscore_alibabacloud":
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
@@ -1353,6 +1314,30 @@ def prowler():
|
||||
)
|
||||
generated_outputs["compliance"].append(generic_compliance)
|
||||
generic_compliance.batch_write_data_to_file()
|
||||
else:
|
||||
# Dynamic fallback: any external/custom provider
|
||||
try:
|
||||
global_provider.generate_compliance_output(
|
||||
finding_outputs,
|
||||
bulk_compliance_frameworks,
|
||||
input_compliance_frameworks,
|
||||
output_options,
|
||||
generated_outputs,
|
||||
)
|
||||
except NotImplementedError:
|
||||
# Last resort: generic compliance
|
||||
for compliance_name in input_compliance_frameworks:
|
||||
filename = (
|
||||
f"{output_options.output_directory}/compliance/"
|
||||
f"{output_options.output_filename}_{compliance_name}.csv"
|
||||
)
|
||||
generic_compliance = GenericCompliance(
|
||||
findings=finding_outputs,
|
||||
compliance=bulk_compliance_frameworks[compliance_name],
|
||||
file_path=filename,
|
||||
)
|
||||
generated_outputs["compliance"].append(generic_compliance)
|
||||
generic_compliance.batch_write_data_to_file()
|
||||
|
||||
# AWS Security Hub Integration
|
||||
if provider == "aws":
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1863,7 +1863,9 @@
|
||||
"Id": "ELB.4",
|
||||
"Name": "Application load balancers should be configured to drop HTTP headers",
|
||||
"Description": "This control evaluates AWS Application Load Balancers (ALB) to ensure they are configured to drop invalid HTTP headers. The control fails if the value of routing.http.drop_invalid_header_fields.enabled is set to false. By default, ALBs are not configured to drop invalid HTTP header values. Removing these header values prevents HTTP desync attacks.",
|
||||
"Checks": [],
|
||||
"Checks": [
|
||||
"elbv2_alb_drop_invalid_header_fields_enabled"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
"ItemId": "ELB.4",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2539,8 +2539,7 @@
|
||||
}
|
||||
],
|
||||
"Checks": [
|
||||
"vpc_subnet_separate_private_public",
|
||||
"vpc_different_regions"
|
||||
"vpc_subnet_separate_private_public"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -2593,8 +2592,8 @@
|
||||
}
|
||||
],
|
||||
"Checks": [
|
||||
"vpc_subnet_different_az",
|
||||
"vpc_different_regions"
|
||||
"vpc_different_regions",
|
||||
"vpc_subnet_different_az"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4262,6 +4261,29 @@
|
||||
],
|
||||
"Checks": []
|
||||
},
|
||||
{
|
||||
"Id": "op.cont.2.aws.vpc.1",
|
||||
"Description": "Plan de continuidad",
|
||||
"Attributes": [
|
||||
{
|
||||
"IdGrupoControl": "op.cont.2",
|
||||
"Marco": "operacional",
|
||||
"Categoria": "continuidad del servicio",
|
||||
"DescripcionControl": "Distribución de las VPCs entre múltiples regiones y zonas de disponibilidad de AWS para garantizar la continuidad del servicio ante fallos regionales o zonales.",
|
||||
"Nivel": "alto",
|
||||
"Tipo": "requisito",
|
||||
"Dimensiones": [
|
||||
"disponibilidad"
|
||||
],
|
||||
"ModoEjecucion": "automático",
|
||||
"Dependencias": []
|
||||
}
|
||||
],
|
||||
"Checks": [
|
||||
"vpc_different_regions",
|
||||
"vpc_subnet_different_az"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Id": "op.cont.3.aws.drs.1",
|
||||
"Description": "Pruebas periódicas",
|
||||
|
||||
@@ -1383,7 +1383,7 @@
|
||||
"Id": "3.7",
|
||||
"Description": "Ensure that 'Public Network Access' is `Disabled' for storage accounts",
|
||||
"Checks": [
|
||||
"storage_blob_public_access_level_is_disabled"
|
||||
"storage_account_public_network_access_disabled"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
@@ -1651,7 +1651,7 @@
|
||||
"Id": "4.6",
|
||||
"Description": "Ensure that 'Public Network Access' is 'Disabled' for storage accounts",
|
||||
"Checks": [
|
||||
"storage_blob_public_access_level_is_disabled"
|
||||
"storage_account_public_network_access_disabled"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
@@ -3021,7 +3021,7 @@
|
||||
"Id": "10.3.2.2",
|
||||
"Description": "Ensure that 'Public Network Access' is 'Disabled' for storage accounts",
|
||||
"Checks": [
|
||||
"storage_blob_public_access_level_is_disabled"
|
||||
"storage_account_public_network_access_disabled"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
@@ -3182,7 +3182,7 @@
|
||||
"Id": "9.3.2.2",
|
||||
"Description": "Ensure that 'Public Network Access' is 'Disabled' for storage accounts",
|
||||
"Checks": [
|
||||
"storage_blob_public_access_level_is_disabled"
|
||||
"storage_account_public_network_access_disabled"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -459,7 +459,7 @@
|
||||
"Id": "2.2.6",
|
||||
"Description": "Ensure that 'Public Network Access' is 'Disabled' for storage accounts",
|
||||
"Checks": [
|
||||
"storage_blob_public_access_level_is_disabled"
|
||||
"storage_account_public_network_access_disabled"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
@@ -0,0 +1,597 @@
|
||||
{
|
||||
"framework": "DORA",
|
||||
"name": "Digital Operational Resilience Act (Regulation (EU) 2022/2554)",
|
||||
"version": "2022/2554",
|
||||
"description": "The Digital Operational Resilience Act (DORA) is a European Union regulation (Regulation (EU) 2022/2554) that sets a uniform framework for the digital operational resilience of the EU financial sector. Mandatory since 17 January 2025, it applies to financial entities (banks, insurers, investment firms, payment institutions, etc.) and to ICT third-party service providers. DORA is structured around five pillars: ICT risk management, ICT-related incident reporting, digital operational resilience testing, ICT third-party risk management, and information sharing. This Prowler mapping covers the technical controls auditable from cloud configuration; the organisational, contractual and supervisory obligations defined in DORA must be addressed outside of Prowler.",
|
||||
"icon": "dora",
|
||||
"attributes_metadata": [
|
||||
{
|
||||
"key": "Pillar",
|
||||
"label": "Pillar",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"enum": [
|
||||
"ICT Risk Management",
|
||||
"ICT-Related Incident Reporting",
|
||||
"Digital Operational Resilience Testing",
|
||||
"ICT Third-Party Risk Management",
|
||||
"Information Sharing"
|
||||
],
|
||||
"output_formats": {
|
||||
"csv": true,
|
||||
"ocsf": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "Article",
|
||||
"label": "Article",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"output_formats": {
|
||||
"csv": true,
|
||||
"ocsf": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "ArticleTitle",
|
||||
"label": "Article Title",
|
||||
"type": "str",
|
||||
"required": true,
|
||||
"output_formats": {
|
||||
"csv": true,
|
||||
"ocsf": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": {
|
||||
"table_config": {
|
||||
"group_by": "Pillar"
|
||||
},
|
||||
"pdf_config": {
|
||||
"language": "en",
|
||||
"primary_color": "#003399",
|
||||
"secondary_color": "#0055A5",
|
||||
"bg_color": "#F0F4FA",
|
||||
"group_by_field": "Pillar",
|
||||
"sections": [
|
||||
"ICT Risk Management",
|
||||
"ICT-Related Incident Reporting",
|
||||
"Digital Operational Resilience Testing",
|
||||
"ICT Third-Party Risk Management",
|
||||
"Information Sharing"
|
||||
],
|
||||
"section_short_names": {
|
||||
"ICT Risk Management": "ICT Risk Mgmt",
|
||||
"ICT-Related Incident Reporting": "Incident Reporting",
|
||||
"Digital Operational Resilience Testing": "Resilience Testing",
|
||||
"ICT Third-Party Risk Management": "Third-Party Risk",
|
||||
"Information Sharing": "Info Sharing"
|
||||
},
|
||||
"charts": [
|
||||
{
|
||||
"id": "pillar_compliance",
|
||||
"type": "horizontal_bar",
|
||||
"group_by": "Pillar",
|
||||
"title": "Compliance Score by DORA Pillar",
|
||||
"y_label": "Pillar",
|
||||
"x_label": "Compliance %",
|
||||
"value_source": "compliance_percent",
|
||||
"color_mode": "by_value"
|
||||
}
|
||||
],
|
||||
"filter": {
|
||||
"only_failed": true,
|
||||
"include_manual": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"requirements": [
|
||||
{
|
||||
"id": "DORA-Art5",
|
||||
"name": "Governance and organisation",
|
||||
"description": "Financial entities shall have a sound, comprehensive and well-documented ICT internal governance and control framework. Senior management is accountable for ICT risk and shall enforce strong identity, authentication and least-privilege policies for privileged identities, including the root account.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 5",
|
||||
"ArticleTitle": "Governance and organisation"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"iam_avoid_root_usage",
|
||||
"iam_no_root_access_key",
|
||||
"iam_root_mfa_enabled",
|
||||
"iam_root_hardware_mfa_enabled",
|
||||
"iam_root_credentials_management_enabled",
|
||||
"iam_password_policy_minimum_length_14",
|
||||
"iam_password_policy_lowercase",
|
||||
"iam_password_policy_uppercase",
|
||||
"iam_password_policy_number",
|
||||
"iam_password_policy_symbol",
|
||||
"iam_password_policy_reuse_24",
|
||||
"iam_password_policy_expires_passwords_within_90_days_or_less",
|
||||
"iam_securityaudit_role_created",
|
||||
"iam_support_role_created",
|
||||
"organizations_account_part_of_organizations",
|
||||
"iam_user_mfa_enabled_console_access",
|
||||
"iam_user_hardware_mfa_enabled"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art6",
|
||||
"name": "ICT risk management framework",
|
||||
"description": "Financial entities shall have an ICT risk management framework that is sound, comprehensive and well-documented, enabling them to address ICT risk quickly, efficiently and comprehensively and to ensure a high level of digital operational resilience. This includes continuous configuration recording, security findings aggregation and an enterprise-wide visibility plane.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 6",
|
||||
"ArticleTitle": "ICT risk management framework"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"config_recorder_all_regions_enabled",
|
||||
"config_recorder_using_aws_service_role",
|
||||
"securityhub_enabled",
|
||||
"accessanalyzer_enabled",
|
||||
"accessanalyzer_enabled_without_findings",
|
||||
"organizations_delegated_administrators",
|
||||
"guardduty_centrally_managed",
|
||||
"guardduty_delegated_admin_enabled_all_regions"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art7",
|
||||
"name": "ICT systems, protocols and tools",
|
||||
"description": "Financial entities shall use and maintain updated ICT systems, protocols and tools that are appropriate to the magnitude of operations supporting ICT functions, technologically resilient, and adequately equipped to securely process data. Cryptographic primitives, certificate hygiene and network segmentation are core to this requirement.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 7",
|
||||
"ArticleTitle": "ICT systems, protocols and tools"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"acm_certificates_with_secure_key_algorithms",
|
||||
"acm_certificates_transparency_logs_enabled",
|
||||
"acm_certificates_expiration_check",
|
||||
"ec2_ebs_default_encryption",
|
||||
"kms_cmk_rotation_enabled",
|
||||
"s3_bucket_secure_transport_policy",
|
||||
"s3_bucket_default_encryption",
|
||||
"s3_bucket_kms_encryption",
|
||||
"vpc_subnet_separate_private_public",
|
||||
"vpc_subnet_no_public_ip_by_default",
|
||||
"elb_insecure_ssl_ciphers",
|
||||
"elbv2_insecure_ssl_ciphers",
|
||||
"elb_ssl_listeners",
|
||||
"elbv2_ssl_listeners",
|
||||
"cloudfront_distributions_using_deprecated_ssl_protocols",
|
||||
"cloudfront_distributions_https_enabled",
|
||||
"rds_instance_transport_encrypted"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art8",
|
||||
"name": "Identification",
|
||||
"description": "Financial entities shall identify, classify and adequately document all ICT supported business functions, roles and responsibilities, the information assets and ICT assets supporting them, and their interdependencies. They shall on a continuous basis identify all sources of ICT risk, in particular the risk exposure to and from other financial entities.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 8",
|
||||
"ArticleTitle": "Identification"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"accessanalyzer_enabled",
|
||||
"accessanalyzer_enabled_without_findings",
|
||||
"macie_is_enabled",
|
||||
"macie_automated_sensitive_data_discovery_enabled",
|
||||
"ec2_securitygroup_not_used",
|
||||
"ec2_elastic_ip_unassigned",
|
||||
"ec2_networkacl_unused",
|
||||
"secretsmanager_secret_unused"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art9",
|
||||
"name": "Protection and prevention",
|
||||
"description": "Financial entities shall continuously monitor and control the security and functioning of ICT systems and tools and minimise the impact of ICT risk by deploying appropriate ICT security tools, policies and procedures. Encryption at rest and in transit, blocking of public exposure, network access controls, secret management and instance hardening are central to this article.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 9",
|
||||
"ArticleTitle": "Protection and prevention"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"kms_key_not_publicly_accessible",
|
||||
"ec2_ebs_volume_encryption",
|
||||
"ec2_ebs_snapshots_encrypted",
|
||||
"ec2_ebs_public_snapshot",
|
||||
"ec2_ebs_snapshot_account_block_public_access",
|
||||
"s3_account_level_public_access_blocks",
|
||||
"s3_bucket_level_public_access_block",
|
||||
"s3_bucket_public_access",
|
||||
"s3_bucket_policy_public_write_access",
|
||||
"s3_bucket_public_write_acl",
|
||||
"s3_bucket_public_list_acl",
|
||||
"s3_bucket_acl_prohibited",
|
||||
"s3_access_point_public_access_block",
|
||||
"ec2_securitygroup_default_restrict_traffic",
|
||||
"ec2_securitygroup_allow_ingress_from_internet_to_all_ports",
|
||||
"ec2_securitygroup_allow_ingress_from_internet_to_any_port",
|
||||
"ec2_securitygroup_allow_ingress_from_internet_to_high_risk_tcp_ports",
|
||||
"ec2_securitygroup_allow_ingress_from_internet_to_tcp_port_22",
|
||||
"ec2_securitygroup_allow_ingress_from_internet_to_tcp_port_3389",
|
||||
"rds_instance_storage_encrypted",
|
||||
"rds_cluster_storage_encrypted",
|
||||
"rds_instance_no_public_access",
|
||||
"rds_snapshots_public_access",
|
||||
"secretsmanager_not_publicly_accessible",
|
||||
"secretsmanager_has_restrictive_resource_policy",
|
||||
"secretsmanager_automatic_rotation_enabled",
|
||||
"dynamodb_tables_kms_cmk_encryption_enabled",
|
||||
"sns_topics_kms_encryption_at_rest_enabled",
|
||||
"sns_topics_not_publicly_accessible",
|
||||
"ec2_instance_imdsv2_enabled",
|
||||
"ec2_instance_account_imdsv2_enabled",
|
||||
"efs_encryption_at_rest_enabled",
|
||||
"awslambda_function_not_publicly_accessible"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art10",
|
||||
"name": "Detection",
|
||||
"description": "Financial entities shall have in place mechanisms to promptly detect anomalous activities, including ICT network performance issues and ICT-related incidents, and to identify potential single points of failure. Threat detection across compute, identity, storage and the API control plane is required for timely detection.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 10",
|
||||
"ArticleTitle": "Detection"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"guardduty_is_enabled",
|
||||
"guardduty_no_high_severity_findings",
|
||||
"guardduty_ec2_malware_protection_enabled",
|
||||
"guardduty_lambda_protection_enabled",
|
||||
"guardduty_rds_protection_enabled",
|
||||
"guardduty_s3_protection_enabled",
|
||||
"guardduty_eks_audit_log_enabled",
|
||||
"guardduty_eks_runtime_monitoring_enabled",
|
||||
"securityhub_enabled",
|
||||
"cloudtrail_threat_detection_enumeration",
|
||||
"cloudtrail_threat_detection_llm_jacking",
|
||||
"cloudtrail_threat_detection_privilege_escalation",
|
||||
"cloudtrail_insights_exist",
|
||||
"inspector2_is_enabled",
|
||||
"inspector2_active_findings_exist",
|
||||
"ec2_elastic_ip_shodan"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art11",
|
||||
"name": "Response and recovery",
|
||||
"description": "Financial entities shall put in place a comprehensive ICT business continuity policy, including ICT response and recovery plans, that ensures the continuity of ICT-supported critical or important functions. Operational alarming, automated event routing and tested recovery actions are essential.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 11",
|
||||
"ArticleTitle": "Response and recovery"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"cloudwatch_alarm_actions_enabled",
|
||||
"cloudwatch_alarm_actions_alarm_state_configured",
|
||||
"eventbridge_global_endpoint_event_replication_enabled",
|
||||
"sns_subscription_not_using_http_endpoints",
|
||||
"backup_plans_exist",
|
||||
"backup_vaults_exist",
|
||||
"rds_instance_critical_event_subscription",
|
||||
"rds_cluster_critical_event_subscription"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art12",
|
||||
"name": "Backup policies and procedures, restoration and recovery procedures and methods",
|
||||
"description": "Financial entities shall develop and document backup policies and procedures specifying the scope of data subject to backup and the minimum frequency of the backup, as well as restoration and recovery procedures and methods. Backups must be encrypted, retained, and resources must be designed for recoverability across availability zones and regions.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 12",
|
||||
"ArticleTitle": "Backup policies and procedures, restoration and recovery procedures and methods"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"backup_plans_exist",
|
||||
"backup_vaults_exist",
|
||||
"backup_vaults_encrypted",
|
||||
"backup_recovery_point_encrypted",
|
||||
"backup_reportplans_exist",
|
||||
"rds_instance_backup_enabled",
|
||||
"rds_cluster_protected_by_backup_plan",
|
||||
"rds_instance_protected_by_backup_plan",
|
||||
"rds_instance_multi_az",
|
||||
"rds_cluster_multi_az",
|
||||
"rds_cluster_backtrack_enabled",
|
||||
"rds_instance_deletion_protection",
|
||||
"rds_cluster_deletion_protection",
|
||||
"rds_snapshots_encrypted",
|
||||
"s3_bucket_object_versioning",
|
||||
"s3_bucket_object_lock",
|
||||
"s3_bucket_cross_region_replication",
|
||||
"s3_bucket_no_mfa_delete",
|
||||
"dynamodb_tables_pitr_enabled",
|
||||
"dynamodb_table_deletion_protection_enabled",
|
||||
"ec2_ebs_volume_protected_by_backup_plan",
|
||||
"ec2_ebs_volume_snapshots_exists",
|
||||
"autoscaling_group_multiple_az",
|
||||
"elb_is_in_multiple_az",
|
||||
"elbv2_is_in_multiple_az",
|
||||
"cloudfront_distributions_multiple_origin_failover_configured",
|
||||
"dynamodb_table_protected_by_backup_plan"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art13",
|
||||
"name": "Learning and evolving",
|
||||
"description": "Financial entities shall have in place capabilities and staff to gather information on vulnerabilities and cyber threats, perform post ICT-related incident reviews, and continuously feed lessons learnt back into the ICT risk assessment process. Findings aggregation and continuous insights drive this cycle.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 13",
|
||||
"ArticleTitle": "Learning and evolving"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"securityhub_enabled",
|
||||
"guardduty_no_high_severity_findings",
|
||||
"inspector2_active_findings_exist",
|
||||
"accessanalyzer_enabled_without_findings",
|
||||
"cloudtrail_insights_exist"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art14",
|
||||
"name": "Communication",
|
||||
"description": "As part of the ICT risk management framework, financial entities shall have in place crisis communication plans enabling a responsible disclosure of ICT-related incidents or major vulnerabilities to clients, counterparts and the public. Reliable, encrypted and access-controlled notification channels are required.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Risk Management",
|
||||
"Article": "Article 14",
|
||||
"ArticleTitle": "Communication"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"sns_topics_kms_encryption_at_rest_enabled",
|
||||
"sns_topics_not_publicly_accessible",
|
||||
"sns_subscription_not_using_http_endpoints",
|
||||
"eventbridge_bus_exposed",
|
||||
"eventbridge_bus_cross_account_access",
|
||||
"eventbridge_schema_registry_cross_account_access",
|
||||
"cloudwatch_alarm_actions_enabled",
|
||||
"cloudwatch_alarm_actions_alarm_state_configured"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art17",
|
||||
"name": "ICT-related incident management process",
|
||||
"description": "Financial entities shall define, establish and implement an ICT-related incident management process to detect, manage and notify ICT-related incidents. Comprehensive trail logging, log integrity protection, retention and centralisation of ICT events are foundational requirements.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT-Related Incident Reporting",
|
||||
"Article": "Article 17",
|
||||
"ArticleTitle": "ICT-related incident management process"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"cloudtrail_multi_region_enabled",
|
||||
"cloudtrail_multi_region_enabled_logging_management_events",
|
||||
"cloudtrail_kms_encryption_enabled",
|
||||
"cloudtrail_log_file_validation_enabled",
|
||||
"cloudtrail_cloudwatch_logging_enabled",
|
||||
"cloudtrail_logs_s3_bucket_access_logging_enabled",
|
||||
"cloudtrail_logs_s3_bucket_is_not_publicly_accessible",
|
||||
"cloudtrail_s3_dataevents_read_enabled",
|
||||
"cloudtrail_s3_dataevents_write_enabled",
|
||||
"cloudtrail_bucket_requires_mfa_delete",
|
||||
"cloudtrail_bedrock_logging_enabled",
|
||||
"cloudwatch_log_group_retention_policy_specific_days_enabled",
|
||||
"cloudwatch_log_group_kms_encryption_enabled",
|
||||
"cloudwatch_log_group_no_secrets_in_logs",
|
||||
"cloudwatch_log_group_not_publicly_accessible",
|
||||
"vpc_flow_logs_enabled",
|
||||
"ec2_client_vpn_endpoint_connection_logging_enabled",
|
||||
"route53_public_hosted_zones_cloudwatch_logging_enabled",
|
||||
"elb_logging_enabled",
|
||||
"elbv2_logging_enabled",
|
||||
"cloudfront_distributions_logging_enabled",
|
||||
"s3_bucket_server_access_logging_enabled"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art18",
|
||||
"name": "Classification of ICT-related incidents and cyber threats",
|
||||
"description": "Financial entities shall classify ICT-related incidents and shall determine their impact based on criteria such as the number of clients affected, duration, geographical spread, data losses, and criticality of the services affected. Severity-aware threat detection across the estate underpins this classification.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT-Related Incident Reporting",
|
||||
"Article": "Article 18",
|
||||
"ArticleTitle": "Classification of ICT-related incidents and cyber threats"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"guardduty_no_high_severity_findings",
|
||||
"guardduty_centrally_managed",
|
||||
"guardduty_delegated_admin_enabled_all_regions",
|
||||
"securityhub_enabled",
|
||||
"inspector2_active_findings_exist",
|
||||
"accessanalyzer_enabled_without_findings",
|
||||
"cloudtrail_threat_detection_enumeration",
|
||||
"cloudtrail_threat_detection_llm_jacking",
|
||||
"cloudtrail_threat_detection_privilege_escalation"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art19",
|
||||
"name": "Reporting of major ICT-related incidents and voluntary notification of significant cyber threats",
|
||||
"description": "Financial entities shall report major ICT-related incidents to the relevant competent authority and may, on a voluntary basis, notify significant cyber threats. Detective metric filters, change-tracking alarms and reliable notification topics are needed to surface and route reportable events.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT-Related Incident Reporting",
|
||||
"Article": "Article 19",
|
||||
"ArticleTitle": "Reporting of major ICT-related incidents and voluntary notification of significant cyber threats"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"cloudwatch_log_metric_filter_authentication_failures",
|
||||
"cloudwatch_log_metric_filter_unauthorized_api_calls",
|
||||
"cloudwatch_log_metric_filter_root_usage",
|
||||
"cloudwatch_log_metric_filter_sign_in_without_mfa",
|
||||
"cloudwatch_log_metric_filter_disable_or_scheduled_deletion_of_kms_cmk",
|
||||
"cloudwatch_log_metric_filter_for_s3_bucket_policy_changes",
|
||||
"cloudwatch_log_metric_filter_policy_changes",
|
||||
"cloudwatch_log_metric_filter_security_group_changes",
|
||||
"cloudwatch_log_metric_filter_aws_organizations_changes",
|
||||
"cloudwatch_log_metric_filter_and_alarm_for_aws_config_configuration_changes_enabled",
|
||||
"cloudwatch_log_metric_filter_and_alarm_for_cloudtrail_configuration_changes_enabled",
|
||||
"cloudwatch_changes_to_network_acls_alarm_configured",
|
||||
"cloudwatch_changes_to_network_gateways_alarm_configured",
|
||||
"cloudwatch_changes_to_network_route_tables_alarm_configured",
|
||||
"cloudwatch_changes_to_vpcs_alarm_configured",
|
||||
"sns_subscription_not_using_http_endpoints"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art24",
|
||||
"name": "General requirements for the performance of digital operational resilience testing",
|
||||
"description": "Financial entities shall establish, maintain and review a sound and comprehensive digital operational resilience testing programme, as an integral part of the ICT risk management framework. Continuous vulnerability discovery, configuration assessment and instance manageability are foundational.",
|
||||
"attributes": {
|
||||
"Pillar": "Digital Operational Resilience Testing",
|
||||
"Article": "Article 24",
|
||||
"ArticleTitle": "General requirements for the performance of digital operational resilience testing"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"inspector2_is_enabled",
|
||||
"inspector2_active_findings_exist",
|
||||
"securityhub_enabled",
|
||||
"ec2_instance_managed_by_ssm",
|
||||
"ec2_instance_with_outdated_ami",
|
||||
"ssm_managed_compliant_patching"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art25",
|
||||
"name": "Testing of ICT tools and systems",
|
||||
"description": "Financial entities shall ensure that tests are undertaken on ICT tools and systems, on critical ICT systems supporting all critical or important functions, at least yearly. Vulnerability assessments, deprecated component detection and certificate hygiene must be tracked.",
|
||||
"attributes": {
|
||||
"Pillar": "Digital Operational Resilience Testing",
|
||||
"Article": "Article 25",
|
||||
"ArticleTitle": "Testing of ICT tools and systems"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"inspector2_is_enabled",
|
||||
"inspector2_active_findings_exist",
|
||||
"guardduty_is_enabled",
|
||||
"guardduty_no_high_severity_findings",
|
||||
"config_recorder_all_regions_enabled",
|
||||
"ec2_instance_with_outdated_ami",
|
||||
"ec2_instance_managed_by_ssm",
|
||||
"ec2_instance_paravirtual_type",
|
||||
"rds_instance_deprecated_engine_version",
|
||||
"acm_certificates_expiration_check",
|
||||
"rds_instance_certificate_expiration",
|
||||
"iam_no_expired_server_certificates_stored",
|
||||
"ssm_managed_compliant_patching"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art28",
|
||||
"name": "General principles (ICT third-party risk)",
|
||||
"description": "Financial entities shall manage ICT third-party risk as an integral component of ICT risk within their ICT risk management framework. Cross-account access, trust boundaries, organization-level controls and dependency visibility are critical to monitor third-party exposure on AWS.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Third-Party Risk Management",
|
||||
"Article": "Article 28",
|
||||
"ArticleTitle": "General principles (ICT third-party risk)"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"iam_role_cross_service_confused_deputy_prevention",
|
||||
"iam_role_cross_account_readonlyaccess_policy",
|
||||
"iam_no_custom_policy_permissive_role_assumption",
|
||||
"accessanalyzer_enabled",
|
||||
"accessanalyzer_enabled_without_findings",
|
||||
"s3_bucket_cross_account_access",
|
||||
"dynamodb_table_cross_account_access",
|
||||
"eventbridge_bus_cross_account_access",
|
||||
"eventbridge_schema_registry_cross_account_access",
|
||||
"cloudwatch_cross_account_sharing_disabled",
|
||||
"organizations_delegated_administrators",
|
||||
"organizations_account_part_of_organizations",
|
||||
"organizations_scp_check_deny_regions",
|
||||
"vpc_endpoint_connections_trust_boundaries",
|
||||
"vpc_endpoint_services_allowed_principals_trust_boundaries",
|
||||
"vpc_peering_routing_tables_with_least_privilege",
|
||||
"awslambda_function_using_cross_account_layers"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art30",
|
||||
"name": "Key contractual provisions",
|
||||
"description": "Contractual arrangements with ICT third-party service providers shall be set out in writing and include, at minimum, agreed service levels and clear allocation of rights and obligations. Privilege boundaries, least-privilege policies and absence of administrative wildcards are the technical guardrails that enforce these contractual constraints inside AWS.",
|
||||
"attributes": {
|
||||
"Pillar": "ICT Third-Party Risk Management",
|
||||
"Article": "Article 30",
|
||||
"ArticleTitle": "Key contractual provisions"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"iam_aws_attached_policy_no_administrative_privileges",
|
||||
"iam_customer_attached_policy_no_administrative_privileges",
|
||||
"iam_customer_unattached_policy_no_administrative_privileges",
|
||||
"iam_inline_policy_no_administrative_privileges",
|
||||
"iam_inline_policy_allows_privilege_escalation",
|
||||
"iam_policy_allows_privilege_escalation",
|
||||
"iam_inline_policy_no_full_access_to_cloudtrail",
|
||||
"iam_inline_policy_no_full_access_to_kms",
|
||||
"iam_policy_no_full_access_to_cloudtrail",
|
||||
"iam_policy_no_full_access_to_kms",
|
||||
"iam_role_administratoraccess_policy",
|
||||
"iam_user_administrator_access_policy",
|
||||
"iam_group_administrator_access_policy",
|
||||
"iam_administrator_access_with_mfa",
|
||||
"iam_policy_attached_only_to_group_or_roles",
|
||||
"accessanalyzer_enabled"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DORA-Art45",
|
||||
"name": "Information-sharing arrangements on cyber threat information and intelligence",
|
||||
"description": "Financial entities may exchange amongst themselves cyber threat information and intelligence, including indicators of compromise, tactics, techniques and procedures, cyber security alerts and configuration tools. Centralised threat detection, sensitive data discovery and trail-based intelligence enable participation in such information-sharing arrangements.",
|
||||
"attributes": {
|
||||
"Pillar": "Information Sharing",
|
||||
"Article": "Article 45",
|
||||
"ArticleTitle": "Information-sharing arrangements on cyber threat information and intelligence"
|
||||
},
|
||||
"checks": {
|
||||
"aws": [
|
||||
"guardduty_is_enabled",
|
||||
"guardduty_centrally_managed",
|
||||
"securityhub_enabled",
|
||||
"macie_is_enabled",
|
||||
"macie_automated_sensitive_data_discovery_enabled",
|
||||
"cloudtrail_threat_detection_enumeration",
|
||||
"cloudtrail_threat_detection_llm_jacking",
|
||||
"cloudtrail_threat_detection_privilege_escalation",
|
||||
"accessanalyzer_enabled_without_findings"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -889,7 +889,7 @@
|
||||
}
|
||||
],
|
||||
"Checks": [
|
||||
"kms_key_rotation_enabled"
|
||||
"kms_key_rotation_max_90_days"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -150,7 +150,7 @@
|
||||
"Id": "1.10",
|
||||
"Description": "Google Cloud Key Management Service stores cryptographic keys in a hierarchical structure designed for useful and elegant access control management. The format for the rotation schedule depends on the client library that is used. For the gcloud command-line tool, the next rotation time must be in `ISO` or `RFC3339` format, and the rotation period must be in the form `INTEGERUNIT`, where units can be one of seconds (s), minutes (m), hours (h) or days (d).",
|
||||
"Checks": [
|
||||
"kms_key_rotation_enabled"
|
||||
"kms_key_rotation_max_90_days"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
@@ -201,7 +201,7 @@
|
||||
"Id": "1.10",
|
||||
"Description": "Ensure KMS Encryption Keys Are Rotated Within a Period of 90 Days",
|
||||
"Checks": [
|
||||
"kms_key_rotation_enabled"
|
||||
"kms_key_rotation_max_90_days"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
@@ -201,7 +201,7 @@
|
||||
"Id": "1.10",
|
||||
"Description": "Ensure KMS Encryption Keys Are Rotated Within a Period of 90 Days",
|
||||
"Checks": [
|
||||
"kms_key_rotation_enabled"
|
||||
"kms_key_rotation_max_90_days"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,7 +117,7 @@
|
||||
"Id": "1.2.4",
|
||||
"Description": "Ensure KMS Encryption Keys Are Rotated Within a Period of 90 Days",
|
||||
"Checks": [
|
||||
"kms_key_rotation_enabled"
|
||||
"kms_key_rotation_max_90_days"
|
||||
],
|
||||
"Attributes": [
|
||||
{
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user