Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de097521f4 | ||
|
|
f0238a82d9 | ||
|
|
17c8ed3757 | ||
|
|
76a2d7bfe6 | ||
|
|
ac3f289de6 | ||
|
|
838f82b255 | ||
|
|
5e00c4bfcf | ||
|
|
f0ae56b8ea | ||
|
|
4484c2f192 | ||
|
|
e6bbcb8043 | ||
|
|
ad04e69c35 | ||
|
|
6cae37174c | ||
|
|
aa6de57430 | ||
|
|
a48aa37f87 | ||
|
|
3cc8f86780 | ||
|
|
4cb02a0ead | ||
|
|
221c558cee | ||
|
|
81c3152ebb | ||
|
|
855e9a043e | ||
|
|
7b5d724bb7 | ||
|
|
441f2a3c48 | ||
|
|
efb86bb7ab | ||
|
|
398a0a484f | ||
|
|
55924d8150 | ||
|
|
0cf6f2f83e | ||
|
|
cf18093261 | ||
|
|
1850e209e6 | ||
|
|
2e37188c9f | ||
|
|
4ae7c67d3f | ||
|
|
a76ba156d5 | ||
|
|
cd90a91158 | ||
|
|
e1b23e2526 | ||
|
|
537c3ea71e | ||
|
|
b6f74c7284 | ||
|
|
8cd008ba91 | ||
|
|
1f13e1d348 | ||
|
|
87a15d7bb8 | ||
|
|
f5bdacd07a | ||
|
|
ce80fcd430 | ||
|
|
c1c080b072 | ||
|
|
587187419f | ||
|
|
050a5915ca | ||
|
|
d4e4d12c5a | ||
|
|
72cf2a65a6 | ||
|
|
48db27481d | ||
|
|
1247c5fb33 | ||
|
|
69321418a3 | ||
|
|
301d13a4b9 | ||
|
|
3f8c1e822f | ||
|
|
1e1c1c018b | ||
|
|
a212916a49 | ||
|
|
883ffa1fdb | ||
|
|
21d9d6192e | ||
|
|
fd38a0ac03 | ||
|
|
af6918d57b | ||
|
|
9a9cbc997b | ||
|
|
aec500ee3b | ||
|
|
8fbc721223 |
@@ -145,19 +145,20 @@ DJANGO_BROKER_VISIBILITY_TIMEOUT=86400
|
||||
DJANGO_SENTRY_DSN=
|
||||
DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
|
||||
|
||||
# Sentry for the web app (server + browser). Empty/unset UI_SENTRY_DSN ⇒
|
||||
# Sentry disabled, zero egress. SENTRY_RELEASE (unprefixed) feeds the web app's
|
||||
# server/edge SDKs.
|
||||
# Sentry for the web app (server + browser). The UI_SENTRY_* values load only
|
||||
# when UI_SENTRY_ENABLE="true"; without it they are ignored (default off, zero
|
||||
# egress). The deprecated NEXT_PUBLIC_SENTRY_DSN still activates Sentry without
|
||||
# the flag. SENTRY_RELEASE (unprefixed) feeds the web app's server/edge SDKs.
|
||||
UI_SENTRY_DSN=
|
||||
UI_SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
# Reserved runtime public config (registered now; no UI consumer yet)
|
||||
# POSTHOG_KEY=
|
||||
# POSTHOG_HOST=
|
||||
# UI_POSTHOG_KEY=
|
||||
# UI_POSTHOG_HOST=
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.32.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.33.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
# v5
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/api"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "component/api"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github_actions"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "npm"
|
||||
# directory: "/ui"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "npm"
|
||||
# - "component/ui"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# - package-ecosystem: "pre-commit"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pre-commit"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "v4"
|
||||
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# - "v4"
|
||||
|
||||
# - package-ecosystem: "docker"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "docker"
|
||||
# - "v4"
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# v3
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v3
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "v3"
|
||||
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v3
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# - "v3"
|
||||
@@ -38,7 +38,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 1 * *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Minors: 8th of every 3 months, Madrid overnight window (22:00-06:00)",
|
||||
@@ -48,7 +48,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 8 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Majors: 15th of every 3 months, Madrid overnight window",
|
||||
@@ -58,7 +58,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 15 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions - single grouped PR, no changelog, scope=ci",
|
||||
|
||||
@@ -9,6 +9,7 @@ on:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-codeql.yml'
|
||||
- '.github/codeql/api-codeql-config.yml'
|
||||
- '!api/CHANGELOG.md'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -17,6 +18,7 @@ on:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-codeql.yml'
|
||||
- '.github/codeql/api-codeql-config.yml'
|
||||
- '!api/CHANGELOG.md'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
|
||||
@@ -215,7 +215,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17@sha256:2cd82735a36356842d5eb1ef80db3ae8f1154172f0f653db48fde079b2a0b7f7
|
||||
image: postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e
|
||||
env:
|
||||
POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
|
||||
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine3.19
|
||||
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
|
||||
env:
|
||||
VALKEY_HOST: ${{ env.VALKEY_HOST }}
|
||||
VALKEY_PORT: ${{ env.VALKEY_PORT }}
|
||||
|
||||
@@ -206,7 +206,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -37,8 +37,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
persist-credentials: false # No write token in the untrusted PR-head tree; public repo so base fetch/changed-files work unauthenticated
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
@@ -50,6 +49,8 @@ jobs:
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: '**'
|
||||
safe_output: false # Raw paths (list read via env var, injection-safe); default escaping backslash-quotes chars like () and breaks the -f test
|
||||
separator: "\n" # Newline-delimited so the reader tolerates spaces and glob chars in paths
|
||||
|
||||
- name: Check for conflict markers
|
||||
id: conflict-check
|
||||
@@ -59,19 +60,18 @@ jobs:
|
||||
CONFLICT_FILES=""
|
||||
HAS_CONFLICTS=false
|
||||
|
||||
# Check each changed file for conflict markers
|
||||
for file in ${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}; do
|
||||
if [ -f "$file" ]; then
|
||||
echo "Checking file: $file"
|
||||
# Read newline-delimited paths so spaces/globs neither word-split nor glob-expand
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] || continue
|
||||
[ -f "$file" ] || continue
|
||||
echo "Checking file: $file"
|
||||
|
||||
# Look for conflict markers (more precise regex)
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
done
|
||||
done <<< "$STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES"
|
||||
|
||||
if [ "$HAS_CONFLICTS" = true ]; then
|
||||
echo "has_conflicts=true" >> $GITHUB_OUTPUT
|
||||
@@ -88,18 +88,49 @@ jobs:
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
|
||||
- name: Check base-branch mergeability
|
||||
id: merge-check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
MERGEABLE=null
|
||||
|
||||
# GitHub computes mergeability async, so .mergeable is null until ready; poll until resolved
|
||||
for attempt in 1 2 3 4 5; do
|
||||
MERGEABLE=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.mergeable')
|
||||
if [ "$MERGEABLE" != "null" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Attempt ${attempt}: mergeability not computed yet, retrying..."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# Keep 'unknown' distinct from 'clean' so we never assert a clean merge we could not confirm
|
||||
case "$MERGEABLE" in
|
||||
false) STATUS=conflict; echo "PR branch cannot be merged cleanly into its base branch" ;;
|
||||
true) STATUS=clean; echo "PR branch merges cleanly into its base branch" ;;
|
||||
*) STATUS=unknown; echo "::warning::Mergeability did not resolve after retries; leaving it undetermined" ;;
|
||||
esac
|
||||
|
||||
echo "merge_status=${STATUS}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Manage conflict label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
LABEL_NAME="has-conflicts"
|
||||
|
||||
# Add or remove label based on conflict status
|
||||
if [ "$HAS_CONFLICTS" = "true" ]; then
|
||||
if [ "$HAS_CONFLICTS" = "true" ] || [ "$MERGE_STATUS" = "conflict" ]; then
|
||||
echo "Adding conflict label to PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
elif [ "$MERGE_STATUS" = "unknown" ]; then
|
||||
# Don't drop the label on an undetermined merge state; a later run will settle it
|
||||
echo "Mergeability undetermined; leaving label unchanged"
|
||||
else
|
||||
echo "Removing conflict label from PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
@@ -121,20 +152,25 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- conflict-checker-comment -->
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠️ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }}
|
||||
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files:
|
||||
${{ (steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict') && '⚠️ **Conflicts Detected**' || (steps.merge-check.outputs.merge_status == 'unknown' && 'ℹ️ **Conflict Check Incomplete**' || '✅ **No Conflicts**') }}
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('
|
||||
**Conflict markers** are present in the following files:
|
||||
|
||||
{0}
|
||||
|
||||
Please resolve these conflicts by:
|
||||
1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>`
|
||||
2. Manually editing the files to resolve the conflicts
|
||||
3. Removing all conflict markers
|
||||
4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }}
|
||||
Resolve them by removing every `<<<<<<<`, `=======`, and `>>>>>>>` marker, then commit and push.', steps.conflict-check.outputs.conflict_files) || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'conflict' && '
|
||||
**Merge conflict with the base branch.** This PR cannot be merged cleanly. Update your branch with the latest base (rebase or merge) and resolve the conflicts.' || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'unknown' && '
|
||||
GitHub had not finished computing mergeability, so base-branch conflict status could not be verified on this run.' || '' }}
|
||||
${{ (steps.conflict-check.outputs.has_conflicts != 'true' && steps.merge-check.outputs.merge_status == 'clean') && '
|
||||
No conflict markers, and the branch merges cleanly into its base.' || '' }}
|
||||
|
||||
- name: Fail workflow if conflicts detected
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict'
|
||||
env:
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
echo "::error::Workflow failed due to conflict markers detected in the PR"
|
||||
[ "$HAS_CONFLICTS" = "true" ] && echo "::error::Conflict markers detected in changed files"
|
||||
[ "$MERGE_STATUS" = "conflict" ] && echo "::error::PR branch has merge conflicts with the base branch"
|
||||
exit 1
|
||||
|
||||
@@ -56,6 +56,6 @@ jobs:
|
||||
"PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }},
|
||||
"PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }},
|
||||
"PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}",
|
||||
"PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}",
|
||||
"PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}"
|
||||
"PROWLER_PR_BASE_BRANCH": ${{ toJson(github.event.pull_request.base.ref) }},
|
||||
"PROWLER_PR_HEAD_BRANCH": ${{ toJson(github.event.pull_request.head.ref) }}
|
||||
}
|
||||
|
||||
@@ -138,6 +138,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -147,6 +148,8 @@ jobs:
|
||||
allowed-endpoints: >
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
public.ecr.aws:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
@@ -173,14 +176,16 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
@@ -206,6 +211,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -221,6 +227,8 @@ jobs:
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
|
||||
|
||||
- name: Login to DockerHub
|
||||
@@ -229,14 +237,16 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -299,7 +309,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install PyYAML
|
||||
run: pip install pyyaml
|
||||
|
||||
@@ -201,7 +201,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
|
||||
@@ -5,71 +5,30 @@ name: UI - E2E Tests (Optimized)
|
||||
# critical paths are changed or if impact analysis fails.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- "v5.*"
|
||||
paths:
|
||||
- '.github/workflows/ui-e2e-tests-v2.yml'
|
||||
- '.github/workflows/test-impact-analysis.yml'
|
||||
- '.github/test-impact.yml'
|
||||
- 'ui/**'
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- labeled
|
||||
- unlabeled
|
||||
branches:
|
||||
- master
|
||||
- "v5.*"
|
||||
paths:
|
||||
- '.github/workflows/ui-e2e-tests-v2.yml'
|
||||
- '.github/workflows/test-impact-analysis.yml'
|
||||
- '.github/test-impact.yml'
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!api/CHANGELOG.md'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
# Trusted PR authors get the opt-in label automatically. This job does not
|
||||
# check out or execute PR code; it only calls the GitHub API for trusted users.
|
||||
auto-label-trusted-pr:
|
||||
if: |
|
||||
github.event_name == 'pull_request' &&
|
||||
(github.event.action == 'opened' || github.event.action == 'reopened') &&
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) &&
|
||||
!contains(github.event.pull_request.labels.*.name, 'run-ui-e2e')
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Add UI E2E opt-in label
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
||||
with:
|
||||
script: |
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.payload.pull_request.number,
|
||||
labels: ['run-ui-e2e'],
|
||||
});
|
||||
|
||||
# UI E2E consumes cloud credentials, so PR runs require explicit maintainer opt-in.
|
||||
# On protected branch pushes, run independently of PR labels.
|
||||
# First, analyze which tests need to run
|
||||
impact-analysis:
|
||||
if: |
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
(github.event_name == 'push' ||
|
||||
contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') ||
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
permissions:
|
||||
contents: read
|
||||
uses: ./.github/workflows/test-impact-analysis.yml
|
||||
|
||||
# Run E2E tests based on impact analysis
|
||||
@@ -77,12 +36,8 @@ jobs:
|
||||
needs: impact-analysis
|
||||
if: |
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
(github.event_name == 'push' ||
|
||||
contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') ||
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) &&
|
||||
(needs.impact-analysis.outputs.has-ui-e2e == 'true' || needs.impact-analysis.outputs.run-all == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
environment: ui-e2e-cloud
|
||||
env:
|
||||
AUTH_SECRET: 'fallback-ci-secret-for-testing'
|
||||
AUTH_TRUST_HOST: true
|
||||
@@ -183,8 +138,8 @@ jobs:
|
||||
|
||||
- name: Add AWS credentials for testing
|
||||
run: |
|
||||
echo "AWS_ACCESS_KEY_ID=${E2E_AWS_PROVIDER_ACCESS_KEY}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${E2E_AWS_PROVIDER_SECRET_KEY}" >> .env
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
|
||||
- name: Build API image from current code
|
||||
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
|
||||
@@ -349,38 +304,11 @@ jobs:
|
||||
run: |
|
||||
docker compose down -v || true
|
||||
|
||||
# Skip job - provides clear feedback when UI E2E is not explicitly authorized.
|
||||
skip-e2e-no-label:
|
||||
if: |
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
github.event_name == 'pull_request' &&
|
||||
!contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') &&
|
||||
!contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: UI E2E skipped - opt-in label missing
|
||||
run: |
|
||||
echo "## UI E2E Tests Skipped" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "UI E2E tests consume cloud credentials and are skipped unless a maintainer adds the run-ui-e2e label." >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Add the label to opt in; remove it to stop secret-consuming UI E2E jobs." >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
# Skip job - provides clear feedback when no E2E tests needed after opt-in.
|
||||
skip-e2e-no-tests:
|
||||
# Skip job - provides clear feedback when no E2E tests needed
|
||||
skip-e2e:
|
||||
needs: impact-analysis
|
||||
if: |
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
(github.event_name == 'push' ||
|
||||
contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') ||
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) &&
|
||||
needs.impact-analysis.outputs.has-ui-e2e != 'true' &&
|
||||
needs.impact-analysis.outputs.run-all != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -37,8 +37,12 @@ jobs:
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
fonts.googleapis.com:443
|
||||
fonts.gstatic.com:443
|
||||
api.iconify.design:443
|
||||
api.simplesvg.com:443
|
||||
api.unisvg.com:443
|
||||
api.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
cdn.playwright.dev:443
|
||||
|
||||
@@ -72,13 +72,13 @@ repos:
|
||||
exclude: contrib
|
||||
priority: 30
|
||||
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/)
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/, docs/scripts/)
|
||||
- repo: https://github.com/myint/autoflake
|
||||
rev: v2.3.3
|
||||
hooks:
|
||||
- id: autoflake
|
||||
name: "SDK - autoflake"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variable"]
|
||||
priority: 20
|
||||
|
||||
@@ -87,7 +87,7 @@ repos:
|
||||
hooks:
|
||||
- id: isort
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
stages: ["pre-commit"]
|
||||
priority: 20
|
||||
@@ -97,7 +97,7 @@ repos:
|
||||
hooks:
|
||||
- id: black
|
||||
name: "SDK - black"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
@@ -105,7 +105,7 @@ repos:
|
||||
hooks:
|
||||
- id: flake8
|
||||
name: "SDK - flake8"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
priority: 30
|
||||
|
||||
@@ -142,6 +142,14 @@ repos:
|
||||
files: { glob: ["mcp_server/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- id: generate-provider-cards
|
||||
name: "Docs - regenerate provider cards snippet"
|
||||
entry: python docs/scripts/generate_provider_cards.py
|
||||
language: system
|
||||
files: { glob: ["docs/user-guide/providers/**/getting-started-*.mdx", "docs/scripts/generate_provider_cards.py", "docs/snippets/provider-cards.mdx", "api/src/backend/api/models.py"] }
|
||||
pass_filenames: false
|
||||
priority: 20
|
||||
|
||||
## PYTHON — uv (API + SDK)
|
||||
- repo: https://github.com/astral-sh/uv-pre-commit
|
||||
rev: 0.11.14
|
||||
@@ -183,7 +191,7 @@ repos:
|
||||
entry: pylint --disable=W,C,R,E -j 0 -rn -sn
|
||||
language: system
|
||||
types: [python]
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
priority: 30
|
||||
|
||||
- id: trufflehog
|
||||
|
||||
@@ -114,6 +114,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing Prowler UI components | `prowler-ui` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/prowler"
|
||||
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
|
||||
@@ -2,19 +2,55 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.33.0] (Prowler UNRELEASED)
|
||||
## [1.34.1] (Prowler UNRELEASED)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Partitioned `compliance_requirements_overviews` and hardened its COPY ingest (batched, single reused connection, first-run DELETE skipped) to stop the database writer from running out of memory during compliance overview ingestion [(#11870)](https://github.com/prowler-cloud/prowler/pull/11870)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.0] (Prowler v5.33.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Compliance PDF reports no longer require provider credentials: findings are enriched from the provider metadata stored in the database, so reports generate even after the provider secret is deleted or its credentials become invalid [(#11845)](https://github.com/prowler-cloud/prowler/pull/11845)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Provider scans now queue behind active provider scans instead of dispatching concurrently, and resource failed-finding counters retry database conflicts with stable row locking [(#11848)](https://github.com/prowler-cloud/prowler/pull/11848)
|
||||
|
||||
---
|
||||
|
||||
## [1.33.1] (Prowler v5.32.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Scan rows now have database defaults for `is_migrated` and `sink_backend` so `scan-perform-scheduled` inserts survive deploy skew [(#11826)](https://github.com/prowler-cloud/prowler/pull/11826)
|
||||
- Invited users now keep their invitation context when completing authentication with Google, GitHub, or SAML, so the invitation is accepted during login [(#11752)](https://github.com/prowler-cloud/prowler/pull/11752)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- User profile updates now allow users to update their own account while requiring user-management permissions to update other users in the same tenant [(#11792)](https://github.com/prowler-cloud/prowler/pull/11792)
|
||||
- Kubernetes provider credentials now reject kubeconfigs using `exec` authentication in Prowler Cloud, preventing user-supplied commands from running on Cloud workers [(#11753)](https://github.com/prowler-cloud/prowler/pull/11753)
|
||||
|
||||
---
|
||||
|
||||
## [1.33.0] (Prowler v5.32.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Timestamp precision support for `/api/v1/findings` `inserted_at` and `updated_at` filters [(#11754)](https://github.com/prowler-cloud/prowler/pull/11754)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Attack Paths: AWS Neptune is now supported as a persistent sink database, selectable via `ATTACK_PATHS_SINK_DATABASE=neptune` (default `neo4j`), Cartography's (bumped to 0.138.1) per-scan ingest database stays on Neo4j [(#11524)](https://github.com/prowler-cloud/prowler/pull/11524)
|
||||
- Attack Paths: Scan task now checks the ingest Neo4j database and configured graph sink before starting graph ingestion [(#11743)](https://github.com/prowler-cloud/prowler/pull/11743)
|
||||
|
||||
---
|
||||
|
||||
## [1.32.2] (Prowler UNRELEASED)
|
||||
- Disable PowerShell telemetry in the API container image [(#11746)](https://github.com/prowler-cloud/prowler/pull/11746)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Provider graph cleanup now deletes Neo4j and Neptune relationships in directed batches before deleting nodes [(#11755)](https://github.com/prowler-cloud/prowler/pull/11755)
|
||||
- `scan-perform` no longer reports an error when a provider is deleted during a running scan [(#11696)](https://github.com/prowler-cloud/prowler/pull/11696)
|
||||
|
||||
---
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
|
||||
ENV POWERSHELL_TELEMETRY_OPTOUT=1
|
||||
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.33.0"
|
||||
version = "1.34.0"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
|
||||
@@ -9,6 +9,7 @@ from api.models import (
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.utils import accept_invitation_for_user
|
||||
from django.db import transaction
|
||||
|
||||
|
||||
@@ -20,6 +21,22 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
except User.DoesNotExist:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_invitation_token(request):
|
||||
for source_name in ("data", "POST"):
|
||||
data = getattr(request, source_name, None) or {}
|
||||
if not hasattr(data, "get"):
|
||||
continue
|
||||
invitation_token = data.get("invitation_token")
|
||||
if invitation_token:
|
||||
return invitation_token
|
||||
|
||||
wrapped_request = getattr(request, "_request", None)
|
||||
if wrapped_request and wrapped_request is not request:
|
||||
return ProwlerSocialAccountAdapter._get_invitation_token(wrapped_request)
|
||||
|
||||
return None
|
||||
|
||||
def pre_social_login(self, request, sociallogin):
|
||||
# Link existing accounts with the same email address
|
||||
email = sociallogin.account.extra_data.get("email")
|
||||
@@ -83,29 +100,38 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
user.name = social_account_name
|
||||
user.save(using=MainRouter.admin_db)
|
||||
|
||||
tenant = Tenant.objects.using(MainRouter.admin_db).create(
|
||||
name=f"{user.email.split('@')[0]} default tenant"
|
||||
)
|
||||
with rls_transaction(str(tenant.id)):
|
||||
Membership.objects.using(MainRouter.admin_db).create(
|
||||
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
|
||||
)
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name="admin",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=True,
|
||||
manage_account=True,
|
||||
manage_billing=True,
|
||||
manage_providers=True,
|
||||
manage_integrations=True,
|
||||
manage_scans=True,
|
||||
unlimited_visibility=True,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
invitation_token = self._get_invitation_token(request)
|
||||
if invitation_token:
|
||||
invitation, _ = accept_invitation_for_user(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
invitation_token=invitation_token,
|
||||
)
|
||||
request.prowler_invitation_token = invitation_token
|
||||
request.prowler_invitation_tenant_id = str(invitation.tenant_id)
|
||||
else:
|
||||
tenant = Tenant.objects.using(MainRouter.admin_db).create(
|
||||
name=f"{user.email.split('@')[0]} default tenant"
|
||||
)
|
||||
with rls_transaction(str(tenant.id)):
|
||||
Membership.objects.using(MainRouter.admin_db).create(
|
||||
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
|
||||
)
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name="admin",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=True,
|
||||
manage_account=True,
|
||||
manage_billing=True,
|
||||
manage_providers=True,
|
||||
manage_integrations=True,
|
||||
manage_scans=True,
|
||||
unlimited_visibility=True,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
else:
|
||||
request.session["saml_user_created"] = str(user.id)
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Shared batched deletion helpers for sink backends."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES = {
|
||||
"outgoing relationship": """
|
||||
MATCH (n:`{provider_label}`)-[r]->()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
"incoming relationship": """
|
||||
MATCH (n:`{provider_label}`)<-[r]-()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
}
|
||||
|
||||
NODE_DELETE_QUERY_TEMPLATE = """
|
||||
MATCH (n:{provider_resource_label}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
"""
|
||||
|
||||
|
||||
def delete_batches(
|
||||
*,
|
||||
session: Any,
|
||||
logger: logging.Logger,
|
||||
log_target: str,
|
||||
provider_id: str,
|
||||
query: str,
|
||||
phase: str,
|
||||
count_key: str,
|
||||
total_key: str,
|
||||
deleted_key: str,
|
||||
initial_total: int,
|
||||
batch_size: int,
|
||||
drop_t0: float,
|
||||
) -> tuple[int, int]:
|
||||
deleted_total = initial_total
|
||||
batches = 0
|
||||
while True:
|
||||
logger.info(
|
||||
"Deleting %s batch from %s "
|
||||
"(provider=%s, batch=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches + 1,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
record = session.run(query, {"batch_size": batch_size}).single()
|
||||
deleted = (record[count_key] if record else 0) or 0
|
||||
if deleted == 0:
|
||||
return deleted_total, batches
|
||||
|
||||
batches += 1
|
||||
deleted_total += deleted
|
||||
logger.info(
|
||||
"Deleted %s batch from %s "
|
||||
"(provider=%s, batch=%s, %s=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches,
|
||||
deleted_key,
|
||||
deleted,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
@@ -17,6 +17,11 @@ import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
@@ -204,10 +209,8 @@ class Neo4jSink(SinkDatabase):
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_nodes = 0
|
||||
deleted_relationships = 0
|
||||
relationship_batches = 0
|
||||
node_batches = 0
|
||||
deleted_nodes = deleted_relationships = 0
|
||||
relationship_batches = node_batches = 0
|
||||
drop_t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
@@ -232,84 +235,44 @@ class Neo4jSink(SinkDatabase):
|
||||
database,
|
||||
provider_id,
|
||||
)
|
||||
# Phase 1: delete relationships incident to provider nodes in
|
||||
# batches. The undirected pattern matches an edge between two
|
||||
# provider nodes from both ends, so `DISTINCT r` dedupes it to
|
||||
# delete a full batch of unique relationships each round.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
next_batch = relationship_batches + 1
|
||||
logger.info(
|
||||
"Deleting relationship batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, total_rels=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
log_target = f"Neo4j sink database {database}"
|
||||
for (
|
||||
phase,
|
||||
query_template,
|
||||
) in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (:`{provider_label}`)-[r]-()
|
||||
WITH DISTINCT r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_rels_count", 0)
|
||||
if deleted_count > 0:
|
||||
relationship_batches += 1
|
||||
deleted_relationships += deleted_count
|
||||
logger.info(
|
||||
"Deleted relationship batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, deleted_rels=%s, "
|
||||
"total_rels=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_count,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
# Phase 2: delete the now relationship-free nodes in batches.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
next_batch = node_batches + 1
|
||||
logger.info(
|
||||
"Deleting node batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, total_nodes=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_nodes_count", 0)
|
||||
if deleted_count > 0:
|
||||
node_batches += 1
|
||||
deleted_nodes += deleted_count
|
||||
logger.info(
|
||||
"Deleted node batch from Neo4j sink database %s "
|
||||
"(provider=%s, batch=%s, deleted_nodes=%s, "
|
||||
"total_nodes=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
node_batches,
|
||||
deleted_count,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == DATABASE_NOT_FOUND_CODE:
|
||||
|
||||
@@ -27,6 +27,11 @@ import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.session import Session as BotoSession
|
||||
@@ -296,78 +301,40 @@ class NeptuneSink(SinkDatabase):
|
||||
"Opened Neptune writer session for provider graph drop (provider=%s)",
|
||||
provider_id,
|
||||
)
|
||||
while True:
|
||||
next_batch = relationship_batches + 1
|
||||
logger.info(
|
||||
"Deleting relationship batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, total_rels=%s, elapsed=%.3fs)",
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (:`{provider_label}`)-[r]-()
|
||||
WITH DISTINCT r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
record = result.single()
|
||||
deleted_rels = (record["deleted_rels_count"] if record else 0) or 0
|
||||
if deleted_rels == 0:
|
||||
break
|
||||
relationship_batches += 1
|
||||
deleted_relationships += deleted_rels
|
||||
logger.info(
|
||||
"Deleted relationship batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, deleted_rels=%s, total_rels=%s, "
|
||||
"elapsed=%.3fs)",
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_rels,
|
||||
deleted_relationships,
|
||||
time.perf_counter() - drop_t0,
|
||||
for phase, query_template in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
deleted_nodes = 0
|
||||
while True:
|
||||
next_batch = node_batches + 1
|
||||
logger.info(
|
||||
"Deleting node batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, total_nodes=%s, elapsed=%.3fs)",
|
||||
provider_id,
|
||||
next_batch,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
record = result.single()
|
||||
deleted = (record["deleted_nodes_count"] if record else 0) or 0
|
||||
if deleted == 0:
|
||||
break
|
||||
node_batches += 1
|
||||
deleted_nodes += deleted
|
||||
logger.info(
|
||||
"Deleted node batch from Neptune sink "
|
||||
"(provider=%s, batch=%s, deleted_nodes=%s, total_nodes=%s, "
|
||||
"elapsed=%.3fs)",
|
||||
provider_id,
|
||||
node_batches,
|
||||
deleted,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Finished dropping provider graph from Neptune sink "
|
||||
|
||||
@@ -67,6 +67,7 @@ from django_filters.rest_framework import (
|
||||
)
|
||||
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
from uuid6 import UUID
|
||||
|
||||
|
||||
class CustomDjangoFilterBackend(DjangoFilterBackend):
|
||||
@@ -672,35 +673,32 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
return queryset.filter(tags__text_search=value)
|
||||
|
||||
|
||||
class FindingFilter(CommonFindingFilters):
|
||||
FINDING_BASE_FILTER_FIELDS = {
|
||||
"id": ["exact", "in"],
|
||||
"uid": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
"delta": ["exact", "in"],
|
||||
"status": ["exact", "in"],
|
||||
"severity": ["exact", "in"],
|
||||
"impact": ["exact", "in"],
|
||||
"check_id": ["exact", "in", "icontains"],
|
||||
}
|
||||
|
||||
|
||||
class BaseFindingFilter(CommonFindingFilters):
|
||||
DATE_FILTER_FIELDS = ()
|
||||
DATE_FILTER_NAMES = ()
|
||||
DATE_RANGE_HELP_TEXT = (
|
||||
f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days."
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = "At least one date filter is required."
|
||||
|
||||
scan = UUIDFilter(method="filter_scan_id")
|
||||
scan__in = UUIDInFilter(method="filter_scan_id_in")
|
||||
|
||||
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(
|
||||
method="filter_inserted_at_gte",
|
||||
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
)
|
||||
inserted_at__lte = DateFilter(
|
||||
method="filter_inserted_at_lte",
|
||||
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Finding
|
||||
fields = {
|
||||
"id": ["exact", "in"],
|
||||
"uid": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
"delta": ["exact", "in"],
|
||||
"status": ["exact", "in"],
|
||||
"severity": ["exact", "in"],
|
||||
"impact": ["exact", "in"],
|
||||
"check_id": ["exact", "in", "icontains"],
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"updated_at": ["gte", "lte"],
|
||||
}
|
||||
fields = FINDING_BASE_FILTER_FIELDS
|
||||
filter_overrides = {
|
||||
FindingDeltaEnumField: {
|
||||
"filter_class": CharFilter,
|
||||
@@ -723,17 +721,13 @@ class FindingFilter(CommonFindingFilters):
|
||||
return queryset.filter(resource_services__contains=[value])
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
|
||||
self.data.get("inserted_at")
|
||||
or self.data.get("inserted_at__date")
|
||||
or self.data.get("inserted_at__gte")
|
||||
or self.data.get("inserted_at__lte")
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not any(
|
||||
self.data.get(filter_name) for filter_name in self.DATE_FILTER_NAMES
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
|
||||
"or filter[inserted_at.lte].",
|
||||
"detail": self.DATE_FILTER_REQUIRED_DETAIL,
|
||||
"status": 400,
|
||||
"source": {"pointer": "/data/attributes/inserted_at"},
|
||||
"code": "required",
|
||||
@@ -742,31 +736,42 @@ class FindingFilter(CommonFindingFilters):
|
||||
)
|
||||
|
||||
cleaned = self.form.cleaned_data
|
||||
exact_date = cleaned.get("inserted_at") or cleaned.get("inserted_at__date")
|
||||
gte_date = cleaned.get("inserted_at__gte") or exact_date
|
||||
lte_date = cleaned.get("inserted_at__lte") or exact_date
|
||||
|
||||
if gte_date is None:
|
||||
gte_date = datetime.now(UTC).date()
|
||||
if lte_date is None:
|
||||
lte_date = datetime.now(UTC).date()
|
||||
|
||||
if abs(lte_date - gte_date) > timedelta(
|
||||
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
"status": 400,
|
||||
"source": {"pointer": "/data/attributes/inserted_at"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
for field_name in self.DATE_FILTER_FIELDS:
|
||||
self.validate_datetime_filter_range(cleaned, field_name)
|
||||
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
def validate_datetime_filter_range(self, cleaned, field_name):
|
||||
exact_value = cleaned.get(field_name) or cleaned.get(f"{field_name}__date")
|
||||
gte_value = cleaned.get(f"{field_name}__gte") or exact_value
|
||||
lte_value = cleaned.get(f"{field_name}__lte") or exact_value
|
||||
|
||||
if not (exact_value or gte_value or lte_value):
|
||||
return
|
||||
|
||||
default_value = datetime.now(UTC).date()
|
||||
gte_value = gte_value or default_value
|
||||
lte_value = lte_value or default_value
|
||||
|
||||
gte_datetime = self.filter_value_to_datetime(gte_value, field_name)
|
||||
lte_datetime = self.filter_value_to_datetime(lte_value, field_name)
|
||||
|
||||
if abs(lte_datetime - gte_datetime) <= timedelta(
|
||||
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
|
||||
):
|
||||
return
|
||||
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
"status": 400,
|
||||
"source": {"pointer": f"/data/attributes/{field_name}"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Convert filter values to UUIDv7 values for use with partitioning
|
||||
def filter_scan_id(self, queryset, name, value):
|
||||
try:
|
||||
@@ -824,27 +829,169 @@ class FindingFilter(CommonFindingFilters):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
start = uuid7_start(datetime_to_uuid7(datetime_value))
|
||||
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
|
||||
|
||||
return queryset.filter(id__gte=start, id__lt=end)
|
||||
|
||||
def filter_inserted_at_gte(self, queryset, name, value):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
start = uuid7_start(datetime_to_uuid7(datetime_value))
|
||||
|
||||
return queryset.filter(id__gte=start)
|
||||
|
||||
def filter_inserted_at_lte(self, queryset, name, value):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
|
||||
|
||||
return queryset.filter(id__lt=end)
|
||||
|
||||
@staticmethod
|
||||
def maybe_date_to_datetime(value):
|
||||
dt = value
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, date):
|
||||
dt = datetime.combine(value, datetime.min.time(), tzinfo=UTC)
|
||||
return dt
|
||||
return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
|
||||
if isinstance(value, str):
|
||||
return parse(value)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def filter_value_to_datetime(cls, value, field_name):
|
||||
try:
|
||||
datetime_value = cls.maybe_date_to_datetime(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "Enter a valid date or datetime.",
|
||||
"status": 400,
|
||||
"source": {"pointer": f"/data/attributes/{field_name}"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
if datetime_value.tzinfo is None:
|
||||
return datetime_value.replace(tzinfo=UTC)
|
||||
return datetime_value.astimezone(UTC)
|
||||
|
||||
|
||||
class FindingFilter(BaseFindingFilter):
|
||||
DATE_FILTER_FIELDS = ("inserted_at", "updated_at")
|
||||
DATE_FILTER_NAMES = (
|
||||
"inserted_at",
|
||||
"inserted_at__date",
|
||||
"inserted_at__gte",
|
||||
"inserted_at__lte",
|
||||
"updated_at",
|
||||
"updated_at__date",
|
||||
"updated_at__gte",
|
||||
"updated_at__lte",
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = (
|
||||
"At least one date filter is required: filter[inserted_at], filter[updated_at], "
|
||||
"filter[inserted_at.gte], filter[updated_at.gte], filter[inserted_at.lte], "
|
||||
"or filter[updated_at.lte]."
|
||||
)
|
||||
|
||||
inserted_at = CharFilter(method="filter_inserted_at")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = CharFilter(
|
||||
method="filter_inserted_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
inserted_at__lte = CharFilter(
|
||||
method="filter_inserted_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
updated_at = CharFilter(method="filter_updated_at")
|
||||
updated_at__date = DateFilter(method="filter_updated_at", lookup_expr="date")
|
||||
updated_at__gte = CharFilter(
|
||||
method="filter_updated_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
updated_at__lte = CharFilter(
|
||||
method="filter_updated_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
|
||||
class Meta(BaseFindingFilter.Meta):
|
||||
fields = FINDING_BASE_FILTER_FIELDS | {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"updated_at": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
def filter_inserted_at(self, queryset, name, value):
|
||||
start, end = self.filter_value_to_datetime_bounds(value, "inserted_at")
|
||||
|
||||
if name.endswith("__gte"):
|
||||
return queryset.filter(id__gte=self.datetime_to_uuid7_boundary(start))
|
||||
if name.endswith("__lte"):
|
||||
return queryset.filter(id__lt=self.datetime_to_uuid7_boundary(end))
|
||||
|
||||
return queryset.filter(
|
||||
id__gte=self.datetime_to_uuid7_boundary(start),
|
||||
id__lt=self.datetime_to_uuid7_boundary(end),
|
||||
)
|
||||
|
||||
def filter_updated_at(self, queryset, name, value):
|
||||
start, end = self.filter_value_to_datetime_bounds(value, "updated_at")
|
||||
|
||||
if name.endswith("__gte"):
|
||||
return queryset.filter(updated_at__gte=start)
|
||||
if name.endswith("__lte"):
|
||||
return queryset.filter(updated_at__lt=end)
|
||||
|
||||
return queryset.filter(updated_at__gte=start, updated_at__lt=end)
|
||||
|
||||
@classmethod
|
||||
def filter_value_to_datetime_bounds(cls, value, field_name):
|
||||
start = cls.filter_value_to_datetime(value, field_name)
|
||||
if cls.is_date_filter_value(value):
|
||||
return start, start + timedelta(days=1)
|
||||
return start, start + timedelta(milliseconds=1)
|
||||
|
||||
@staticmethod
|
||||
def datetime_to_uuid7_boundary(datetime_value):
|
||||
timestamp_ms = int(datetime_value.timestamp() * 1000) & 0xFFFFFFFFFFFF
|
||||
uuid_int = timestamp_ms << 80
|
||||
uuid_int |= 0x7 << 76
|
||||
uuid_int |= 0x2 << 62
|
||||
return UUID(int=uuid_int)
|
||||
|
||||
@staticmethod
|
||||
def is_date_filter_value(value):
|
||||
if isinstance(value, datetime):
|
||||
return False
|
||||
if isinstance(value, date):
|
||||
return True
|
||||
return isinstance(value, str) and len(value.strip()) == 10
|
||||
|
||||
|
||||
class FindingMetadataFilter(BaseFindingFilter):
|
||||
DATE_FILTER_FIELDS = ("inserted_at",)
|
||||
DATE_FILTER_NAMES = (
|
||||
"inserted_at",
|
||||
"inserted_at__date",
|
||||
"inserted_at__gte",
|
||||
"inserted_at__lte",
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = (
|
||||
"At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
|
||||
"or filter[inserted_at.lte]."
|
||||
)
|
||||
|
||||
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(
|
||||
method="filter_inserted_at_gte",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
inserted_at__lte = DateFilter(
|
||||
method="filter_inserted_at_lte",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
|
||||
class Meta(BaseFindingFilter.Meta):
|
||||
fields = FINDING_BASE_FILTER_FIELDS | {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
|
||||
class LatestFindingFilter(CommonFindingFilters):
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0096_attack_paths_scan_is_migrated"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="attackpathsscan",
|
||||
name="is_migrated",
|
||||
field=models.BooleanField(db_default=False, default=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="attackpathsscan",
|
||||
name="sink_backend",
|
||||
field=models.CharField(
|
||||
choices=[("neo4j", "Neo4j"), ("neptune", "Neptune")],
|
||||
db_default="neo4j",
|
||||
default="neo4j",
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Convert ``compliance_requirements_overviews`` into a RANGE-partitioned table.
|
||||
|
||||
The compliance-overview ingest rewrites every scan's rows (delete-then-reinsert),
|
||||
which — together with old scans ageing out — generates heavy autovacuum churn on a
|
||||
plain table. Partitioning RANGE by the UUIDv7 ``id`` (same strategy as ``findings``)
|
||||
lets old data be reclaimed with ``DROP PARTITION`` instead of ``DELETE`` + CASCADE.
|
||||
|
||||
Existing data is preserved WITHOUT a copy: the current table is renamed and
|
||||
re-attached as the ``default`` partition of a new partitioned parent (ATTACH is a
|
||||
metadata operation, so there is no row rewrite). Existing rows keep their uuid4
|
||||
ids and live in ``default``; new rows carry uuid7 ids and route to the monthly
|
||||
partitions created by ``pgpartition``.
|
||||
|
||||
NOTE for production: rename + ATTACH take a brief ACCESS EXCLUSIVE lock and adding
|
||||
the parent FKs validates the default partition. Run during a low-traffic window (or
|
||||
``--fake`` + apply manually) on large tables, and rehearse against a prod snapshot —
|
||||
this is delicate DDL with no in-repo precedent. See skills/django-migration-psql.
|
||||
"""
|
||||
|
||||
import api.rls
|
||||
import psqlextra.manager.manager
|
||||
import uuid6
|
||||
from api.rls import RowLevelSecurityConstraint
|
||||
from django.db import migrations, models
|
||||
|
||||
TABLE = "compliance_requirements_overviews"
|
||||
DEFAULT = f"{TABLE}_default"
|
||||
INDEX = "cro_scan_comp_reg_idx"
|
||||
UNIQUE = "unique_tenant_compliance_requirement_overview"
|
||||
PARENT_RLS = "rls_on_compliancerequirementoverview"
|
||||
DEFAULT_RLS = "rls_on_compliancerequirementoverview_default"
|
||||
|
||||
|
||||
def _partition_table(apps, schema_editor):
|
||||
model = apps.get_model("api", "ComplianceRequirementOverview")
|
||||
cursor = schema_editor.connection.cursor
|
||||
|
||||
# 1. Drop the old RLS (created in 0027 with SELECT/INSERT/UPDATE/DELETE) via the
|
||||
# constraint object so the DB_USER-scoped policy/grant names resolve correctly.
|
||||
schema_editor.remove_constraint(
|
||||
model,
|
||||
RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name=PARENT_RLS,
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
)
|
||||
|
||||
with cursor() as cur:
|
||||
# 2. Drop the old business unique (5 cols) — after partitioning, dedup is
|
||||
# handled by the delete-then-reinsert ingest (the model keeps no unique).
|
||||
cur.execute(f"ALTER TABLE {TABLE} DROP CONSTRAINT {UNIQUE};")
|
||||
|
||||
# 3. Rename the existing table and every object whose name the new parent
|
||||
# will reuse, moving them under the ``_default`` name. The default keeps
|
||||
# its own FKs, so the parent FKs added below skip re-validation on ATTACH.
|
||||
cur.execute(f"ALTER TABLE {TABLE} RENAME TO {DEFAULT};")
|
||||
cur.execute(
|
||||
f"ALTER TABLE {DEFAULT} RENAME CONSTRAINT {TABLE}_pkey TO {DEFAULT}_pkey;"
|
||||
)
|
||||
cur.execute(f"ALTER INDEX {INDEX} RENAME TO {INDEX}_default;")
|
||||
|
||||
# 4. Create the partitioned parent and re-establish its structure.
|
||||
cur.execute(
|
||||
f"CREATE TABLE {TABLE} (LIKE {DEFAULT} INCLUDING DEFAULTS) "
|
||||
"PARTITION BY RANGE (id);"
|
||||
)
|
||||
cur.execute(
|
||||
f"ALTER TABLE {TABLE} ADD CONSTRAINT {TABLE}_pkey PRIMARY KEY (id);"
|
||||
)
|
||||
# Parent index ON ONLY (no data), then attach the pre-built default index so
|
||||
# the existing rows are NOT re-indexed.
|
||||
cur.execute(
|
||||
f"CREATE INDEX {INDEX} ON ONLY {TABLE} "
|
||||
"(tenant_id, scan_id, compliance_id, region);"
|
||||
)
|
||||
|
||||
# 5. Add FKs on the STILL-EMPTY parent (instant, no scan) so new partitions
|
||||
# inherit referential integrity. Postgres cannot add NOT VALID FKs to a
|
||||
# partitioned table, but because the default partition already carries an
|
||||
# equivalent valid FK, the ATTACH below skips FK re-validation. Cascade on
|
||||
# scan/tenant deletion is still handled at the ORM level.
|
||||
cur.execute(
|
||||
f"ALTER TABLE {TABLE} ADD CONSTRAINT {TABLE}_scan_id_fk "
|
||||
"FOREIGN KEY (scan_id) REFERENCES scans (id) DEFERRABLE INITIALLY DEFERRED;"
|
||||
)
|
||||
cur.execute(
|
||||
f"ALTER TABLE {TABLE} ADD CONSTRAINT {TABLE}_tenant_id_fk "
|
||||
"FOREIGN KEY (tenant_id) REFERENCES tenants (id) DEFERRABLE INITIALLY DEFERRED;"
|
||||
)
|
||||
|
||||
# 6. Attach the existing table (with all its rows) as the DEFAULT partition
|
||||
# and attach its pre-built index to the parent's partitioned index.
|
||||
cur.execute(f"ALTER TABLE {TABLE} ATTACH PARTITION {DEFAULT} DEFAULT;")
|
||||
cur.execute(f"ALTER INDEX {INDEX} ATTACH PARTITION {INDEX}_default;")
|
||||
|
||||
# 7. Re-establish RLS on the parent and the default partition (DB_USER-aware).
|
||||
schema_editor.add_constraint(
|
||||
model,
|
||||
RowLevelSecurityConstraint(
|
||||
"tenant_id", name=PARENT_RLS, statements=["SELECT", "INSERT", "DELETE"]
|
||||
),
|
||||
)
|
||||
schema_editor.add_constraint(
|
||||
model,
|
||||
RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name=DEFAULT_RLS,
|
||||
partition_name="default",
|
||||
statements=["SELECT", "INSERT", "DELETE"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _unpartition_table(apps, schema_editor):
|
||||
"""Best-effort reverse: detach the default partition and restore a plain table."""
|
||||
model = apps.get_model("api", "ComplianceRequirementOverview")
|
||||
cursor = schema_editor.connection.cursor
|
||||
|
||||
schema_editor.remove_constraint(
|
||||
model,
|
||||
RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name=DEFAULT_RLS,
|
||||
partition_name="default",
|
||||
statements=["SELECT", "INSERT", "DELETE"],
|
||||
),
|
||||
)
|
||||
schema_editor.remove_constraint(
|
||||
model,
|
||||
RowLevelSecurityConstraint(
|
||||
"tenant_id", name=PARENT_RLS, statements=["SELECT", "INSERT", "DELETE"]
|
||||
),
|
||||
)
|
||||
|
||||
with cursor() as cur:
|
||||
# Detach the default partition (keeps its own pkey/index/FKs) and drop the
|
||||
# partitioned parent, then rename the default back to the original table.
|
||||
cur.execute(f"ALTER INDEX {INDEX} DETACH PARTITION {INDEX}_default;")
|
||||
cur.execute(f"ALTER TABLE {TABLE} DETACH PARTITION {DEFAULT};")
|
||||
cur.execute(f"DROP TABLE {TABLE};")
|
||||
cur.execute(f"ALTER TABLE {DEFAULT} RENAME TO {TABLE};")
|
||||
cur.execute(
|
||||
f"ALTER TABLE {TABLE} RENAME CONSTRAINT {DEFAULT}_pkey TO {TABLE}_pkey;"
|
||||
)
|
||||
cur.execute(f"ALTER INDEX {INDEX}_default RENAME TO {INDEX};")
|
||||
# Restore the original 5-column business unique (the FKs survived on the
|
||||
# detached table, so they are not re-created here).
|
||||
cur.execute(
|
||||
f"ALTER TABLE {TABLE} ADD CONSTRAINT {UNIQUE} "
|
||||
"UNIQUE (tenant_id, scan_id, compliance_id, requirement_id, region);"
|
||||
)
|
||||
|
||||
schema_editor.add_constraint(
|
||||
model,
|
||||
RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name=PARENT_RLS,
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0097_attack_paths_scan_db_defaults"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.SeparateDatabaseAndState(
|
||||
# State: mirror what pgmakemigrations detected (minus unrelated stowaways)
|
||||
# so the ORM state matches the partitioned model definition.
|
||||
state_operations=[
|
||||
migrations.AlterModelOptions(
|
||||
name="compliancerequirementoverview",
|
||||
options={"base_manager_name": "objects"},
|
||||
),
|
||||
migrations.AlterModelManagers(
|
||||
name="compliancerequirementoverview",
|
||||
managers=[
|
||||
("objects", psqlextra.manager.manager.PostgresManager()),
|
||||
],
|
||||
),
|
||||
migrations.RemoveConstraint(
|
||||
model_name="compliancerequirementoverview",
|
||||
name=UNIQUE,
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="compliancerequirementoverview",
|
||||
name="id",
|
||||
field=models.UUIDField(
|
||||
default=uuid6.uuid7,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="compliancerequirementoverview",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id", name=DEFAULT_RLS
|
||||
),
|
||||
),
|
||||
],
|
||||
database_operations=[
|
||||
migrations.RunPython(_partition_table, _unpartition_table),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -814,9 +814,10 @@ class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
# still using the previous graph shape. Query catalog selection uses this
|
||||
# flag; physical read routing uses sink_backend below.
|
||||
# TODO: drop after Neptune cutover
|
||||
is_migrated = models.BooleanField(default=False)
|
||||
is_migrated = models.BooleanField(default=False, db_default=False)
|
||||
sink_backend = models.CharField(
|
||||
choices=SinkBackendChoices.choices,
|
||||
db_default=SinkBackendChoices.NEO4J,
|
||||
default=SinkBackendChoices.NEO4J,
|
||||
max_length=16,
|
||||
)
|
||||
@@ -1586,8 +1587,25 @@ class ComplianceOverview(RowLevelSecurityProtectedModel):
|
||||
resource_name = "compliance-overviews"
|
||||
|
||||
|
||||
class ComplianceRequirementOverview(RowLevelSecurityProtectedModel):
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
class ComplianceRequirementOverview(
|
||||
PostgresPartitionedModel, RowLevelSecurityProtectedModel
|
||||
):
|
||||
"""
|
||||
Per-requirement compliance rows, materialized once per scan via COPY.
|
||||
|
||||
Partitioned RANGE by the UUIDv7 ``id`` (same strategy as ``Finding``) so that
|
||||
ageing scans' compliance rows can be reclaimed with ``DROP PARTITION`` instead
|
||||
of ``DELETE`` + CASCADE, removing the autovacuum churn that a per-scan delete
|
||||
otherwise generates.
|
||||
|
||||
Note when creating migrations you must use ``python manage.py pgmakemigrations``.
|
||||
"""
|
||||
|
||||
class PartitioningMeta:
|
||||
method = PostgresPartitioningMethod.RANGE
|
||||
key = ["id"]
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
compliance_id = models.TextField(blank=False)
|
||||
framework = models.TextField(blank=False)
|
||||
@@ -1612,23 +1630,25 @@ class ComplianceRequirementOverview(RowLevelSecurityProtectedModel):
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "compliance_requirements_overviews"
|
||||
base_manager_name = "objects"
|
||||
|
||||
# No UniqueConstraint on (tenant, scan, compliance, requirement, region):
|
||||
# Postgres would force the partition key ``id`` into it, making it trivial
|
||||
# (``id`` is the PK). Business-level dedup is guaranteed by the
|
||||
# delete-then-reinsert ingest in ``create_compliance_requirements`` instead
|
||||
# (same approach as the partitioned ``Finding`` model, which has none).
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=(
|
||||
"tenant_id",
|
||||
"scan_id",
|
||||
"compliance_id",
|
||||
"requirement_id",
|
||||
"region",
|
||||
),
|
||||
name="unique_tenant_compliance_requirement_overview",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "DELETE"],
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s_default",
|
||||
partition_name="default",
|
||||
statements=["SELECT", "INSERT", "DELETE"],
|
||||
),
|
||||
]
|
||||
indexes = [
|
||||
models.Index(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from collections.abc import Generator
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from api.models import Finding, ResourceFindingMapping
|
||||
from api.models import ComplianceRequirementOverview, Finding, ResourceFindingMapping
|
||||
from api.rls import RowLevelSecurityConstraint
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from dateutil.relativedelta import relativedelta
|
||||
@@ -196,5 +196,22 @@ manager = PostgresPartitioningManager(
|
||||
rls_statements=["SELECT"],
|
||||
),
|
||||
),
|
||||
# ComplianceRequirementOverview: partition by its own UUIDv7 id so old
|
||||
# scans' compliance rows age out via DROP PARTITION instead of DELETE.
|
||||
PostgresPartitioningConfig(
|
||||
model=ComplianceRequirementOverview,
|
||||
strategy=PostgresUUIDv7PartitioningStrategy(
|
||||
start_date=datetime.now(UTC),
|
||||
size=PostgresTimePartitionSize(
|
||||
months=settings.COMPLIANCE_REQ_OVERVIEW_PARTITION_MONTHS
|
||||
),
|
||||
count=settings.COMPLIANCE_REQ_OVERVIEW_PARTITION_COUNT,
|
||||
max_age=relative_days_or_none(
|
||||
settings.COMPLIANCE_REQ_OVERVIEW_PARTITION_MAX_AGE_MONTHS
|
||||
),
|
||||
name_format="%Y_%b",
|
||||
rls_statements=["SELECT", "INSERT", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
from allauth.socialaccount.models import SocialLogin
|
||||
from api.adapters import ProwlerSocialAccountAdapter
|
||||
from api.db_router import MainRouter
|
||||
from api.models import SAMLConfiguration
|
||||
from api.models import Invitation, Membership, SAMLConfiguration, Tenant
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
User = get_user_model()
|
||||
@@ -188,6 +188,44 @@ class TestProwlerSocialAccountAdapter:
|
||||
_, called_user = call_args[0]
|
||||
assert called_user.email == create_test_user.email
|
||||
|
||||
def test_save_user_social_with_invitation_joins_invited_tenant(
|
||||
self, rf, create_test_user, tenants_fixture
|
||||
):
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
invited_tenant = tenants_fixture[2]
|
||||
invited_email = "frank-invited@example.com"
|
||||
invitation = Invitation.objects.create(
|
||||
tenant=invited_tenant,
|
||||
email=invited_email,
|
||||
inviter=create_test_user,
|
||||
)
|
||||
request = rf.post("/", data={"invitation_token": invitation.token})
|
||||
request.session = {}
|
||||
|
||||
sociallogin = MagicMock(spec=SocialLogin)
|
||||
sociallogin.provider = MagicMock()
|
||||
sociallogin.provider.id = "google"
|
||||
sociallogin.account = MagicMock()
|
||||
sociallogin.account.extra_data = {"name": "Frank"}
|
||||
|
||||
real_user = User.objects.create_user(
|
||||
name="Frank", email=invited_email, password="Secret123!"
|
||||
)
|
||||
tenants_before = Tenant.objects.count()
|
||||
|
||||
with patch("api.adapters.super") as mock_super:
|
||||
mock_super.return_value.save_user.return_value = real_user
|
||||
adapter.save_user(request, sociallogin)
|
||||
|
||||
invitation.refresh_from_db()
|
||||
assert invitation.state == Invitation.State.ACCEPTED
|
||||
assert Tenant.objects.count() == tenants_before
|
||||
assert Membership.objects.filter(
|
||||
user=real_user,
|
||||
tenant=invited_tenant,
|
||||
role=Membership.RoleChoices.MEMBER,
|
||||
).exists()
|
||||
|
||||
def test_save_user_saml_sets_session_flag(self, rf):
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
request = rf.get("/")
|
||||
|
||||
@@ -103,20 +103,84 @@ class TestUserViewSet:
|
||||
assert response.json()["data"]["attributes"]["name"] == "Updated Name"
|
||||
|
||||
def test_partial_update_user_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, create_test_user
|
||||
self, authenticated_client_no_permissions_rbac, create_test_user_rbac_limited
|
||||
):
|
||||
updated_data = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user_rbac_limited.id),
|
||||
"attributes": {"name": "Updated Name"},
|
||||
}
|
||||
}
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
reverse("user-detail", kwargs={"pk": create_test_user_rbac_limited.id}),
|
||||
data=updated_data,
|
||||
format="vnd.api+json",
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["data"]["attributes"]["name"] == "Updated Name"
|
||||
|
||||
def test_partial_update_other_user_with_no_permissions_denied(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
original_email = "target-rbac-update@example.com"
|
||||
original_password = "OriginalPassword123@"
|
||||
target_user = User.objects.create_user(
|
||||
name="target_rbac_update",
|
||||
email=original_email,
|
||||
password=original_password,
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenants_fixture[0])
|
||||
updated_data = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {
|
||||
"email": "updated-target-rbac@example.com",
|
||||
"password": "UpdatedPassword123@",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=updated_data,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.email == original_email
|
||||
assert target_user.check_password(original_password)
|
||||
|
||||
def test_partial_update_other_user_with_manage_users_allowed(
|
||||
self, authenticated_client_rbac_manage_users_only
|
||||
):
|
||||
user = authenticated_client_rbac_manage_users_only.user
|
||||
tenant = Membership.objects.filter(user=user).first().tenant
|
||||
target_user = User.objects.create_user(
|
||||
name="target_manage_users_update",
|
||||
email="target-manage-users-update@example.com",
|
||||
password="Password123@",
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenant)
|
||||
updated_data = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {"name": "Updated Target Name"},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client_rbac_manage_users_only.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=updated_data,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.name == "Updated Target Name"
|
||||
|
||||
def test_delete_user_with_all_permissions(
|
||||
self, authenticated_client_rbac, create_test_user_rbac
|
||||
@@ -540,9 +604,7 @@ class TestLimitedVisibility:
|
||||
TEST_PASSWORD = "Thisisapassword123@"
|
||||
|
||||
@pytest.fixture
|
||||
def limited_admin_user(
|
||||
self, django_db_setup, django_db_blocker, tenants_fixture, providers_fixture
|
||||
):
|
||||
def limited_admin_user(self, django_db_blocker, tenants_fixture, providers_fixture):
|
||||
with django_db_blocker.unblock():
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
@@ -626,10 +688,10 @@ class TestLimitedVisibility:
|
||||
response.json()["data"]["relationships"]["providers"]["meta"]["count"] == 1
|
||||
)
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overviews_providers(
|
||||
self,
|
||||
authenticated_client_rbac_limited,
|
||||
scan_summaries_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
@@ -648,6 +710,7 @@ class TestLimitedVisibility:
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == 0
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name",
|
||||
[
|
||||
@@ -659,7 +722,6 @@ class TestLimitedVisibility:
|
||||
self,
|
||||
endpoint_name,
|
||||
authenticated_client_rbac_limited,
|
||||
scan_summaries_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
@@ -684,10 +746,10 @@ class TestLimitedVisibility:
|
||||
data = response.json()["data"]["attributes"].values()
|
||||
assert all(value == 0 for value in data)
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overviews_services(
|
||||
self,
|
||||
authenticated_client_rbac_limited,
|
||||
scan_summaries_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
from api.v1.serializer_utils.integrations import S3ConfigSerializer
|
||||
from api.v1.serializers import ImageProviderSecret
|
||||
from api.v1.serializers import ImageProviderSecret, KubernetesProviderSecret
|
||||
from rest_framework.exceptions import ValidationError
|
||||
|
||||
|
||||
@@ -132,3 +132,74 @@ class TestImageProviderSecret:
|
||||
serializer = ImageProviderSecret(data={"registry_password": "pass"})
|
||||
assert not serializer.is_valid()
|
||||
assert "non_field_errors" in serializer.errors
|
||||
|
||||
|
||||
class TestKubernetesProviderSecret:
|
||||
def test_valid_static_kubeconfig_is_accepted(self):
|
||||
kubeconfig_content = """
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test-cluster
|
||||
cluster:
|
||||
server: https://kubernetes.example.test
|
||||
users:
|
||||
- name: test-user
|
||||
user:
|
||||
token: test-token
|
||||
contexts:
|
||||
- name: test-context
|
||||
context:
|
||||
cluster: test-cluster
|
||||
user: test-user
|
||||
current-context: test-context
|
||||
"""
|
||||
|
||||
serializer = KubernetesProviderSecret(
|
||||
data={"kubeconfig_content": kubeconfig_content}
|
||||
)
|
||||
|
||||
assert serializer.is_valid()
|
||||
|
||||
def test_kubeconfig_with_exec_authentication_is_rejected(self):
|
||||
kubeconfig_content = """
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test-cluster
|
||||
cluster:
|
||||
server: https://kubernetes.example.test
|
||||
users:
|
||||
- name: test-user
|
||||
user:
|
||||
exec:
|
||||
apiVersion: client.authentication.k8s.io/v1
|
||||
command: kubectl
|
||||
contexts:
|
||||
- name: test-context
|
||||
context:
|
||||
cluster: test-cluster
|
||||
user: test-user
|
||||
current-context: test-context
|
||||
"""
|
||||
|
||||
serializer = KubernetesProviderSecret(
|
||||
data={"kubeconfig_content": kubeconfig_content}
|
||||
)
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "kubeconfig_content" in serializer.errors
|
||||
|
||||
def test_malformed_kubeconfig_is_rejected(self):
|
||||
serializer = KubernetesProviderSecret(
|
||||
data={"kubeconfig_content": "apiVersion: ["}
|
||||
)
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "kubeconfig_content" in serializer.errors
|
||||
|
||||
def test_non_mapping_kubeconfig_is_rejected(self):
|
||||
serializer = KubernetesProviderSecret(data={"kubeconfig_content": "[]"})
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "kubeconfig_content" in serializer.errors
|
||||
|
||||
@@ -186,6 +186,25 @@ def _session_ctx(session: MagicMock) -> MagicMock:
|
||||
return ctx
|
||||
|
||||
|
||||
def _count_result(key: str, count: int) -> MagicMock:
|
||||
return MagicMock(single=MagicMock(return_value={key: count}))
|
||||
|
||||
|
||||
def _directed_drop_results(
|
||||
outgoing_rels: int,
|
||||
incoming_rels: int,
|
||||
nodes: int,
|
||||
) -> list[MagicMock]:
|
||||
return [
|
||||
_count_result("deleted_rels_count", outgoing_rels),
|
||||
_count_result("deleted_rels_count", 0),
|
||||
_count_result("deleted_rels_count", incoming_rels),
|
||||
_count_result("deleted_rels_count", 0),
|
||||
_count_result("deleted_nodes_count", nodes),
|
||||
_count_result("deleted_nodes_count", 0),
|
||||
]
|
||||
|
||||
|
||||
class TestNeo4jSinkSyncWrites:
|
||||
def test_ensure_sync_indexes_runs_create_index_idempotent(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
@@ -310,65 +329,48 @@ class TestNeptuneSinkSyncWrites:
|
||||
|
||||
|
||||
class TestNeptuneSinkDropSubgraph:
|
||||
def test_drop_subgraph_deletes_rels_before_nodes_in_bounded_batches(self):
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
|
||||
rel_record_first = MagicMock()
|
||||
rel_record_first.__getitem__ = lambda _self, key: 50
|
||||
rel_record_drain = MagicMock()
|
||||
rel_record_drain.__getitem__ = lambda _self, key: 0
|
||||
node_record_first = MagicMock()
|
||||
node_record_first.__getitem__ = lambda _self, key: 10
|
||||
node_record_drain = MagicMock()
|
||||
node_record_drain.__getitem__ = lambda _self, key: 0
|
||||
|
||||
run_results = [
|
||||
MagicMock(single=MagicMock(return_value=rel_record_first)),
|
||||
MagicMock(single=MagicMock(return_value=rel_record_drain)),
|
||||
MagicMock(single=MagicMock(return_value=node_record_first)),
|
||||
MagicMock(single=MagicMock(return_value=node_record_drain)),
|
||||
]
|
||||
session.run.side_effect = run_results
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
deleted = sink.drop_subgraph("ignored", "provider-1")
|
||||
|
||||
assert deleted == 10
|
||||
first_query = session.run.call_args_list[0].args[0]
|
||||
assert "DELETE r" in first_query
|
||||
assert "DETACH DELETE" not in first_query
|
||||
# DISTINCT avoids double-counting relationships matched from both ends.
|
||||
assert "DISTINCT r" in first_query
|
||||
third_query = session.run.call_args_list[2].args[0]
|
||||
assert "DELETE n" in third_query
|
||||
assert session.run.call_count == 6
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
|
||||
assert ")-[r]->()" in queries[0]
|
||||
assert ")<-[r]-()" in queries[2]
|
||||
assert "DELETE n" in queries[4]
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
assert all("DISTINCT r" not in query for query in queries)
|
||||
|
||||
first_node = next(i for i, q in enumerate(queries) if "DELETE n" in q)
|
||||
last_rel = max(i for i, q in enumerate(queries) if "DELETE r" in q)
|
||||
assert last_rel < first_node
|
||||
|
||||
|
||||
class TestNeo4jSinkDropSubgraph:
|
||||
"""Neo4j drop deletes relationships then nodes in batches (no ``DETACH DELETE``)."""
|
||||
|
||||
def test_drop_subgraph_deletes_rels_before_nodes_in_bounded_batches(self):
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
|
||||
rel_first = MagicMock()
|
||||
rel_first.get = lambda key, default=0: 50
|
||||
rel_drain = MagicMock()
|
||||
rel_drain.get = lambda key, default=0: 0
|
||||
node_first = MagicMock()
|
||||
node_first.get = lambda key, default=0: 10
|
||||
node_drain = MagicMock()
|
||||
node_drain.get = lambda key, default=0: 0
|
||||
session.run.side_effect = [
|
||||
MagicMock(single=MagicMock(return_value=rel_first)),
|
||||
MagicMock(single=MagicMock(return_value=rel_drain)),
|
||||
MagicMock(single=MagicMock(return_value=node_first)),
|
||||
MagicMock(single=MagicMock(return_value=node_drain)),
|
||||
]
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
|
||||
provider_id = "00000000-0000-0000-0000-000000000abc"
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -376,19 +378,20 @@ class TestNeo4jSinkDropSubgraph:
|
||||
|
||||
# Only phase-2 node counts contribute to the return value.
|
||||
assert deleted == 10
|
||||
assert session.run.call_count == 4
|
||||
assert session.run.call_count == 6
|
||||
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
# Regression guard: the memory blow-up was caused by DETACH DELETE.
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
assert all("DISTINCT r" not in query for query in queries)
|
||||
|
||||
first_query = queries[0]
|
||||
assert "DELETE r" in first_query
|
||||
# DISTINCT avoids double-counting relationships matched from both ends.
|
||||
assert "DISTINCT r" in first_query
|
||||
assert ")-[r]->()" in first_query
|
||||
assert ":`_Provider_00000000000000000000000000000abc`" in first_query
|
||||
|
||||
assert "DELETE n" in queries[2]
|
||||
assert ")<-[r]-()" in queries[2]
|
||||
assert "DELETE n" in queries[4]
|
||||
|
||||
# Relationships must be fully drained before nodes are deleted.
|
||||
first_node = next(i for i, q in enumerate(queries) if "DELETE n" in q)
|
||||
|
||||
@@ -57,9 +57,15 @@ from api.models import (
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from api.v1.serializers import TokenSerializer
|
||||
from api.v1.views import ComplianceOverviewViewSet, TenantFinishACSView
|
||||
from api.v1.views import (
|
||||
ComplianceOverviewViewSet,
|
||||
CustomSAMLLoginView,
|
||||
TenantFinishACSView,
|
||||
)
|
||||
from botocore.exceptions import ClientError, NoCredentialsError
|
||||
from celery import states
|
||||
from conftest import (
|
||||
API_JSON_CONTENT_TYPE,
|
||||
TEST_PASSWORD,
|
||||
@@ -243,6 +249,63 @@ class TestUserViewSet:
|
||||
create_test_user.refresh_from_db()
|
||||
assert create_test_user.company_name == new_company_name
|
||||
|
||||
def test_users_partial_update_same_tenant_other_user_password_denied(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
original_password = "OriginalPassword123@"
|
||||
new_password = "UpdatedPassword123@"
|
||||
target_user = User.objects.create_user(
|
||||
password=original_password,
|
||||
email="target-password-update@example.com",
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenants_fixture[0])
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
},
|
||||
}
|
||||
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=payload,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.check_password(original_password)
|
||||
assert not target_user.check_password(new_password)
|
||||
|
||||
def test_users_partial_update_same_tenant_other_user_email_denied(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
original_email = "target-email-update@example.com"
|
||||
new_email = "updated-target-email@example.com"
|
||||
target_user = User.objects.create_user(
|
||||
password="OriginalPassword123@",
|
||||
email=original_email,
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenants_fixture[0])
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {"email": new_email},
|
||||
},
|
||||
}
|
||||
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=payload,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.email == original_email
|
||||
|
||||
def test_users_partial_update_invalid_content_type(
|
||||
self, authenticated_client, create_test_user
|
||||
):
|
||||
@@ -1490,13 +1553,13 @@ class TestProviderViewSet:
|
||||
("provider_groups", ["provider-groups"]),
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("create_provider_group_relationship")
|
||||
def test_providers_list_include(
|
||||
self,
|
||||
include_values,
|
||||
expected_resources,
|
||||
authenticated_client,
|
||||
providers_fixture,
|
||||
create_provider_group_relationship,
|
||||
):
|
||||
response = authenticated_client.get(
|
||||
reverse("provider-list"), {"include": include_values}
|
||||
@@ -2897,7 +2960,24 @@ class TestProviderSecretViewSet:
|
||||
Provider.ProviderChoices.KUBERNETES.value,
|
||||
ProviderSecret.TypeChoices.STATIC,
|
||||
{
|
||||
"kubeconfig_content": "kubeconfig-content",
|
||||
"kubeconfig_content": """
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test-cluster
|
||||
cluster:
|
||||
server: https://kubernetes.example.test
|
||||
users:
|
||||
- name: test-user
|
||||
user:
|
||||
token: test-token
|
||||
contexts:
|
||||
- name: test-context
|
||||
context:
|
||||
cluster: test-cluster
|
||||
user: test-user
|
||||
current-context: test-context
|
||||
""",
|
||||
},
|
||||
),
|
||||
# M365 client secret credentials
|
||||
@@ -3541,7 +3621,7 @@ class TestScanViewSet:
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scan_json_payload, expected_scanner_args",
|
||||
"scan_json_payload, _expected_scanner_args",
|
||||
[
|
||||
# Case 1: No scanner_args in payload (should use provider's scanner_args)
|
||||
(
|
||||
@@ -3582,21 +3662,15 @@ class TestScanViewSet:
|
||||
),
|
||||
],
|
||||
)
|
||||
@patch("api.v1.views.Task.objects.get")
|
||||
@patch("api.v1.views.perform_scan_task.apply_async")
|
||||
@patch("api.v1.views.enqueue_scan_execution_on_commit")
|
||||
def test_scans_create_valid(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
mock_task_get,
|
||||
mock_enqueue_scan_execution,
|
||||
authenticated_client,
|
||||
scan_json_payload,
|
||||
expected_scanner_args,
|
||||
_expected_scanner_args,
|
||||
providers_fixture,
|
||||
tasks_fixture,
|
||||
):
|
||||
prowler_task = tasks_fixture[0]
|
||||
mock_perform_scan_task.return_value.id = prowler_task.id
|
||||
mock_task_get.return_value = prowler_task
|
||||
*_, provider5 = providers_fixture
|
||||
# Provider5 has these scanner_args
|
||||
# scanner_args={"key1": "value1", "key2": {"key21": "value21"}}
|
||||
@@ -3621,8 +3695,121 @@ class TestScanViewSet:
|
||||
assert scan.name == scan_json_payload["data"]["attributes"]["name"]
|
||||
assert scan.provider == provider5
|
||||
assert scan.trigger == Scan.TriggerChoices.MANUAL
|
||||
mock_enqueue_scan_execution.assert_called_once()
|
||||
# assert scan.scanner_args == expected_scanner_args
|
||||
|
||||
@patch("tasks.tasks.perform_scan_task.apply_async")
|
||||
def test_scans_create_queues_scan_when_provider_has_active_scan(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
authenticated_client,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider, *_ = providers_fixture
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-perform",
|
||||
status=states.PENDING,
|
||||
)
|
||||
prowler_task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
tenant_id=tenant.id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
Scan.objects.create(
|
||||
name="Active scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
tenant_id=tenant.id,
|
||||
task=prowler_task,
|
||||
)
|
||||
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = authenticated_client.post(
|
||||
reverse("scan-list"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "scans",
|
||||
"attributes": {"name": "Duplicate Scan"},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert response.json()["data"]["id"] != str(prowler_task.id)
|
||||
assert Scan.objects.count() == 2
|
||||
queued_scan = Scan.objects.exclude(task=prowler_task).get()
|
||||
assert queued_scan.trigger == Scan.TriggerChoices.MANUAL
|
||||
assert queued_scan.state == StateChoices.AVAILABLE
|
||||
assert queued_scan.task.task_runner_task.status == "QUEUED"
|
||||
mock_perform_scan_task.assert_not_called()
|
||||
|
||||
@patch("tasks.tasks.perform_scan_task.apply_async")
|
||||
def test_scans_create_queues_scan_when_scheduled_scan_is_claimed(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
authenticated_client,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider, *_ = providers_fixture
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-perform-scheduled",
|
||||
status=states.STARTED,
|
||||
)
|
||||
prowler_task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
tenant_id=tenant.id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
Scan.objects.create(
|
||||
name="Claimed scheduled scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
tenant_id=tenant.id,
|
||||
task=prowler_task,
|
||||
)
|
||||
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = authenticated_client.post(
|
||||
reverse("scan-list"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "scans",
|
||||
"attributes": {"name": "Manual Scan"},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert response.json()["data"]["id"] != str(prowler_task.id)
|
||||
assert Scan.objects.count() == 2
|
||||
queued_scan = Scan.objects.exclude(task=prowler_task).get()
|
||||
assert queued_scan.trigger == Scan.TriggerChoices.MANUAL
|
||||
assert queued_scan.state == StateChoices.AVAILABLE
|
||||
assert queued_scan.task.task_runner_task.status == "QUEUED"
|
||||
mock_perform_scan_task.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scan_json_payload, error_code",
|
||||
[
|
||||
@@ -4069,7 +4256,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
presigned_url = (
|
||||
@@ -4175,7 +4362,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.TaskSerializer",
|
||||
lambda *args, **kwargs: type("S", (), {"data": dummy}),
|
||||
lambda *_args, **_kwargs: type("S", (), {"data": dummy}),
|
||||
)
|
||||
|
||||
framework = get_compliance_frameworks(scan.provider.provider)[0]
|
||||
@@ -4233,7 +4420,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
match_key = "path/compliance/mitre_attack_aws.csv"
|
||||
@@ -4244,6 +4431,7 @@ class TestScanViewSet:
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
del Prefix
|
||||
return {"Contents": [{"Key": match_key}]}
|
||||
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
@@ -4275,7 +4463,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
old_key = "path/compliance/prowler-output-aws-20240101000000_cis_1.4_aws.csv"
|
||||
@@ -4283,6 +4471,7 @@ class TestScanViewSet:
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
del Prefix
|
||||
return {
|
||||
"Contents": [
|
||||
{
|
||||
@@ -4356,11 +4545,12 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
del Prefix
|
||||
return {"Contents": []}
|
||||
|
||||
def get_object(self, Bucket, Key):
|
||||
@@ -4546,7 +4736,7 @@ class TestScanViewSet:
|
||||
inserted_at=base + timedelta(hours=1)
|
||||
)
|
||||
|
||||
mock_task_serializer.side_effect = lambda instance, *a, **k: SimpleNamespace(
|
||||
mock_task_serializer.side_effect = lambda instance, *_a, **_k: SimpleNamespace(
|
||||
data={"id": str(instance.id), "state": StateChoices.EXECUTING}
|
||||
)
|
||||
|
||||
@@ -6278,9 +6468,8 @@ class TestResourceViewSet:
|
||||
)
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_resources_metadata_retrieve(
|
||||
self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_resources_metadata_retrieve(self, authenticated_client, resources_fixture):
|
||||
resource_1, *_ = resources_fixture
|
||||
response = authenticated_client.get(
|
||||
reverse("resource-metadata"),
|
||||
@@ -6300,8 +6489,9 @@ class TestResourceViewSet:
|
||||
assert set(data["data"]["attributes"]["types"]) == expected_resource_types
|
||||
assert set(data["data"]["attributes"]["groups"]) == expected_groups
|
||||
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_resources_metadata_resource_filter_retrieve(
|
||||
self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture
|
||||
self, authenticated_client, resources_fixture
|
||||
):
|
||||
resource_1, *_ = resources_fixture
|
||||
response = authenticated_client.get(
|
||||
@@ -7218,6 +7408,26 @@ class TestFindingViewSet:
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert response.json()["errors"][0]["code"] == "invalid"
|
||||
|
||||
def test_findings_updated_at_range_too_large_with_inserted_at_filter(
|
||||
self, authenticated_client
|
||||
):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[inserted_at]": TODAY,
|
||||
"filter[updated_at.gte]": today_after_n_days(
|
||||
-(settings.FINDINGS_MAX_DAYS_IN_RANGE + 1)
|
||||
),
|
||||
"filter[updated_at.lte]": TODAY,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert response.json()["errors"][0]["code"] == "invalid"
|
||||
assert response.json()["errors"][0]["source"]["pointer"] == (
|
||||
"/data/attributes/updated_at"
|
||||
)
|
||||
|
||||
def test_findings_list(self, authenticated_client, findings_fixture):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"), {"filter[inserted_at]": TODAY}
|
||||
@@ -7229,6 +7439,170 @@ class TestFindingViewSet:
|
||||
== findings_fixture[0].status
|
||||
)
|
||||
|
||||
def test_findings_list_inserted_at_accepts_timestamp_precision_filters(
|
||||
self, authenticated_client, scans_fixture
|
||||
):
|
||||
scan, *_ = scans_fixture
|
||||
|
||||
def create_finding(uid, inserted_at):
|
||||
finding = Finding.objects.create(
|
||||
id=datetime_to_uuid7(inserted_at),
|
||||
tenant_id=scan.tenant_id,
|
||||
uid=uid,
|
||||
scan=scan,
|
||||
status=Status.FAIL,
|
||||
status_extended="timestamp precision status",
|
||||
impact=Severity.medium,
|
||||
severity=Severity.medium,
|
||||
check_id="timestamp_precision_check",
|
||||
check_metadata={
|
||||
"CheckId": "timestamp_precision_check",
|
||||
"Description": "timestamp precision check",
|
||||
"servicename": "ec2",
|
||||
},
|
||||
first_seen_at=inserted_at,
|
||||
)
|
||||
Finding.all_objects.filter(pk=finding.pk).update(
|
||||
inserted_at=inserted_at,
|
||||
updated_at=inserted_at,
|
||||
)
|
||||
finding.refresh_from_db()
|
||||
return finding
|
||||
|
||||
create_finding(
|
||||
"timestamp_precision_early",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 100000, tzinfo=UTC),
|
||||
)
|
||||
late_finding = create_finding(
|
||||
"timestamp_precision_late",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[inserted_at.gte]": "2026-01-15T10:30:00.150Z",
|
||||
"filter[inserted_at.lte]": "2026-01-15T10:30:00.250Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{"filter[inserted_at]": "2026-01-15T10:30:00.200Z"},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
def test_findings_list_updated_at_accepts_timestamp_precision_filters(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
early_finding, late_finding, *_ = findings_fixture
|
||||
early_updated_at = datetime(2026, 1, 15, 10, 30, 0, 100000, tzinfo=UTC)
|
||||
late_updated_at = datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC)
|
||||
Finding.all_objects.filter(pk=early_finding.pk).update(
|
||||
updated_at=early_updated_at
|
||||
)
|
||||
Finding.all_objects.filter(pk=late_finding.pk).update(
|
||||
updated_at=late_updated_at
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[updated_at.gte]": "2026-01-15T10:30:00.150Z",
|
||||
"filter[updated_at.lte]": "2026-01-15T10:30:00.250Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{"filter[updated_at]": "2026-01-15T10:30:00.200Z"},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {late_finding.uid}
|
||||
|
||||
def test_findings_list_inserted_at_and_updated_at_filters_are_combined(
|
||||
self, authenticated_client, scans_fixture
|
||||
):
|
||||
scan, *_ = scans_fixture
|
||||
|
||||
def create_finding(uid, inserted_at, updated_at):
|
||||
finding = Finding.objects.create(
|
||||
id=datetime_to_uuid7(inserted_at),
|
||||
tenant_id=scan.tenant_id,
|
||||
uid=uid,
|
||||
scan=scan,
|
||||
status=Status.FAIL,
|
||||
status_extended="timestamp precision status",
|
||||
impact=Severity.medium,
|
||||
severity=Severity.medium,
|
||||
check_id="timestamp_precision_check",
|
||||
check_metadata={
|
||||
"CheckId": "timestamp_precision_check",
|
||||
"Description": "timestamp precision check",
|
||||
"servicename": "ec2",
|
||||
},
|
||||
first_seen_at=inserted_at,
|
||||
)
|
||||
Finding.all_objects.filter(pk=finding.pk).update(
|
||||
inserted_at=inserted_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
finding.refresh_from_db()
|
||||
return finding
|
||||
|
||||
matching_finding = create_finding(
|
||||
"timestamp_precision_combined_match",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC),
|
||||
datetime(2026, 1, 15, 11, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
create_finding(
|
||||
"timestamp_precision_combined_inserted_only",
|
||||
datetime(2026, 1, 15, 10, 30, 0, 200000, tzinfo=UTC),
|
||||
datetime(2026, 1, 15, 12, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
create_finding(
|
||||
"timestamp_precision_combined_updated_only",
|
||||
datetime(2026, 1, 15, 9, 30, 0, 200000, tzinfo=UTC),
|
||||
datetime(2026, 1, 15, 11, 30, 0, 200000, tzinfo=UTC),
|
||||
)
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-list"),
|
||||
{
|
||||
"filter[inserted_at.gte]": "2026-01-15T10:30:00.150Z",
|
||||
"filter[inserted_at.lte]": "2026-01-15T10:30:00.250Z",
|
||||
"filter[updated_at.gte]": "2026-01-15T11:30:00.150Z",
|
||||
"filter[updated_at.lte]": "2026-01-15T11:30:00.250Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
returned_uids = {
|
||||
finding["attributes"]["uid"] for finding in response.json()["data"]
|
||||
}
|
||||
assert returned_uids == {matching_finding.uid}
|
||||
|
||||
def test_findings_list_resource_tags_no_n_plus_one(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
@@ -7611,9 +7985,8 @@ class TestFindingViewSet:
|
||||
)
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_findings_metadata_retrieve(
|
||||
self, authenticated_client, findings_fixture, backfill_scan_metadata_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_findings_metadata_retrieve(self, authenticated_client, findings_fixture):
|
||||
finding_1, *_ = findings_fixture
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata"),
|
||||
@@ -7636,8 +8009,9 @@ class TestFindingViewSet:
|
||||
)
|
||||
# assert data["data"]["attributes"]["tags"] == expected_tags
|
||||
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_findings_metadata_resource_filter_retrieve(
|
||||
self, authenticated_client, findings_fixture, backfill_scan_metadata_fixture
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
finding_1, *_ = findings_fixture
|
||||
response = authenticated_client.get(
|
||||
@@ -7694,6 +8068,23 @@ class TestFindingViewSet:
|
||||
]
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filter_name",
|
||||
["inserted_at", "inserted_at.gte", "inserted_at.lte"],
|
||||
)
|
||||
def test_findings_metadata_rejects_timestamp_precision_filters(
|
||||
self, authenticated_client, filter_name
|
||||
):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata"),
|
||||
{f"filter[{filter_name}]": "2048-01-01T10:30:00Z"},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
error = response.json()["errors"][0]
|
||||
assert error["detail"] == "Enter a valid date."
|
||||
assert error["code"] == "invalid"
|
||||
|
||||
def test_findings_metadata_backfill(
|
||||
self, authenticated_client, scans_fixture, findings_fixture
|
||||
):
|
||||
@@ -7806,9 +8197,8 @@ class TestFindingViewSet:
|
||||
attributes = response.json()["data"]["attributes"]
|
||||
assert set(attributes["categories"]) == {"gen-ai", "security"}
|
||||
|
||||
def test_findings_metadata_latest_categories(
|
||||
self, authenticated_client, latest_scan_finding_with_categories
|
||||
):
|
||||
@pytest.mark.usefixtures("latest_scan_finding_with_categories")
|
||||
def test_findings_metadata_latest_categories(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata_latest"),
|
||||
)
|
||||
@@ -7816,9 +8206,8 @@ class TestFindingViewSet:
|
||||
attributes = response.json()["data"]["attributes"]
|
||||
assert set(attributes["categories"]) == {"gen-ai", "iam"}
|
||||
|
||||
def test_findings_metadata_latest_groups(
|
||||
self, authenticated_client, latest_scan_finding_with_categories
|
||||
):
|
||||
@pytest.mark.usefixtures("latest_scan_finding_with_categories")
|
||||
def test_findings_metadata_latest_groups(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata_latest"),
|
||||
)
|
||||
@@ -8338,16 +8727,14 @@ class TestInvitationViewSet:
|
||||
expires_at=self.TOMORROW,
|
||||
)
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
assert not Membership.objects.filter(
|
||||
user__email__iexact=user.email, tenant=tenant
|
||||
).exists()
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
invitation.refresh_from_db()
|
||||
@@ -8356,13 +8743,46 @@ class TestInvitationViewSet:
|
||||
).exists()
|
||||
assert invitation.state == Invitation.State.ACCEPTED.value
|
||||
|
||||
def test_invitations_accept_invitation_invalid_token(self, authenticated_client):
|
||||
data = {
|
||||
"invitation_token": "invalid_token",
|
||||
}
|
||||
def test_invitations_accept_invitation_existing_membership(
|
||||
self,
|
||||
authenticated_client,
|
||||
create_test_user,
|
||||
tenants_fixture,
|
||||
):
|
||||
*_, tenant = tenants_fixture
|
||||
user = create_test_user
|
||||
|
||||
invitation = Invitation.objects.create(
|
||||
tenant=tenant,
|
||||
email=TEST_USER,
|
||||
inviter=user,
|
||||
expires_at=self.TOMORROW,
|
||||
)
|
||||
Membership.objects.create(user=user, tenant=tenant)
|
||||
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"),
|
||||
data=data,
|
||||
format="vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
invitation.refresh_from_db()
|
||||
assert invitation.state == Invitation.State.ACCEPTED.value
|
||||
assert (
|
||||
Membership.objects.filter(
|
||||
user__email__iexact=user.email, tenant=tenant
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_invitations_accept_invitation_invalid_token(self, authenticated_client):
|
||||
data = {"invitation_token": "invalid_token"}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
@@ -8376,12 +8796,10 @@ class TestInvitationViewSet:
|
||||
invitation.email = TEST_USER
|
||||
invitation.save()
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_410_GONE
|
||||
@@ -8417,12 +8835,10 @@ class TestInvitationViewSet:
|
||||
invitation.email = TEST_USER
|
||||
invitation.save()
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
@@ -8440,12 +8856,10 @@ class TestInvitationViewSet:
|
||||
invitation.email = TEST_USER
|
||||
invitation.save()
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
@@ -10521,9 +10935,8 @@ class TestOverviewViewSet:
|
||||
response = authenticated_client.put(reverse("overview-list"))
|
||||
assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED
|
||||
|
||||
def test_overview_providers_list(
|
||||
self, authenticated_client, scan_summaries_fixture, resources_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_providers_list(self, authenticated_client, resources_fixture):
|
||||
response = authenticated_client.get(reverse("overview-providers"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == 1
|
||||
@@ -10534,10 +10947,10 @@ class TestOverviewViewSet:
|
||||
# Aggregated resources include all AWS providers present in the tenant
|
||||
assert response.json()["data"][0]["attributes"]["resources"]["total"] == 3
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_providers_aggregates_same_provider_type(
|
||||
self,
|
||||
authenticated_client,
|
||||
scan_summaries_fixture,
|
||||
resources_fixture,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
@@ -10588,10 +11001,10 @@ class TestOverviewViewSet:
|
||||
assert attributes["findings"]["muted"] == 7
|
||||
assert attributes["resources"]["total"] == 4
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_providers_count(
|
||||
self,
|
||||
authenticated_client,
|
||||
scan_summaries_fixture,
|
||||
resources_fixture,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
@@ -11057,15 +11470,15 @@ class TestOverviewViewSet:
|
||||
assert data[0]["id"] == str(snapshot1.id)
|
||||
assert data[0]["attributes"]["overall_score"] == "55.55"
|
||||
|
||||
def test_overview_services_list_no_required_filters(
|
||||
self, authenticated_client, scan_summaries_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_services_list_no_required_filters(self, authenticated_client):
|
||||
response = authenticated_client.get(reverse("overview-services"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
# Should return services from latest scans
|
||||
assert len(response.json()["data"]) == 2
|
||||
|
||||
def test_overview_regions_list(self, authenticated_client, scan_summaries_fixture):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_regions_list(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("overview-regions"), {"filter[inserted_at]": TODAY}
|
||||
)
|
||||
@@ -11091,7 +11504,8 @@ class TestOverviewViewSet:
|
||||
assert regions["aws:region2"]["fail"] == 1
|
||||
assert regions["aws:region2"]["muted"] == 3
|
||||
|
||||
def test_overview_services_list(self, authenticated_client, scan_summaries_fixture):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_services_list(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("overview-services"), {"filter[inserted_at]": TODAY}
|
||||
)
|
||||
@@ -11739,9 +12153,8 @@ class TestOverviewViewSet:
|
||||
assert results_by_type["internet-exposed"]["total_findings"] == 10
|
||||
assert results_by_type["internet-exposed"]["failed_findings"] == 5
|
||||
|
||||
def test_overview_services_region_filter(
|
||||
self, authenticated_client, scan_summaries_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_services_region_filter(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("overview-services"),
|
||||
{"filter[region]": "region1"},
|
||||
@@ -11809,7 +12222,7 @@ class TestOverviewViewSet:
|
||||
assert "gcp-service" not in service_ids
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status_filter,field_to_check",
|
||||
"status_filter,_field_to_check",
|
||||
[
|
||||
("FAIL", "fail"),
|
||||
("PASS", "_pass"),
|
||||
@@ -11821,7 +12234,7 @@ class TestOverviewViewSet:
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
status_filter,
|
||||
field_to_check,
|
||||
_field_to_check,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
@@ -12461,8 +12874,9 @@ class TestOverviewViewSet:
|
||||
assert data[0]["attributes"]["new_failed_findings"] == 5
|
||||
assert data[0]["attributes"]["resources_count"] == 10
|
||||
|
||||
@pytest.mark.usefixtures("tenant_compliance_summary_fixture")
|
||||
def test_compliance_watchlist_no_filters_uses_tenant_summary(
|
||||
self, authenticated_client, tenant_compliance_summary_fixture
|
||||
self, authenticated_client
|
||||
):
|
||||
response = authenticated_client.get(reverse("overview-compliance-watchlist"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
@@ -12482,10 +12896,10 @@ class TestOverviewViewSet:
|
||||
assert by_id["gdpr_aws"]["requirements_failed"] == 0
|
||||
assert by_id["gdpr_aws"]["total_requirements"] == 7
|
||||
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_with_provider_filter_uses_provider_scores(
|
||||
self,
|
||||
authenticated_client,
|
||||
provider_compliance_scores_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
provider1 = providers_fixture[0]
|
||||
@@ -12502,9 +12916,8 @@ class TestOverviewViewSet:
|
||||
assert by_id["aws_cis_2.0"]["requirements_manual"] == 1
|
||||
assert by_id["aws_cis_2.0"]["total_requirements"] == 3
|
||||
|
||||
def test_compliance_watchlist_fail_dominant_logic(
|
||||
self, authenticated_client, provider_compliance_scores_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_fail_dominant_logic(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
f"{reverse('overview-compliance-watchlist')}?filter[provider_type]=aws"
|
||||
)
|
||||
@@ -12519,10 +12932,10 @@ class TestOverviewViewSet:
|
||||
assert aws_cis["requirements_manual"] == 1
|
||||
assert aws_cis["total_requirements"] == 3
|
||||
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_provider_id_in_filter(
|
||||
self,
|
||||
authenticated_client,
|
||||
provider_compliance_scores_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
@@ -12535,10 +12948,10 @@ class TestOverviewViewSet:
|
||||
data = response.json()["data"]
|
||||
assert len(data) >= 1
|
||||
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_provider_groups_filter(
|
||||
self,
|
||||
authenticated_client,
|
||||
provider_compliance_scores_fixture,
|
||||
providers_fixture,
|
||||
provider_groups_fixture,
|
||||
tenants_fixture,
|
||||
@@ -13438,6 +13851,26 @@ class TestSAMLTokenValidation:
|
||||
assert response2.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCustomSAMLLoginView:
|
||||
def test_dispatch_clears_stale_callback_url_when_request_has_none(self):
|
||||
request = RequestFactory().get("/api/v1/saml/login/testtenant/")
|
||||
request.session = {
|
||||
"saml_callback_url": "/invitation/accept?invitation_token=old-token"
|
||||
}
|
||||
|
||||
with patch(
|
||||
"allauth.socialaccount.providers.saml.views.LoginView.dispatch",
|
||||
return_value=JsonResponse({}),
|
||||
):
|
||||
response = CustomSAMLLoginView.as_view()(
|
||||
request, organization_slug="testtenant"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert "saml_callback_url" not in request.session
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestSAMLInitiateAPIView:
|
||||
def test_valid_email_domain_and_certificates(
|
||||
@@ -13449,7 +13882,7 @@ class TestSAMLInitiateAPIView:
|
||||
url = reverse("api_saml_initiate")
|
||||
payload = {"email_domain": saml_setup["email"]}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="json")
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
assert (
|
||||
@@ -13458,11 +13891,42 @@ class TestSAMLInitiateAPIView:
|
||||
)
|
||||
assert "SAMLRequest" not in response.url
|
||||
|
||||
def test_valid_email_domain_preserves_safe_callback_url(
|
||||
self, authenticated_client, saml_setup
|
||||
):
|
||||
url = reverse("api_saml_initiate")
|
||||
callback_url = "/invitation/accept?invitation_token=test-token"
|
||||
payload = {
|
||||
"email_domain": saml_setup["email"],
|
||||
"callback_url": callback_url,
|
||||
}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
query_params = parse_qs(urlparse(response.url).query)
|
||||
assert query_params["callback_url"] == [callback_url]
|
||||
|
||||
def test_valid_email_domain_rejects_external_callback_url(
|
||||
self, authenticated_client, saml_setup
|
||||
):
|
||||
url = reverse("api_saml_initiate")
|
||||
payload = {
|
||||
"email_domain": saml_setup["email"],
|
||||
"callback_url": "https://attacker.example/invitation",
|
||||
}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
query_params = parse_qs(urlparse(response.url).query)
|
||||
assert "callback_url" not in query_params
|
||||
|
||||
def test_invalid_email_domain(self, authenticated_client):
|
||||
url = reverse("api_saml_initiate")
|
||||
payload = {"email_domain": "user@unauthorized.com"}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="json")
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
assert response.json()["errors"]["detail"] == "Unauthorized domain."
|
||||
@@ -13645,7 +14109,8 @@ class TestTenantFinishACSView:
|
||||
)
|
||||
)
|
||||
request.user = user
|
||||
request.session = {}
|
||||
callback_url = "/invitation/accept?invitation_token=test-token"
|
||||
request.session = {"saml_callback_url": callback_url}
|
||||
|
||||
with (
|
||||
patch(
|
||||
@@ -13687,6 +14152,7 @@ class TestTenantFinishACSView:
|
||||
assert parsed_url.netloc == expected_callback_host
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
assert "id" in query_params
|
||||
assert query_params["callbackUrl"] == [callback_url]
|
||||
|
||||
token_id = query_params["id"][0]
|
||||
token_obj = SAMLToken.objects.get(id=token_id)
|
||||
@@ -18090,10 +18556,10 @@ class TestFindingGroupViewSet:
|
||||
],
|
||||
ids=["summary_path", "finding_level_path"],
|
||||
)
|
||||
@pytest.mark.usefixtures("finding_groups_title_variants_fixture")
|
||||
def test_check_title_icontains_includes_all_title_variants(
|
||||
self,
|
||||
authenticated_client,
|
||||
finding_groups_title_variants_fixture,
|
||||
extra_filters,
|
||||
):
|
||||
"""
|
||||
|
||||
@@ -7,9 +7,19 @@ from allauth.socialaccount.providers.oauth2.client import OAuth2Client
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import InvitationTokenExpiredException
|
||||
from api.models import Integration, Invitation, Processor, Provider, Resource
|
||||
from api.models import (
|
||||
Integration,
|
||||
Invitation,
|
||||
Membership,
|
||||
Processor,
|
||||
Provider,
|
||||
Resource,
|
||||
Role,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.v1.serializers import FindingMetadataSerializer
|
||||
from django.contrib.postgres.aggregates import ArrayAgg
|
||||
from django.db import transaction
|
||||
from django.db.models import Subquery
|
||||
from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
@@ -538,6 +548,35 @@ def validate_invitation(
|
||||
return invitation
|
||||
|
||||
|
||||
def accept_invitation_for_user(
|
||||
*, user, invitation_token: str, raise_not_found: bool = False
|
||||
):
|
||||
with transaction.atomic(using=MainRouter.admin_db):
|
||||
invitation = validate_invitation(
|
||||
invitation_token, user.email, raise_not_found=raise_not_found
|
||||
)
|
||||
with rls_transaction(str(invitation.tenant_id), using=MainRouter.admin_db):
|
||||
membership, _ = Membership.objects.using(MainRouter.admin_db).get_or_create(
|
||||
user=user,
|
||||
tenant=invitation.tenant,
|
||||
defaults={"role": Membership.RoleChoices.MEMBER},
|
||||
)
|
||||
invitation_roles = Role.objects.using(MainRouter.admin_db).filter(
|
||||
invitations=invitation
|
||||
)
|
||||
for role in invitation_roles:
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).get_or_create(
|
||||
user=user,
|
||||
role=role,
|
||||
defaults={"tenant": invitation.tenant},
|
||||
)
|
||||
|
||||
invitation.state = Invitation.State.ACCEPTED
|
||||
invitation.save(using=MainRouter.admin_db)
|
||||
|
||||
return invitation, membership
|
||||
|
||||
|
||||
# ToRemove after removing the fallback mechanism in /findings/metadata
|
||||
def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
|
||||
filtered_ids = filtered_queryset.order_by().values("id")
|
||||
|
||||
@@ -213,7 +213,8 @@ from rest_framework_json_api import serializers
|
||||
"properties": {
|
||||
"kubeconfig_content": {
|
||||
"type": "string",
|
||||
"description": "The content of the Kubernetes kubeconfig file, encoded as a string.",
|
||||
"description": "The content of the Kubernetes kubeconfig file, encoded as a string. "
|
||||
"Kubeconfig exec authentication is not supported in Prowler Cloud for security reasons.",
|
||||
}
|
||||
},
|
||||
"required": ["kubeconfig_content"],
|
||||
|
||||
@@ -2,6 +2,7 @@ import base64
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import yaml
|
||||
from api.db_router import MainRouter
|
||||
from api.exceptions import ConflictException
|
||||
from api.models import (
|
||||
@@ -1530,6 +1531,32 @@ class FindingMetadataSerializer(BaseSerializerV1):
|
||||
|
||||
|
||||
# Provider secrets
|
||||
KUBERNETES_KUBECONFIG_EXEC_ERROR = (
|
||||
"Kubernetes kubeconfig exec authentication is not supported in Prowler Cloud "
|
||||
"for security reasons."
|
||||
)
|
||||
KUBERNETES_KUBECONFIG_INVALID_ERROR = "Invalid Kubernetes kubeconfig content."
|
||||
|
||||
|
||||
def kubeconfig_contains_exec_auth(kubeconfig: dict) -> bool:
|
||||
users = kubeconfig.get("users", [])
|
||||
if not isinstance(users, list):
|
||||
raise ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
for user_entry in users:
|
||||
if not isinstance(user_entry, dict):
|
||||
raise ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
user = user_entry.get("user", {})
|
||||
if not isinstance(user, dict):
|
||||
raise ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
if "exec" in user:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
|
||||
@staticmethod
|
||||
def validate_secret_based_on_provider(
|
||||
@@ -1711,6 +1738,22 @@ class MongoDBAtlasProviderSecret(serializers.Serializer):
|
||||
class KubernetesProviderSecret(serializers.Serializer):
|
||||
kubeconfig_content = serializers.CharField()
|
||||
|
||||
def validate_kubeconfig_content(self, kubeconfig_content):
|
||||
try:
|
||||
kubeconfig = yaml.safe_load(kubeconfig_content)
|
||||
except yaml.YAMLError as exc:
|
||||
raise serializers.ValidationError(
|
||||
KUBERNETES_KUBECONFIG_INVALID_ERROR
|
||||
) from exc
|
||||
|
||||
if not isinstance(kubeconfig, dict):
|
||||
raise serializers.ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
if kubeconfig_contains_exec_auth(kubeconfig):
|
||||
raise serializers.ValidationError(KUBERNETES_KUBECONFIG_EXEC_ERROR)
|
||||
|
||||
return kubeconfig_content
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
@@ -3147,6 +3190,9 @@ class ProcessorUpdateSerializer(BaseWriteSerializer):
|
||||
|
||||
class SamlInitiateSerializer(BaseSerializerV1):
|
||||
email_domain = serializers.CharField()
|
||||
callback_url = serializers.CharField(
|
||||
required=False, allow_blank=True, max_length=2048
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "saml-initiate"
|
||||
|
||||
@@ -9,7 +9,7 @@ from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
||||
from urllib.parse import urljoin
|
||||
from urllib.parse import urlencode, urljoin
|
||||
|
||||
import sentry_sdk
|
||||
from allauth.socialaccount.models import SocialAccount, SocialApp
|
||||
@@ -50,6 +50,7 @@ from api.filters import (
|
||||
FindingGroupAggregatedComputedFilter,
|
||||
FindingGroupFilter,
|
||||
FindingGroupSummaryFilter,
|
||||
FindingMetadataFilter,
|
||||
IntegrationFilter,
|
||||
IntegrationJiraFindingsFilter,
|
||||
InvitationFilter,
|
||||
@@ -128,6 +129,7 @@ from api.renderers import APIJSONRenderer, PlainTextRenderer
|
||||
from api.rls import Tenant
|
||||
from api.utils import (
|
||||
CustomOAuth2Client,
|
||||
accept_invitation_for_user,
|
||||
get_findings_metadata_no_aggregations,
|
||||
initialize_prowler_integration,
|
||||
initialize_prowler_provider,
|
||||
@@ -235,7 +237,7 @@ from api.v1.serializers import (
|
||||
UserUpdateSerializer,
|
||||
)
|
||||
from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
|
||||
from celery import chain, states
|
||||
from celery import chain
|
||||
from celery.result import AsyncResult
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
@@ -281,7 +283,6 @@ from django.utils.dateparse import parse_date
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.views.decorators.cache import cache_control
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from drf_spectacular.settings import spectacular_settings
|
||||
from drf_spectacular.types import OpenApiTypes
|
||||
from drf_spectacular.utils import (
|
||||
@@ -320,17 +321,20 @@ from tasks.beat import schedule_provider_scan
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.export import get_s3_client
|
||||
from tasks.tasks import (
|
||||
QUEUED_SCAN_TASK_STATE,
|
||||
backfill_compliance_summaries_task,
|
||||
backfill_scan_resource_summaries_task,
|
||||
check_integration_connection_task,
|
||||
check_lighthouse_connection_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
check_provider_connection_task,
|
||||
create_scan_task_record,
|
||||
delete_provider_task,
|
||||
delete_tenant_task,
|
||||
enqueue_scan_execution_on_commit,
|
||||
get_active_provider_scan,
|
||||
jira_integration_task,
|
||||
mute_historical_findings_task,
|
||||
perform_scan_task,
|
||||
reaggregate_all_finding_group_summaries_task,
|
||||
refresh_lighthouse_provider_models_task,
|
||||
)
|
||||
@@ -541,6 +545,46 @@ class SchemaView(SpectacularAPIView):
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
|
||||
SAML_CALLBACK_SESSION_KEY = "saml_callback_url"
|
||||
|
||||
|
||||
def _safe_callback_path(value):
|
||||
if not value or not isinstance(value, str):
|
||||
return None
|
||||
if not value.startswith("/") or value.startswith("//"):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _get_request_invitation_token(request):
|
||||
for source_name in ("data", "POST"):
|
||||
data = getattr(request, source_name, None) or {}
|
||||
if not hasattr(data, "get"):
|
||||
continue
|
||||
invitation_token = data.get("invitation_token")
|
||||
if invitation_token:
|
||||
return invitation_token
|
||||
|
||||
wrapped_request = getattr(request, "_request", None)
|
||||
if wrapped_request and wrapped_request is not request:
|
||||
return _get_request_invitation_token(wrapped_request)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _accept_social_invitation(request, user):
|
||||
invitation_token = _get_request_invitation_token(request)
|
||||
tenant_id = getattr(request, "prowler_invitation_tenant_id", None)
|
||||
if invitation_token and not tenant_id:
|
||||
invitation, _ = accept_invitation_for_user(
|
||||
user=user,
|
||||
invitation_token=invitation_token,
|
||||
raise_not_found=True,
|
||||
)
|
||||
tenant_id = str(invitation.tenant_id)
|
||||
return tenant_id
|
||||
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
class GoogleSocialLoginView(SocialLoginView):
|
||||
adapter_class = GoogleOAuth2Adapter
|
||||
@@ -551,7 +595,11 @@ class GoogleSocialLoginView(SocialLoginView):
|
||||
original_response = super().get_response()
|
||||
|
||||
if self.user and self.user.is_authenticated:
|
||||
serializer = TokenSocialLoginSerializer(data={"email": self.user.email})
|
||||
tenant_id = _accept_social_invitation(self.request, self.user)
|
||||
serializer_data = {"email": self.user.email}
|
||||
if tenant_id:
|
||||
serializer_data["tenant_id"] = tenant_id
|
||||
serializer = TokenSocialLoginSerializer(data=serializer_data)
|
||||
try:
|
||||
serializer.is_valid(raise_exception=True)
|
||||
except TokenError as e:
|
||||
@@ -576,7 +624,11 @@ class GithubSocialLoginView(SocialLoginView):
|
||||
original_response = super().get_response()
|
||||
|
||||
if self.user and self.user.is_authenticated:
|
||||
serializer = TokenSocialLoginSerializer(data={"email": self.user.email})
|
||||
tenant_id = _accept_social_invitation(self.request, self.user)
|
||||
serializer_data = {"email": self.user.email}
|
||||
if tenant_id:
|
||||
serializer_data["tenant_id"] = tenant_id
|
||||
serializer = TokenSocialLoginSerializer(data=serializer_data)
|
||||
|
||||
try:
|
||||
serializer.is_valid(raise_exception=True)
|
||||
@@ -636,6 +688,10 @@ class CustomSAMLLoginView(LoginView):
|
||||
|
||||
This approach maintains security while providing better UX.
|
||||
"""
|
||||
callback_url = _safe_callback_path(request.GET.get("callback_url"))
|
||||
request.session.pop(SAML_CALLBACK_SESSION_KEY, None)
|
||||
if callback_url:
|
||||
request.session[SAML_CALLBACK_SESSION_KEY] = callback_url
|
||||
if request.method == "GET":
|
||||
# Convert GET to POST while preserving parameters
|
||||
request.method = "POST"
|
||||
@@ -680,6 +736,11 @@ class SAMLInitiateAPIView(GenericAPIView):
|
||||
"saml_login", kwargs={"organization_slug": config.email_domain}
|
||||
)
|
||||
login_url = urljoin(api_host, login_path)
|
||||
callback_url = _safe_callback_path(
|
||||
serializer.validated_data.get("callback_url")
|
||||
)
|
||||
if callback_url:
|
||||
login_url = f"{login_url}?{urlencode({'callback_url': callback_url})}"
|
||||
|
||||
return redirect(login_url)
|
||||
|
||||
@@ -895,7 +956,13 @@ class TenantFinishACSView(FinishACSView):
|
||||
token=token_data, user=user
|
||||
)
|
||||
callback_url = env.str("SAML_SSO_CALLBACK_URL")
|
||||
redirect_url = f"{callback_url}?id={saml_token.id}"
|
||||
redirect_params = {"id": str(saml_token.id)}
|
||||
saml_callback_url = _safe_callback_path(
|
||||
request.session.pop(SAML_CALLBACK_SESSION_KEY, None)
|
||||
)
|
||||
if saml_callback_url:
|
||||
redirect_params["callbackUrl"] = saml_callback_url
|
||||
redirect_url = f"{callback_url}?{urlencode(redirect_params)}"
|
||||
request.session.pop("saml_user_created", None)
|
||||
|
||||
return redirect(redirect_url)
|
||||
@@ -947,8 +1014,8 @@ class UserViewSet(BaseUserViewset):
|
||||
"""
|
||||
Returns the required permissions based on the request method.
|
||||
"""
|
||||
if self.action == "me":
|
||||
# No permissions required for me request
|
||||
if self.action in ["me", "partial_update"]:
|
||||
# No permissions required for me and partial_update requests
|
||||
self.required_permissions = []
|
||||
else:
|
||||
# Require permission for the rest of the requests
|
||||
@@ -1002,6 +1069,24 @@ class UserViewSet(BaseUserViewset):
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
user = self.get_object()
|
||||
if user.id != self.request.user.id:
|
||||
role = get_role(self.request.user, self.request.tenant_id)
|
||||
if not getattr(role, Permissions.MANAGE_USERS.value, False):
|
||||
raise ValidationError(
|
||||
"Only users with manage users permission can update other users."
|
||||
)
|
||||
|
||||
serializer = self.get_serializer(user, data=request.data, partial=True)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
self.perform_update(serializer)
|
||||
|
||||
if getattr(user, "_prefetched_objects_cache", None):
|
||||
user._prefetched_objects_cache = {}
|
||||
|
||||
return Response(serializer.data)
|
||||
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
if kwargs["pk"] != str(self.request.user.id):
|
||||
raise ValidationError("Only the current user can be deleted.")
|
||||
@@ -2634,12 +2719,23 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
def create(self, request, *args, **kwargs):
|
||||
input_serializer = self.get_serializer(data=request.data)
|
||||
input_serializer.is_valid(raise_exception=True)
|
||||
provider = input_serializer.validated_data.get("provider")
|
||||
active_scan = None
|
||||
|
||||
# Broker publish is deferred to on_commit so the worker cannot read
|
||||
# Scan before BaseRLSViewSet's dispatch-wide atomic commits.
|
||||
pre_task_id = str(uuid.uuid4())
|
||||
|
||||
with transaction.atomic():
|
||||
if provider:
|
||||
provider = Provider.objects.select_for_update().get(
|
||||
id=provider.id,
|
||||
tenant_id=self.request.tenant_id,
|
||||
)
|
||||
active_scan = get_active_provider_scan(
|
||||
self.request.tenant_id, provider.id
|
||||
)
|
||||
|
||||
scan = input_serializer.save()
|
||||
scan.task_id = pre_task_id
|
||||
scan.save(update_fields=["task_id"])
|
||||
@@ -2650,29 +2746,18 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
|
||||
task_result, _ = TaskResult.objects.get_or_create(
|
||||
task_id=pre_task_id,
|
||||
defaults={"status": states.PENDING, "task_name": "scan-perform"},
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=pre_task_id,
|
||||
prowler_task = create_scan_task_record(
|
||||
tenant_id=self.request.tenant_id,
|
||||
defaults={"task_runner_task": task_result},
|
||||
task_id=pre_task_id,
|
||||
task_status=(QUEUED_SCAN_TASK_STATE if active_scan else None),
|
||||
)
|
||||
|
||||
scan_kwargs = {
|
||||
"tenant_id": self.request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
}
|
||||
|
||||
transaction.on_commit(
|
||||
lambda: perform_scan_task.apply_async(
|
||||
kwargs=scan_kwargs, task_id=pre_task_id
|
||||
if not active_scan:
|
||||
enqueue_scan_execution_on_commit(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan=scan,
|
||||
task_id=pre_task_id,
|
||||
)
|
||||
)
|
||||
|
||||
self.response_serializer_class = TaskSerializer
|
||||
output_serializer = self.get_serializer(prowler_task)
|
||||
@@ -3833,6 +3918,8 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
def get_filterset_class(self):
|
||||
if self.action in ["latest", "metadata_latest"]:
|
||||
return LatestFindingFilter
|
||||
if self.action == "metadata":
|
||||
return FindingMetadataFilter
|
||||
return FindingFilter
|
||||
|
||||
def get_queryset(self):
|
||||
@@ -4368,25 +4455,12 @@ class InvitationAcceptViewSet(BaseRLSViewSet):
|
||||
invitation_token = serializer.validated_data["invitation_token"]
|
||||
user_email = request.user.email
|
||||
|
||||
invitation = validate_invitation(
|
||||
invitation_token, user_email, raise_not_found=True
|
||||
)
|
||||
|
||||
# Proceed with accepting the invitation
|
||||
user = User.objects.using(MainRouter.admin_db).get(email=user_email)
|
||||
membership = Membership.objects.using(MainRouter.admin_db).create(
|
||||
invitation, membership = accept_invitation_for_user(
|
||||
user=user,
|
||||
tenant=invitation.tenant,
|
||||
invitation_token=invitation_token,
|
||||
raise_not_found=True,
|
||||
)
|
||||
user_role = []
|
||||
for role in invitation.roles.all():
|
||||
user_role.append(
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user, role=role, tenant=invitation.tenant
|
||||
)
|
||||
)
|
||||
invitation.state = Invitation.State.ACCEPTED
|
||||
invitation.save(using=MainRouter.admin_db)
|
||||
|
||||
self.response_serializer_class = MembershipSerializer
|
||||
membership_serializer = self.get_serializer(membership)
|
||||
|
||||
@@ -14,3 +14,17 @@ FINDINGS_TABLE_PARTITION_COUNT = env.int("FINDINGS_TABLE_PARTITION_COUNT", 7)
|
||||
FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS = env.int(
|
||||
"FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS", None
|
||||
)
|
||||
|
||||
# Compliance requirement overviews partitioning (RANGE by UUIDv7 id).
|
||||
# Mirrors the findings settings; kept separate so the ingest-heavy compliance
|
||||
# table can be tuned (and aged out) independently from findings.
|
||||
COMPLIANCE_REQ_OVERVIEW_PARTITION_MONTHS = env.int(
|
||||
"COMPLIANCE_REQ_OVERVIEW_PARTITION_MONTHS", 1
|
||||
)
|
||||
COMPLIANCE_REQ_OVERVIEW_PARTITION_COUNT = env.int(
|
||||
"COMPLIANCE_REQ_OVERVIEW_PARTITION_COUNT", 7
|
||||
)
|
||||
# Setting this to None keeps partitions indefinitely (matches findings default).
|
||||
COMPLIANCE_REQ_OVERVIEW_PARTITION_MAX_AGE_MONTHS = env.int(
|
||||
"COMPLIANCE_REQ_OVERVIEW_PARTITION_MAX_AGE_MONTHS", None
|
||||
)
|
||||
|
||||
@@ -11,7 +11,6 @@ from uuid import UUID
|
||||
from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
|
||||
from api.utils import initialize_prowler_provider
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_TMP_OUTPUT_DIRECTORY
|
||||
from prowler.lib.check.compliance_models import (
|
||||
@@ -27,6 +26,7 @@ from tasks.jobs.reports import (
|
||||
ENSReportGenerator,
|
||||
NIS2ReportGenerator,
|
||||
ThreatScoreReportGenerator,
|
||||
build_provider_metadata,
|
||||
)
|
||||
from tasks.jobs.threatscore import compute_threatscore_metrics
|
||||
from tasks.jobs.threatscore_utils import (
|
||||
@@ -841,24 +841,12 @@ def generate_compliance_reports(
|
||||
tenant_id, scan_id
|
||||
)
|
||||
|
||||
# Initialize the Prowler provider once for the whole report batch. Each
|
||||
# generator used to re-init this in _load_compliance_data, paying the
|
||||
# boto3/Azure-SDK construction cost 5 times per scan. The instance is
|
||||
# only used by FindingOutput.transform_api_finding to enrich findings,
|
||||
# so a single shared instance is correct.
|
||||
logger.info("Initializing prowler_provider once for all reports (scan %s)", scan_id)
|
||||
try:
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
except Exception as init_error:
|
||||
# If init fails the generators will fall back to lazy init in
|
||||
# _load_compliance_data; we just log and continue.
|
||||
logger.warning(
|
||||
"Could not pre-initialize prowler_provider for scan %s: %s",
|
||||
scan_id,
|
||||
init_error,
|
||||
)
|
||||
prowler_provider = None
|
||||
# Build a credential-free provider metadata stub once for the whole
|
||||
# report batch. FindingOutput.transform_api_finding only reads static
|
||||
# attributes (type plus a few identity fields), so reports never decrypt
|
||||
# the ProviderSecret nor construct a cloud SDK session — generation keeps
|
||||
# working after credentials are deleted or invalidated (PROWLER-2145).
|
||||
prowler_provider = build_provider_metadata(provider_obj)
|
||||
|
||||
# Create shared findings cache up front so the eviction closure below
|
||||
# can reference it. Defined BEFORE the closure to avoid the UnboundLocalError
|
||||
|
||||
@@ -98,6 +98,7 @@ from .config import (
|
||||
from .csa import CSAReportGenerator
|
||||
from .ens import ENSReportGenerator
|
||||
from .nis2 import NIS2ReportGenerator
|
||||
from .provider_metadata import build_provider_metadata
|
||||
from .threatscore import ThreatScoreReportGenerator
|
||||
|
||||
__all__ = [
|
||||
@@ -105,6 +106,7 @@ __all__ = [
|
||||
"BaseComplianceReportGenerator",
|
||||
"ComplianceData",
|
||||
"RequirementData",
|
||||
"build_provider_metadata",
|
||||
"create_pdf_styles",
|
||||
"get_requirement_metadata",
|
||||
# Framework-specific generators
|
||||
|
||||
@@ -11,7 +11,6 @@ from typing import Any
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, StatusChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from celery.utils.log import get_task_logger
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
@@ -52,6 +51,7 @@ from .config import (
|
||||
PADDING_SMALL,
|
||||
FrameworkConfig,
|
||||
)
|
||||
from .provider_metadata import build_provider_metadata
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -178,7 +178,8 @@ class ComplianceData:
|
||||
attributes_by_requirement_id: Mapping of requirement IDs to their attributes
|
||||
findings_by_check_id: Mapping of check IDs to their findings
|
||||
provider_obj: Provider model object
|
||||
prowler_provider: Initialized Prowler provider
|
||||
prowler_provider: Credential-free provider metadata stub (see
|
||||
``build_provider_metadata``)
|
||||
"""
|
||||
|
||||
tenant_id: str
|
||||
@@ -439,10 +440,10 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Optional pre-fetched Provider object
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings cache
|
||||
prowler_provider: Optional pre-initialized Prowler provider. When
|
||||
generating multiple reports for the same scan the master
|
||||
function initializes this once and passes it in to avoid
|
||||
re-running boto3/Azure-SDK setup per framework.
|
||||
prowler_provider: Optional provider metadata stub (see
|
||||
``build_provider_metadata``). When generating multiple
|
||||
reports for the same scan the master function builds it
|
||||
once and passes it in.
|
||||
**kwargs: Additional framework-specific arguments
|
||||
"""
|
||||
framework = self.config.display_name
|
||||
@@ -896,9 +897,9 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Optional pre-fetched Provider
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings
|
||||
prowler_provider: Optional pre-initialized Prowler provider. When
|
||||
the master function initializes it once and passes it in,
|
||||
we skip the per-report ``initialize_prowler_provider`` call.
|
||||
prowler_provider: Optional provider metadata stub. When the
|
||||
master function builds it once and passes it in, we skip
|
||||
the per-report ``build_provider_metadata`` call.
|
||||
|
||||
Returns:
|
||||
Aggregated ComplianceData object
|
||||
@@ -909,7 +910,7 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj = Provider.objects.get(id=provider_id)
|
||||
|
||||
if prowler_provider is None:
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
prowler_provider = build_provider_metadata(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework — fall back to the universal loader
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from prowler.providers.github.models import GithubIdentityInfo
|
||||
|
||||
|
||||
def build_provider_metadata(provider) -> SimpleNamespace:
|
||||
"""Build a credential-free stand-in for the Prowler SDK provider.
|
||||
|
||||
``FindingOutput.transform_api_finding`` only reads static attributes
|
||||
from the provider (``type`` plus a few identity/metadata fields used to
|
||||
label accounts), so compliance reports never need the decrypted
|
||||
``ProviderSecret`` nor a live cloud SDK session. This builds an object
|
||||
exposing exactly those attributes from the ``Provider`` DB row, which
|
||||
keeps report generation working when the provider secret has been
|
||||
deleted or its credentials are no longer valid (PROWLER-2145).
|
||||
|
||||
Args:
|
||||
provider: The API ``Provider`` model instance (only ``provider``,
|
||||
``uid`` and ``alias`` are read).
|
||||
|
||||
Returns:
|
||||
A ``SimpleNamespace`` mimicking the SDK provider attributes consumed
|
||||
by ``FindingOutput.transform_api_finding`` / ``generate_output``.
|
||||
"""
|
||||
provider_type = provider.provider
|
||||
uid = provider.uid
|
||||
display_name = provider.alias or uid
|
||||
|
||||
# Defaults cover every attribute read unconditionally in
|
||||
# FindingOutput.generate_output (``provider.auth_method`` is accessed
|
||||
# directly for several provider types); identity lookups go through
|
||||
# get_nested_attribute/getattr, which tolerate missing attributes.
|
||||
stub = SimpleNamespace(
|
||||
type=provider_type,
|
||||
auth_method="",
|
||||
identity=SimpleNamespace(),
|
||||
)
|
||||
|
||||
if provider_type == "aws":
|
||||
stub.identity = SimpleNamespace(account=uid)
|
||||
elif provider_type == "azure":
|
||||
stub.identity = SimpleNamespace(
|
||||
identity_type="",
|
||||
identity_id="",
|
||||
tenant_ids=[""],
|
||||
tenant_domain="",
|
||||
subscriptions={uid: display_name},
|
||||
)
|
||||
elif provider_type == "gcp":
|
||||
stub.identity = SimpleNamespace(profile="")
|
||||
stub.projects = {
|
||||
uid: SimpleNamespace(
|
||||
id=uid,
|
||||
name=display_name,
|
||||
labels={},
|
||||
organization=None,
|
||||
)
|
||||
}
|
||||
elif provider_type == "kubernetes":
|
||||
stub.identity = SimpleNamespace(context=uid, cluster=uid)
|
||||
elif provider_type == "m365":
|
||||
stub.identity = SimpleNamespace(
|
||||
identity_type="",
|
||||
identity_id="",
|
||||
tenant_domain=uid,
|
||||
tenant_id="",
|
||||
)
|
||||
elif provider_type == "github":
|
||||
# generate_output assigns account fields only inside
|
||||
# isinstance(identity, Github*IdentityInfo) branches, so the stub
|
||||
# must carry a real GithubIdentityInfo instance.
|
||||
stub.identity = GithubIdentityInfo(
|
||||
account_id=uid,
|
||||
account_name=display_name,
|
||||
account_url="",
|
||||
)
|
||||
elif provider_type == "mongodbatlas":
|
||||
stub.identity = SimpleNamespace(
|
||||
organization_id=uid,
|
||||
organization_name=display_name,
|
||||
)
|
||||
elif provider_type == "iac":
|
||||
stub.provider_uid = uid
|
||||
elif provider_type == "oraclecloud":
|
||||
stub.identity = SimpleNamespace(
|
||||
tenancy_id=uid,
|
||||
tenancy_name=display_name,
|
||||
)
|
||||
elif provider_type == "alibabacloud":
|
||||
stub.identity = SimpleNamespace(
|
||||
identity_arn="",
|
||||
account_id=uid,
|
||||
account_name=display_name,
|
||||
)
|
||||
elif provider_type == "cloudflare":
|
||||
stub.identity = SimpleNamespace(
|
||||
audited_accounts=[uid],
|
||||
accounts=[],
|
||||
)
|
||||
elif provider_type == "openstack":
|
||||
stub.identity = SimpleNamespace(
|
||||
username="",
|
||||
project_id=uid,
|
||||
project_name=display_name,
|
||||
)
|
||||
elif provider_type == "googleworkspace":
|
||||
stub.identity = SimpleNamespace(
|
||||
delegated_user="",
|
||||
customer_id=uid,
|
||||
domain=display_name,
|
||||
)
|
||||
elif provider_type == "vercel":
|
||||
stub.identity = SimpleNamespace(
|
||||
team=None,
|
||||
user_id=uid,
|
||||
username=display_name,
|
||||
)
|
||||
elif provider_type == "okta":
|
||||
stub.identity = SimpleNamespace(
|
||||
org_domain=uid,
|
||||
client_id="",
|
||||
)
|
||||
|
||||
return stub
|
||||
@@ -1,11 +1,12 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from contextlib import ExitStack
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
@@ -70,6 +71,7 @@ from tasks.jobs.queries import (
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||||
)
|
||||
from tasks.utils import CustomEncoder, batched
|
||||
from uuid6 import uuid7
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -98,6 +100,17 @@ COMPLIANCE_REQUIREMENT_COPY_COLUMNS = (
|
||||
FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=3000)
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres.
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=1000)
|
||||
# Controls how many rows each compliance-overview COPY batch streams to Postgres.
|
||||
# Smaller batches keep each COPY (and its commit/fsync) short so the writer can
|
||||
# checkpoint and autovacuum between flushes, avoiding the FreeableMemory cliff
|
||||
# observed when a single client streams back-to-back multi-second COPY statements.
|
||||
COMPLIANCE_COPY_BATCH_SIZE = env.int("DJANGO_COMPLIANCE_COPY_BATCH_SIZE", default=5000)
|
||||
# Optional pause (seconds) inserted between compliance-overview COPY batches so the
|
||||
# writer gets breathing room for checkpoints/autovacuum under sustained ingest.
|
||||
# Default 0 preserves current behavior; ops can raise it to spread the write load.
|
||||
COMPLIANCE_COPY_THROTTLE_SECONDS = env.float(
|
||||
"DJANGO_COMPLIANCE_COPY_THROTTLE_SECONDS", default=0.0
|
||||
)
|
||||
# Throttle scan progress persistence: minimum progress delta (fraction 0-1)
|
||||
# between two persisted progress updates.
|
||||
PROGRESS_THROTTLE_DELTA = env.float("DJANGO_SCAN_PROGRESS_THROTTLE_DELTA", default=0.01)
|
||||
@@ -306,16 +319,71 @@ def _store_resources(
|
||||
return resource_instance, (resource_instance.uid, resource_instance.region)
|
||||
|
||||
|
||||
def _bulk_update_resource_failed_findings_counts(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
resources_to_update: list[Resource],
|
||||
) -> None:
|
||||
"""Persist failed finding counters with stable row locking and retry."""
|
||||
if not resources_to_update:
|
||||
return
|
||||
|
||||
sorted_resources = sorted(
|
||||
resources_to_update, key=lambda resource: str(resource.id)
|
||||
)
|
||||
for start in range(0, len(sorted_resources), SCAN_DB_BATCH_SIZE):
|
||||
chunk = sorted_resources[start : start + SCAN_DB_BATCH_SIZE]
|
||||
chunk_ids = [resource.id for resource in chunk]
|
||||
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
list(
|
||||
Resource.objects.select_for_update()
|
||||
.filter(id__in=chunk_ids)
|
||||
.order_by("id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
Resource.objects.bulk_update(
|
||||
chunk,
|
||||
["failed_findings_count"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
break
|
||||
except OperationalError:
|
||||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||||
logger.warning(
|
||||
"Resource failed findings count update hit a database "
|
||||
"conflict on scan %s. Retrying chunk %s/%s "
|
||||
"(attempt %s/%s).",
|
||||
scan_id,
|
||||
start // SCAN_DB_BATCH_SIZE + 1,
|
||||
(len(sorted_resources) + SCAN_DB_BATCH_SIZE - 1)
|
||||
// SCAN_DB_BATCH_SIZE,
|
||||
attempt + 1,
|
||||
CELERY_DEADLOCK_ATTEMPTS,
|
||||
)
|
||||
time.sleep((0.1 * (2**attempt)) + random.uniform(0, 0.1))
|
||||
continue
|
||||
raise
|
||||
|
||||
|
||||
def _copy_compliance_requirement_rows(
|
||||
tenant_id: str, rows: list[dict[str, Any]]
|
||||
connection, tenant_id: str, rows: list[dict[str, Any]]
|
||||
) -> None:
|
||||
"""Stream compliance requirement rows into Postgres using COPY.
|
||||
|
||||
We leverage the admin connection (when available) to bypass the COPY + RLS
|
||||
restriction, writing only the fields required by
|
||||
``ComplianceRequirementOverview``.
|
||||
Runs on a caller-supplied admin connection (which bypasses the COPY + RLS
|
||||
restriction), writing only the fields required by
|
||||
``ComplianceRequirementOverview``. Each batch is committed on its own so the
|
||||
writer releases locks and can checkpoint between flushes; the connection
|
||||
itself is reused across batches to avoid per-batch reconnection churn. The
|
||||
tenant GUC is set inside every batch's transaction because
|
||||
``SET_CONFIG_QUERY`` uses ``is_local=TRUE`` (transaction-scoped), so it is
|
||||
cleared on each commit.
|
||||
|
||||
Args:
|
||||
connection: Open admin psycopg connection (``autocommit=False``).
|
||||
tenant_id: Target tenant UUID.
|
||||
rows: List of row dictionaries prepared by
|
||||
:func:`create_compliance_requirements`.
|
||||
@@ -355,34 +423,38 @@ def _copy_compliance_requirement_rows(
|
||||
)
|
||||
|
||||
try:
|
||||
with psycopg_connection(MainRouter.admin_db) as connection:
|
||||
connection.autocommit = False
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||||
cursor.copy_expert(copy_sql, csv_buffer)
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
with connection.cursor() as cursor:
|
||||
# is_local=TRUE ⇒ must be re-applied inside each batch's transaction.
|
||||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||||
cursor.copy_expert(copy_sql, csv_buffer)
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
csv_buffer.close()
|
||||
|
||||
|
||||
def _persist_compliance_requirement_rows(
|
||||
tenant_id: str, rows: Iterable[dict[str, Any]], batch_size: int = 10000
|
||||
tenant_id: str,
|
||||
rows: Iterable[dict[str, Any]],
|
||||
batch_size: int = COMPLIANCE_COPY_BATCH_SIZE,
|
||||
throttle_seconds: float = COMPLIANCE_COPY_THROTTLE_SECONDS,
|
||||
) -> int:
|
||||
"""Persist compliance requirement rows using batched COPY with ORM fallback.
|
||||
|
||||
``rows`` is consumed lazily in batches, so peak memory stays at ~``batch_size``
|
||||
rows instead of the full set. A batch that fails COPY falls back to an ORM
|
||||
``bulk_create`` of just that batch.
|
||||
rows instead of the full set. A single admin connection is reused across all
|
||||
batches (each batch committed separately); an optional pause between batches
|
||||
lets the writer checkpoint/autovacuum under sustained ingest. A batch that
|
||||
fails COPY falls back to an ORM ``bulk_create`` of just that batch.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID.
|
||||
rows: Iterable of row dictionaries reflecting the compliance overview
|
||||
state for a scan.
|
||||
batch_size: Number of rows per COPY batch (default: 10000).
|
||||
batch_size: Number of rows per COPY batch.
|
||||
throttle_seconds: Optional pause between batches (0 disables throttling).
|
||||
|
||||
Returns:
|
||||
int: total number of rows persisted.
|
||||
@@ -390,49 +462,63 @@ def _persist_compliance_requirement_rows(
|
||||
total_rows = 0
|
||||
batch_num = 0
|
||||
|
||||
for batch, _is_last in batched(rows, batch_size):
|
||||
if not batch:
|
||||
continue
|
||||
batch_num += 1
|
||||
try:
|
||||
_copy_compliance_requirement_rows(tenant_id, batch)
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
f"COPY bulk insert for compliance requirements batch {batch_num} "
|
||||
"failed; falling back to ORM bulk_create for this batch",
|
||||
exc_info=error,
|
||||
)
|
||||
fallback_objects = [
|
||||
ComplianceRequirementOverview(
|
||||
id=row["id"],
|
||||
tenant_id=row["tenant_id"],
|
||||
inserted_at=row["inserted_at"],
|
||||
compliance_id=row["compliance_id"],
|
||||
framework=row["framework"],
|
||||
version=row["version"],
|
||||
description=row["description"],
|
||||
region=row["region"],
|
||||
requirement_id=row["requirement_id"],
|
||||
requirement_status=row["requirement_status"],
|
||||
passed_checks=row["passed_checks"],
|
||||
failed_checks=row["failed_checks"],
|
||||
total_checks=row["total_checks"],
|
||||
passed_findings=row.get("passed_findings", 0),
|
||||
total_findings=row.get("total_findings", 0),
|
||||
scan_id=row["scan_id"],
|
||||
)
|
||||
for row in batch
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.bulk_create(
|
||||
fallback_objects, batch_size=500
|
||||
# A single admin connection is opened lazily (only when the first non-empty
|
||||
# batch arrives) and reused for every batch; ``ExitStack`` guarantees it is
|
||||
# closed on exit. A COPY failure rolls back inside
|
||||
# ``_copy_compliance_requirement_rows``, leaving the connection reusable, so
|
||||
# the same connection is kept for the ORM-fallback path and later batches.
|
||||
with ExitStack() as stack:
|
||||
copy_connection = None
|
||||
for batch, _is_last in batched(rows, batch_size):
|
||||
if not batch:
|
||||
continue
|
||||
batch_num += 1
|
||||
if throttle_seconds > 0 and batch_num > 1:
|
||||
time.sleep(throttle_seconds)
|
||||
try:
|
||||
if copy_connection is None:
|
||||
copy_connection = stack.enter_context(
|
||||
psycopg_connection(MainRouter.admin_db)
|
||||
)
|
||||
copy_connection.autocommit = False
|
||||
_copy_compliance_requirement_rows(copy_connection, tenant_id, batch)
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
f"COPY bulk insert for compliance requirements batch {batch_num} "
|
||||
"failed; falling back to ORM bulk_create for this batch",
|
||||
exc_info=error,
|
||||
)
|
||||
fallback_objects = [
|
||||
ComplianceRequirementOverview(
|
||||
id=row["id"],
|
||||
tenant_id=row["tenant_id"],
|
||||
inserted_at=row["inserted_at"],
|
||||
compliance_id=row["compliance_id"],
|
||||
framework=row["framework"],
|
||||
version=row["version"],
|
||||
description=row["description"],
|
||||
region=row["region"],
|
||||
requirement_id=row["requirement_id"],
|
||||
requirement_status=row["requirement_status"],
|
||||
passed_checks=row["passed_checks"],
|
||||
failed_checks=row["failed_checks"],
|
||||
total_checks=row["total_checks"],
|
||||
passed_findings=row.get("passed_findings", 0),
|
||||
total_findings=row.get("total_findings", 0),
|
||||
scan_id=row["scan_id"],
|
||||
)
|
||||
for row in batch
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.bulk_create(
|
||||
fallback_objects, batch_size=500
|
||||
)
|
||||
|
||||
total_rows += len(batch)
|
||||
logger.info(
|
||||
f"Compliance COPY batch {batch_num}: inserted {len(batch)} rows "
|
||||
f"({total_rows} total)"
|
||||
)
|
||||
total_rows += len(batch)
|
||||
logger.info(
|
||||
f"Compliance COPY batch {batch_num}: inserted {len(batch)} rows "
|
||||
f"({total_rows} total)"
|
||||
)
|
||||
|
||||
return total_rows
|
||||
|
||||
@@ -835,15 +921,19 @@ def _process_finding_micro_batch(
|
||||
# Denormalized resource arrays populated directly on insert
|
||||
# (was previously a separate bulk_update; saves a CASE WHEN
|
||||
# over thousands of rows per micro-batch).
|
||||
resource_regions=[resource_instance.region]
|
||||
if resource_instance.region
|
||||
else [],
|
||||
resource_services=[resource_instance.service]
|
||||
if resource_instance.service
|
||||
else [],
|
||||
resource_types=[resource_instance.type]
|
||||
if resource_instance.type
|
||||
else [],
|
||||
resource_regions=(
|
||||
[resource_instance.region]
|
||||
if resource_instance.region
|
||||
else []
|
||||
),
|
||||
resource_services=(
|
||||
[resource_instance.service]
|
||||
if resource_instance.service
|
||||
else []
|
||||
),
|
||||
resource_types=(
|
||||
[resource_instance.type] if resource_instance.type else []
|
||||
),
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append(
|
||||
@@ -1182,16 +1272,11 @@ def perform_prowler_scan(
|
||||
resources_to_update.append(resource_instance)
|
||||
|
||||
if resources_to_update:
|
||||
# Single rls_transaction wrapping the bulk_update (previously
|
||||
# `update_objects_in_batches` opened one rls_transaction per
|
||||
# chunk; for tenants with many resources this collapsed N
|
||||
# BEGINs/COMMITs into 1).
|
||||
with rls_transaction(tenant_id):
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_update,
|
||||
["failed_findings_count"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resources_to_update=resources_to_update,
|
||||
)
|
||||
|
||||
except ProviderDeletedException as e:
|
||||
logger.warning(str(e))
|
||||
@@ -1701,7 +1786,10 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
requirement_statuses[key]["pass_count"] += 1
|
||||
|
||||
yield {
|
||||
"id": uuid.uuid4(),
|
||||
# UUIDv7 so the COPY-inserted rows route to the
|
||||
# dated partitions of compliance_requirements_overviews
|
||||
# (uuid4 would always land in the default partition).
|
||||
"id": uuid7(),
|
||||
"tenant_id": tenant_id_str,
|
||||
"inserted_at": utc_datetime_now,
|
||||
"compliance_id": compliance_id,
|
||||
@@ -1720,8 +1808,15 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
}
|
||||
|
||||
# Idempotent re-run: clear this scan's rows before re-inserting.
|
||||
# First-run scans have no prior rows, so skip the DELETE entirely to
|
||||
# avoid an empty write transaction and needless dead-tuple churn; the
|
||||
# ``.exists()`` probe is an index-only lookup on cro_scan_comp_reg_idx.
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete()
|
||||
existing_rows = ComplianceRequirementOverview.objects.filter(
|
||||
scan_id=scan_id
|
||||
)
|
||||
if existing_rows.exists():
|
||||
existing_rows.delete()
|
||||
|
||||
requirements_created = _persist_compliance_requirement_rows(
|
||||
tenant_id, _iter_compliance_requirement_rows()
|
||||
|
||||
@@ -178,7 +178,9 @@ def _load_findings_for_requirement_checks(
|
||||
tenant_id (str): The tenant ID for Row-Level Security context.
|
||||
scan_id (str): The ID of the scan to retrieve findings for.
|
||||
check_ids (list[str]): List of check IDs to load findings for.
|
||||
prowler_provider: The initialized Prowler provider instance.
|
||||
prowler_provider: Credential-free provider metadata stub (see
|
||||
``tasks.jobs.reports.build_provider_metadata``) consumed by
|
||||
``FindingOutput.transform_api_finding``.
|
||||
findings_cache (dict, optional): Cache of already loaded findings.
|
||||
If provided, checks are first looked up in cache before querying database.
|
||||
total_counts_out (dict, optional): If provided, populated with
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from shutil import rmtree
|
||||
from uuid import uuid4
|
||||
|
||||
from api.compliance import (
|
||||
get_compliance_frameworks,
|
||||
@@ -10,14 +11,24 @@ from api.compliance import (
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import delete_related_daily_task, rls_transaction
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
|
||||
from api.models import (
|
||||
Finding,
|
||||
Integration,
|
||||
Provider,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
Task,
|
||||
)
|
||||
from api.utils import initialize_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
from celery import chain, group, shared_task
|
||||
from celery import chain, group, shared_task, states
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.celery import RLSTask
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
|
||||
from django.db import transaction
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.compliance.compliance import (
|
||||
process_universal_compliance_frameworks,
|
||||
@@ -85,6 +96,220 @@ from tasks.utils import (
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
QUEUED_SCAN_TASK_STATE = "QUEUED"
|
||||
DISPATCHED_SCAN_TASK_STATES = (states.PENDING, states.STARTED, "PROGRESS")
|
||||
|
||||
|
||||
def _get_dispatched_provider_scan(tenant_id: str, provider_id: str):
|
||||
"""Return a scan that has already been dispatched for a provider."""
|
||||
executing_scan = (
|
||||
Scan.objects.select_for_update()
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
.order_by("-inserted_at")
|
||||
.first()
|
||||
)
|
||||
if executing_scan:
|
||||
return executing_scan
|
||||
|
||||
return (
|
||||
Scan.objects.select_for_update(of=("self",))
|
||||
.select_related("task__task_runner_task")
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
state__in=(StateChoices.AVAILABLE, StateChoices.SCHEDULED),
|
||||
task__isnull=False,
|
||||
task__task_runner_task__status__in=DISPATCHED_SCAN_TASK_STATES,
|
||||
)
|
||||
.order_by("-inserted_at")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def _get_queued_provider_scan(tenant_id: str, provider_id: str):
|
||||
"""Return the next DB-queued scan for a provider."""
|
||||
return (
|
||||
Scan.objects.select_for_update(of=("self",))
|
||||
.select_related("task__task_runner_task")
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task__isnull=False,
|
||||
task__task_runner_task__status=QUEUED_SCAN_TASK_STATE,
|
||||
)
|
||||
.order_by("inserted_at", "id")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def get_active_provider_scan(tenant_id: str, provider_id: str):
|
||||
"""Return a dispatched or DB-queued scan for a provider."""
|
||||
return _get_dispatched_provider_scan(
|
||||
tenant_id, provider_id
|
||||
) or _get_queued_provider_scan(tenant_id, provider_id)
|
||||
|
||||
|
||||
def create_scan_task_record(
|
||||
tenant_id: str,
|
||||
task_id: str,
|
||||
task_name: str = "scan-perform",
|
||||
task_status: str | None = states.PENDING,
|
||||
) -> Task:
|
||||
if task_status is None:
|
||||
task_status = states.PENDING
|
||||
|
||||
task_result, _ = TaskResult.objects.update_or_create(
|
||||
task_id=str(task_id),
|
||||
defaults={"status": task_status, "task_name": task_name},
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=str(task_id),
|
||||
tenant_id=tenant_id,
|
||||
defaults={"task_runner_task": task_result},
|
||||
)
|
||||
return prowler_task
|
||||
|
||||
|
||||
def enqueue_scan_execution_on_commit(
|
||||
tenant_id: str,
|
||||
scan: Scan,
|
||||
task_id: str,
|
||||
) -> None:
|
||||
transaction.on_commit(
|
||||
lambda: perform_scan_task.apply_async(
|
||||
kwargs={
|
||||
"tenant_id": str(tenant_id),
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
},
|
||||
task_id=str(task_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _get_queued_scheduled_scan(tenant_id: str, provider_id: str):
|
||||
return (
|
||||
Scan.objects.select_for_update(of=("self",))
|
||||
.select_related("task__task_runner_task")
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task__isnull=False,
|
||||
task__task_runner_task__status=QUEUED_SCAN_TASK_STATE,
|
||||
)
|
||||
.order_by("inserted_at", "id")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def _get_or_create_queued_scheduled_scan(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
periodic_task_instance: PeriodicTask,
|
||||
scheduled_at: datetime,
|
||||
) -> Scan:
|
||||
queued_scan = _get_queued_scheduled_scan(tenant_id, provider_id)
|
||||
if queued_scan:
|
||||
return queued_scan
|
||||
|
||||
task_id = str(uuid4())
|
||||
queued_task = create_scan_task_record(
|
||||
tenant_id=tenant_id,
|
||||
task_id=task_id,
|
||||
task_status=QUEUED_SCAN_TASK_STATE,
|
||||
)
|
||||
return Scan.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
name="Daily scheduled scan",
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduled_at=scheduled_at,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
task=queued_task,
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_next_queued_provider_scan(tenant_id: str, provider_id: str):
|
||||
with rls_transaction(tenant_id):
|
||||
if not Provider.objects.select_for_update().filter(pk=provider_id).exists():
|
||||
return None
|
||||
|
||||
if _get_dispatched_provider_scan(tenant_id, provider_id):
|
||||
return None
|
||||
|
||||
queued_scan = _get_queued_provider_scan(tenant_id, provider_id)
|
||||
if not queued_scan or not queued_scan.task:
|
||||
return None
|
||||
|
||||
task_result = queued_scan.task.task_runner_task
|
||||
task_result.status = states.PENDING
|
||||
task_result.task_name = "scan-perform"
|
||||
task_result.save(update_fields=["status", "task_name"])
|
||||
enqueue_scan_execution_on_commit(
|
||||
tenant_id=tenant_id,
|
||||
scan=queued_scan,
|
||||
task_id=str(queued_scan.task_id),
|
||||
)
|
||||
return queued_scan
|
||||
|
||||
|
||||
def _dispatch_next_queued_provider_scan_best_effort(
|
||||
tenant_id: str, provider_id: str
|
||||
) -> None:
|
||||
try:
|
||||
_dispatch_next_queued_provider_scan(tenant_id, provider_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to dispatch next queued scan for provider %s", provider_id
|
||||
)
|
||||
|
||||
|
||||
def _get_or_create_next_scheduled_scan(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
periodic_task_instance: PeriodicTask,
|
||||
next_scan_datetime: datetime,
|
||||
) -> Scan:
|
||||
interval = periodic_task_instance.interval
|
||||
now = datetime.now(UTC)
|
||||
while next_scan_datetime <= now:
|
||||
next_scan_datetime += timedelta(**{interval.period: interval.every})
|
||||
|
||||
return _get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=next_scan_datetime,
|
||||
update_state=True,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_next_scheduled_scan_best_effort(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
periodic_task_instance: PeriodicTask,
|
||||
next_scan_datetime: datetime,
|
||||
) -> None:
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
_get_or_create_next_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
next_scan_datetime=next_scan_datetime,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to ensure next scheduled scan for provider %s", provider_id
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_orphan_scheduled_scans(
|
||||
@@ -117,6 +342,7 @@ def _cleanup_orphan_scheduled_scans(
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
task__isnull=True,
|
||||
)
|
||||
|
||||
scheduled_scan_exists = Scan.objects.filter(
|
||||
@@ -292,16 +518,17 @@ def perform_scan_task(
|
||||
)
|
||||
return None
|
||||
|
||||
result = perform_prowler_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
provider_id=provider_id,
|
||||
checks_to_execute=checks_to_execute,
|
||||
)
|
||||
|
||||
_perform_scan_complete_tasks(tenant_id, scan_id, provider_id)
|
||||
|
||||
return result
|
||||
try:
|
||||
result = perform_prowler_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
provider_id=provider_id,
|
||||
checks_to_execute=checks_to_execute,
|
||||
)
|
||||
_perform_scan_complete_tasks(tenant_id, scan_id, provider_id)
|
||||
return result
|
||||
finally:
|
||||
_dispatch_next_queued_provider_scan_best_effort(tenant_id, provider_id)
|
||||
|
||||
|
||||
# acks_late=False: like scan-perform; a dropped run is re-fired by Beat on the next tick.
|
||||
@@ -335,7 +562,7 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
task_id = self.request.id
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
if not Provider.objects.filter(pk=provider_id).exists():
|
||||
if not Provider.objects.select_for_update().filter(pk=provider_id).exists():
|
||||
logger.warning(
|
||||
"scheduled scan-perform skipped: provider %s no longer exists "
|
||||
"(tenant=%s)",
|
||||
@@ -348,22 +575,6 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
periodic_task_instance = PeriodicTask.objects.get(
|
||||
name=f"scan-perform-scheduled-{provider_id}"
|
||||
)
|
||||
executing_scan = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
)
|
||||
if executing_scan:
|
||||
logger.warning(
|
||||
f"Scheduled scan already executing for provider {provider_id}. Skipping."
|
||||
)
|
||||
return ScanTaskSerializer(instance=executing_scan).data
|
||||
|
||||
executed_scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
@@ -388,6 +599,26 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
)
|
||||
|
||||
active_scan = get_active_provider_scan(tenant_id, provider_id)
|
||||
if active_scan:
|
||||
logger.warning(
|
||||
"Scan already queued or executing for provider %s. Queueing scheduled run.",
|
||||
provider_id,
|
||||
)
|
||||
queued_scheduled_scan = _get_or_create_queued_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
scheduled_at=current_scan_datetime,
|
||||
)
|
||||
_get_or_create_next_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
next_scan_datetime=next_scan_datetime,
|
||||
)
|
||||
return ScanTaskSerializer(instance=queued_scheduled_scan).data
|
||||
|
||||
scan_instance = _get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
@@ -403,24 +634,16 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scan_id=str(scan_instance.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
|
||||
return result
|
||||
finally:
|
||||
with rls_transaction(tenant_id):
|
||||
now = datetime.now(UTC)
|
||||
if next_scan_datetime <= now:
|
||||
interval_delta = timedelta(**{interval.period: interval.every})
|
||||
while next_scan_datetime <= now:
|
||||
next_scan_datetime += interval_delta
|
||||
_get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=next_scan_datetime,
|
||||
update_state=True,
|
||||
)
|
||||
|
||||
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
|
||||
|
||||
return result
|
||||
_ensure_next_scheduled_scan_best_effort(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
next_scan_datetime=next_scan_datetime,
|
||||
)
|
||||
_dispatch_next_queued_provider_scan_best_effort(tenant_id, provider_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-summary", queue="overview")
|
||||
|
||||
@@ -2,8 +2,10 @@ from contextlib import nullcontext
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
@@ -15,6 +17,7 @@ from api.models import (
|
||||
StatusChoices,
|
||||
Task,
|
||||
)
|
||||
from django.db import DEFAULT_DB_ALIAS
|
||||
from django_celery_results.models import TaskResult
|
||||
from prowler.lib.check.models import Severity
|
||||
from tasks.jobs.attack_paths import findings as findings_module
|
||||
@@ -2244,6 +2247,58 @@ class TestInternetAnalysis:
|
||||
class TestAttackPathsDbUtilsGraphDataReady:
|
||||
"""Tests for db_utils functions related to graph_data_ready lifecycle."""
|
||||
|
||||
def test_database_defaults_allow_legacy_insert_without_cutover_columns(
|
||||
self, tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan_id = uuid4()
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
with rls_transaction(str(tenant.id), using=DEFAULT_DB_ALIAS) as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO attack_paths_scans (
|
||||
id,
|
||||
inserted_at,
|
||||
updated_at,
|
||||
state,
|
||||
progress,
|
||||
graph_data_ready,
|
||||
started_at,
|
||||
tenant_id,
|
||||
provider_id,
|
||||
scan_id
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
[
|
||||
attack_paths_scan_id,
|
||||
now,
|
||||
now,
|
||||
StateChoices.SCHEDULED,
|
||||
0,
|
||||
False,
|
||||
now,
|
||||
tenant.id,
|
||||
provider.id,
|
||||
scan.id,
|
||||
],
|
||||
)
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.get(id=attack_paths_scan_id)
|
||||
|
||||
assert attack_paths_scan.is_migrated is False
|
||||
assert (
|
||||
attack_paths_scan.sink_backend == AttackPathsScan.SinkBackendChoices.NEO4J
|
||||
)
|
||||
|
||||
def test_create_attack_paths_scan_first_scan_defaults_to_false(
|
||||
self, tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
|
||||
@@ -1044,10 +1044,10 @@ class TestStaleCleanupProtectionHelpers:
|
||||
class TestGenerateThreatscoreReportFunction:
|
||||
"""Test suite for generate_threatscore_report function."""
|
||||
|
||||
@patch("tasks.jobs.reports.base.initialize_prowler_provider")
|
||||
@patch("tasks.jobs.reports.base.build_provider_metadata")
|
||||
def test_generate_threatscore_report_exception_handling(
|
||||
self,
|
||||
mock_initialize_provider,
|
||||
mock_build_provider_metadata,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
providers_fixture,
|
||||
@@ -1057,7 +1057,7 @@ class TestGenerateThreatscoreReportFunction:
|
||||
scan = scans_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
|
||||
mock_initialize_provider.side_effect = Exception("Test exception")
|
||||
mock_build_provider_metadata.side_effect = Exception("Test exception")
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
generate_threatscore_report(
|
||||
@@ -1167,7 +1167,6 @@ class TestGenerateComplianceReportsOptimized:
|
||||
assert result["cis"] == {"upload": False, "path": ""}
|
||||
mock_cis.assert_not_called()
|
||||
|
||||
@patch("api.utils.initialize_prowler_provider")
|
||||
@patch("tasks.jobs.report.rmtree")
|
||||
@patch("tasks.jobs.report._upload_to_s3")
|
||||
@patch("tasks.jobs.report.generate_cis_report")
|
||||
@@ -1194,7 +1193,6 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_cis,
|
||||
mock_upload_to_s3,
|
||||
mock_rmtree,
|
||||
mock_init_provider,
|
||||
):
|
||||
"""After each framework finishes, exclusive entries are evicted.
|
||||
|
||||
@@ -1223,7 +1221,6 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_aggregate_stats.return_value = {}
|
||||
mock_generate_output_dir.return_value = "/tmp/tenant/scan/x/prowler-out"
|
||||
mock_upload_to_s3.return_value = "s3://bucket/tenant/scan/x/report.pdf"
|
||||
mock_init_provider.return_value = Mock(name="prowler_provider")
|
||||
|
||||
# Seed the cache as if both frameworks had already loaded their
|
||||
# findings. We mutate it indirectly: each generator wrapper is a
|
||||
@@ -1266,7 +1263,7 @@ class TestGenerateComplianceReportsOptimized:
|
||||
"shared must remain in cache because ENS still needs it"
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.report.initialize_prowler_provider")
|
||||
@patch("tasks.jobs.report.build_provider_metadata")
|
||||
@patch("tasks.jobs.report.rmtree")
|
||||
@patch("tasks.jobs.report._upload_to_s3")
|
||||
@patch("tasks.jobs.report.generate_cis_report")
|
||||
@@ -1279,7 +1276,7 @@ class TestGenerateComplianceReportsOptimized:
|
||||
@patch("tasks.jobs.report.Compliance.get_bulk")
|
||||
@patch("tasks.jobs.report.Provider.objects.get")
|
||||
@patch("tasks.jobs.report.ScanSummary.objects.filter")
|
||||
def test_prowler_provider_initialized_once(
|
||||
def test_provider_metadata_built_once(
|
||||
self,
|
||||
mock_scan_summary_filter,
|
||||
mock_provider_get,
|
||||
@@ -1293,11 +1290,11 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_cis,
|
||||
mock_upload_to_s3,
|
||||
mock_rmtree,
|
||||
mock_init_provider,
|
||||
mock_build_metadata,
|
||||
):
|
||||
"""``initialize_prowler_provider`` must be called exactly once for
|
||||
the whole batch (PROWLER-1733). Previously each generator re-init'd
|
||||
the SDK provider in ``_load_compliance_data`` → 5 inits per scan.
|
||||
"""``build_provider_metadata`` must be called exactly once for the
|
||||
whole batch and its result shared across all 5 reports
|
||||
(PROWLER-1733 / PROWLER-2145).
|
||||
"""
|
||||
mock_scan_summary_filter.return_value.exists.return_value = True
|
||||
mock_provider_get.return_value = Mock(uid="provider-uid", provider="aws")
|
||||
@@ -1306,7 +1303,7 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_aggregate_stats.return_value = {}
|
||||
mock_generate_output_dir.return_value = "/tmp/tenant/scan/x/prowler-out"
|
||||
mock_upload_to_s3.return_value = "s3://bucket/tenant/scan/x/report.pdf"
|
||||
mock_init_provider.return_value = Mock(name="prowler_provider")
|
||||
mock_build_metadata.return_value = Mock(name="prowler_provider")
|
||||
|
||||
generate_compliance_reports(
|
||||
tenant_id=str(uuid.uuid4()),
|
||||
@@ -1325,14 +1322,14 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_nis2.assert_called_once()
|
||||
mock_csa.assert_called_once()
|
||||
mock_cis.assert_called_once()
|
||||
# …but the SDK provider was initialized only once.
|
||||
assert mock_init_provider.call_count == 1, (
|
||||
f"expected 1 init, got {mock_init_provider.call_count} "
|
||||
# …but the provider metadata stub was built only once.
|
||||
assert mock_build_metadata.call_count == 1, (
|
||||
f"expected 1 build, got {mock_build_metadata.call_count} "
|
||||
f"(prowler_provider must be shared across reports)"
|
||||
)
|
||||
|
||||
# The shared instance must reach every wrapper as kwargs.
|
||||
shared = mock_init_provider.return_value
|
||||
shared = mock_build_metadata.return_value
|
||||
for mock_wrapper in (
|
||||
mock_threatscore,
|
||||
mock_ens,
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Tests for the credential-free provider metadata stub (PROWLER-2145).
|
||||
|
||||
Every provider object used here is a plain ``SimpleNamespace`` WITHOUT a
|
||||
``secret`` attribute: any code path trying to read ``provider.secret`` (the
|
||||
coupling these tests guard against) would raise ``AttributeError`` and fail
|
||||
the test. No database is required.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from api.models import Provider
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
from prowler.providers.github.models import GithubIdentityInfo
|
||||
from tasks.jobs.reports import build_provider_metadata
|
||||
|
||||
PROVIDER_UID = "provider-uid-123"
|
||||
PROVIDER_ALIAS = "my-provider-alias"
|
||||
|
||||
|
||||
def _provider_row(provider_type: str, alias: str | None = PROVIDER_ALIAS):
|
||||
"""Mimic the Provider DB row attributes read by build_provider_metadata."""
|
||||
return SimpleNamespace(provider=provider_type, uid=PROVIDER_UID, alias=alias)
|
||||
|
||||
|
||||
class TestBuildProviderMetadata:
|
||||
@pytest.mark.parametrize("provider_type", Provider.ProviderChoices.values)
|
||||
def test_every_provider_type_gets_safe_defaults(self, provider_type):
|
||||
stub = build_provider_metadata(_provider_row(provider_type))
|
||||
|
||||
assert stub.type == provider_type
|
||||
assert isinstance(stub.auth_method, str)
|
||||
assert hasattr(stub, "identity")
|
||||
|
||||
def test_aws_identity_account_is_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("aws"))
|
||||
assert stub.identity.account == PROVIDER_UID
|
||||
|
||||
def test_azure_identity_covers_generate_output_accesses(self):
|
||||
stub = build_provider_metadata(_provider_row("azure"))
|
||||
|
||||
# generate_output indexes tenant_ids[0] and reads these directly.
|
||||
assert stub.identity.tenant_ids
|
||||
assert stub.identity.identity_type == ""
|
||||
assert stub.identity.identity_id == ""
|
||||
assert stub.identity.subscriptions == {PROVIDER_UID: PROVIDER_ALIAS}
|
||||
|
||||
def test_gcp_projects_keyed_by_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("gcp"))
|
||||
|
||||
project = stub.projects[PROVIDER_UID]
|
||||
assert project.id == PROVIDER_UID
|
||||
assert project.name == PROVIDER_ALIAS
|
||||
assert project.labels == {}
|
||||
# generate_output calls getattr(project, "organization") without a
|
||||
# default, so the attribute must exist (None skips the org branch).
|
||||
assert project.organization is None
|
||||
|
||||
def test_kubernetes_identity_context_and_cluster(self):
|
||||
stub = build_provider_metadata(_provider_row("kubernetes"))
|
||||
assert stub.identity.context == PROVIDER_UID
|
||||
assert stub.identity.cluster == PROVIDER_UID
|
||||
|
||||
def test_github_identity_is_real_identity_info(self):
|
||||
# generate_output only assigns account fields inside
|
||||
# isinstance(identity, Github*IdentityInfo) branches.
|
||||
stub = build_provider_metadata(_provider_row("github"))
|
||||
assert isinstance(stub.identity, GithubIdentityInfo)
|
||||
assert stub.identity.account_id == PROVIDER_UID
|
||||
assert stub.identity.account_name == PROVIDER_ALIAS
|
||||
|
||||
def test_iac_provider_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("iac"))
|
||||
assert stub.provider_uid == PROVIDER_UID
|
||||
|
||||
def test_alias_falls_back_to_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("azure", alias=None))
|
||||
assert stub.identity.subscriptions == {PROVIDER_UID: PROVIDER_UID}
|
||||
|
||||
|
||||
def _check_metadata_dict(provider_type: str, check_id: str) -> dict:
|
||||
return {
|
||||
"provider": provider_type,
|
||||
"checkid": check_id,
|
||||
"checktitle": "Test check title",
|
||||
"checktype": [],
|
||||
# CheckMetadata validates ServiceName == check_id.split("_")[0]
|
||||
"servicename": check_id.split("_")[0],
|
||||
"subservicename": "",
|
||||
"severity": "high",
|
||||
"resourcetype": "resource-type",
|
||||
"description": "",
|
||||
"risk": "",
|
||||
"relatedurl": "",
|
||||
"remediation": {
|
||||
"recommendation": {"text": "", "url": ""},
|
||||
"code": {"nativeiac": "", "terraform": "", "cli": "", "other": ""},
|
||||
},
|
||||
"resourceidtemplate": "",
|
||||
"categories": [],
|
||||
"dependson": [],
|
||||
"relatedto": [],
|
||||
"notes": "",
|
||||
}
|
||||
|
||||
|
||||
class _FakeFinding:
|
||||
"""Attribute-faithful Finding stand-in.
|
||||
|
||||
A plain object instead of ``Mock``: only the attributes the Django model
|
||||
exposes exist, so any new provider-attribute read in generate_output
|
||||
(e.g. cloudflare's ``getattr(finding, "account_id", ...)``) hits the
|
||||
same missing-attribute path it would hit in production instead of being
|
||||
masked by Mock auto-created attributes.
|
||||
"""
|
||||
|
||||
|
||||
def _finding_model(provider_type: str, check_id: str, region: str):
|
||||
"""Mimic the Django Finding row attributes read by transform_api_finding."""
|
||||
resource = SimpleNamespace(
|
||||
uid="resource-uid",
|
||||
name="resource-name",
|
||||
metadata="{}",
|
||||
details="",
|
||||
region=region,
|
||||
tags=SimpleNamespace(all=lambda: []),
|
||||
)
|
||||
finding = _FakeFinding()
|
||||
finding.resources = SimpleNamespace(first=lambda: resource)
|
||||
finding.check_metadata = _check_metadata_dict(provider_type, check_id)
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "failed for testing"
|
||||
finding.muted = False
|
||||
return finding
|
||||
|
||||
|
||||
_FINDING_REGION = "region-x"
|
||||
|
||||
# Expected (account_uid, region) of the transformed finding per provider
|
||||
# type, with resource.region = _FINDING_REGION. Keyed by every
|
||||
# Provider.ProviderChoices value so that adding a new provider type without
|
||||
# extending build_provider_metadata (and this table) fails the test below
|
||||
# instead of breaking PDF generation at runtime.
|
||||
_EXPECTED_TRANSFORM = {
|
||||
"aws": (PROVIDER_UID, _FINDING_REGION),
|
||||
"azure": (PROVIDER_UID, _FINDING_REGION),
|
||||
"gcp": (PROVIDER_UID, _FINDING_REGION),
|
||||
# transform_api_finding strips the "namespace: " prefix and
|
||||
# generate_output re-adds it.
|
||||
"kubernetes": (PROVIDER_UID, f"namespace: {_FINDING_REGION}"),
|
||||
"m365": (PROVIDER_UID, _FINDING_REGION),
|
||||
# For GitHub the owner comes from resource.region.
|
||||
"github": (_FINDING_REGION, _FINDING_REGION),
|
||||
"mongodbatlas": (PROVIDER_UID, _FINDING_REGION),
|
||||
"iac": (PROVIDER_UID, _FINDING_REGION),
|
||||
"oraclecloud": (PROVIDER_UID, _FINDING_REGION),
|
||||
"alibabacloud": (PROVIDER_UID, _FINDING_REGION),
|
||||
# Cloudflare uses the zone name (falls back to resource.name) as region.
|
||||
"cloudflare": (PROVIDER_UID, "resource-name"),
|
||||
"openstack": (PROVIDER_UID, _FINDING_REGION),
|
||||
"image": ("image", _FINDING_REGION),
|
||||
"googleworkspace": (PROVIDER_UID, _FINDING_REGION),
|
||||
"vercel": (PROVIDER_UID, "global"),
|
||||
"okta": (PROVIDER_UID, "global"),
|
||||
}
|
||||
|
||||
|
||||
class TestTransformApiFindingWithMetadataStub:
|
||||
"""transform_api_finding must work end-to-end with the stub — i.e.
|
||||
without a credentialed SDK provider — for EVERY API provider type."""
|
||||
|
||||
@pytest.mark.parametrize("provider_type", Provider.ProviderChoices.values)
|
||||
def test_transform_with_stub(self, provider_type):
|
||||
assert provider_type in _EXPECTED_TRANSFORM, (
|
||||
f"New provider type {provider_type!r}: add a branch to "
|
||||
f"build_provider_metadata covering the attributes read by "
|
||||
f"FindingOutput.generate_output, then add its expected "
|
||||
f"(account_uid, region) here."
|
||||
)
|
||||
expected_account_uid, expected_region = _EXPECTED_TRANSFORM[provider_type]
|
||||
|
||||
stub = build_provider_metadata(_provider_row(provider_type))
|
||||
check_id = f"{provider_type}_test_check"
|
||||
finding_model = _finding_model(provider_type, check_id, _FINDING_REGION)
|
||||
|
||||
output = FindingOutput.transform_api_finding(finding_model, stub)
|
||||
|
||||
assert output.check_id == check_id
|
||||
assert output.status == "FAIL"
|
||||
assert output.account_uid == expected_account_uid
|
||||
assert output.region == expected_region
|
||||
assert output.resource_name
|
||||
assert output.resource_uid
|
||||
@@ -21,11 +21,13 @@ from api.models import (
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
)
|
||||
from django.db import IntegrityError, OperationalError
|
||||
from prowler.lib.check.models import Severity
|
||||
from prowler.lib.outputs.finding import Status
|
||||
from tasks.jobs.scan import (
|
||||
_ATTACK_SURFACE_MAPPING_CACHE,
|
||||
_aggregate_findings_by_region,
|
||||
_bulk_update_resource_failed_findings_counts,
|
||||
_copy_compliance_requirement_rows,
|
||||
_create_compliance_summaries,
|
||||
_create_finding_delta,
|
||||
@@ -858,6 +860,98 @@ class TestPerformScan:
|
||||
# Assert that failed_findings_count was reset to 0 during the scan
|
||||
assert resource.failed_findings_count == 0
|
||||
|
||||
def test_failed_findings_count_update_retries_deadlock_in_stable_order(
|
||||
self, resources_fixture, monkeypatch
|
||||
):
|
||||
resource1, resource2, _ = resources_fixture
|
||||
tenant_id = str(resource1.tenant_id)
|
||||
resource1.failed_findings_count = 2
|
||||
resource2.failed_findings_count = 3
|
||||
resources_to_update = [resource2, resource1]
|
||||
expected_order = [
|
||||
str(resource.id)
|
||||
for resource in sorted(resources_to_update, key=lambda item: str(item.id))
|
||||
]
|
||||
original_bulk_update = Resource.objects.bulk_update
|
||||
bulk_update_calls = []
|
||||
|
||||
def flaky_bulk_update(objects, fields, batch_size=None):
|
||||
bulk_update_calls.append([str(obj.id) for obj in objects])
|
||||
if len(bulk_update_calls) == 1:
|
||||
raise OperationalError("deadlock detected")
|
||||
return original_bulk_update(objects, fields, batch_size=batch_size)
|
||||
|
||||
monkeypatch.setattr("tasks.jobs.scan.SCAN_DB_BATCH_SIZE", 10)
|
||||
monkeypatch.setattr(Resource.objects, "bulk_update", flaky_bulk_update)
|
||||
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=tenant_id,
|
||||
scan_id="scan-id",
|
||||
resources_to_update=resources_to_update,
|
||||
)
|
||||
|
||||
resource1.refresh_from_db()
|
||||
resource2.refresh_from_db()
|
||||
assert resource1.failed_findings_count == 2
|
||||
assert resource2.failed_findings_count == 3
|
||||
assert bulk_update_calls == [expected_order, expected_order]
|
||||
|
||||
def test_failed_findings_count_update_does_not_retry_integrity_error(
|
||||
self, resources_fixture, monkeypatch
|
||||
):
|
||||
resource, *_ = resources_fixture
|
||||
resource.failed_findings_count = 2
|
||||
bulk_update_calls = []
|
||||
sleep_calls = []
|
||||
|
||||
def failing_bulk_update(objects, fields, batch_size=None):
|
||||
bulk_update_calls.append([str(obj.id) for obj in objects])
|
||||
raise IntegrityError("constraint violation")
|
||||
|
||||
monkeypatch.setattr(Resource.objects, "bulk_update", failing_bulk_update)
|
||||
monkeypatch.setattr("tasks.jobs.scan.time.sleep", sleep_calls.append)
|
||||
|
||||
with pytest.raises(IntegrityError, match="constraint violation"):
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=str(resource.tenant_id),
|
||||
scan_id="scan-id",
|
||||
resources_to_update=[resource],
|
||||
)
|
||||
|
||||
assert len(bulk_update_calls) == 1
|
||||
assert sleep_calls == []
|
||||
|
||||
def test_failed_findings_count_update_adds_jitter_to_retry_backoff(
|
||||
self, resources_fixture, monkeypatch
|
||||
):
|
||||
from tasks.jobs import scan as scan_jobs
|
||||
|
||||
resource, *_ = resources_fixture
|
||||
resource.failed_findings_count = 2
|
||||
bulk_update_calls = []
|
||||
sleep_calls = []
|
||||
original_bulk_update = Resource.objects.bulk_update
|
||||
|
||||
def flaky_bulk_update(objects, fields, batch_size=None):
|
||||
bulk_update_calls.append([str(obj.id) for obj in objects])
|
||||
if len(bulk_update_calls) == 1:
|
||||
raise OperationalError("deadlock detected")
|
||||
return original_bulk_update(objects, fields, batch_size=batch_size)
|
||||
|
||||
monkeypatch.setattr(Resource.objects, "bulk_update", flaky_bulk_update)
|
||||
monkeypatch.setattr(scan_jobs, "random", MagicMock())
|
||||
scan_jobs.random.uniform.return_value = 0.037
|
||||
monkeypatch.setattr("tasks.jobs.scan.time.sleep", sleep_calls.append)
|
||||
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=str(resource.tenant_id),
|
||||
scan_id="scan-id",
|
||||
resources_to_update=[resource],
|
||||
)
|
||||
|
||||
scan_jobs.random.uniform.assert_called_once_with(0, 0.1)
|
||||
assert sleep_calls == [0.137]
|
||||
|
||||
def test_perform_prowler_scan_with_active_mute_rules(
|
||||
self,
|
||||
tenants_fixture,
|
||||
@@ -2315,25 +2409,13 @@ class TestCreateComplianceRequirements:
|
||||
|
||||
|
||||
class TestComplianceRequirementCopy:
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_streams_csv(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
def test_copy_compliance_requirement_rows_streams_csv(self):
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -2360,9 +2442,8 @@ class TestComplianceRequirementCopy:
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(connection, str(row["tenant_id"]), [row])
|
||||
|
||||
mock_psycopg_connection.assert_called_once_with("admin")
|
||||
connection.cursor.assert_called_once()
|
||||
cursor.execute.assert_called_once()
|
||||
cursor.copy_expert.assert_called_once()
|
||||
@@ -2372,6 +2453,7 @@ class TestComplianceRequirementCopy:
|
||||
assert csv_rows[0][5] == ""
|
||||
assert csv_rows[0][-1] == str(row["scan_id"])
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
@@ -2379,7 +2461,7 @@ class TestComplianceRequirementCopy:
|
||||
side_effect=Exception("copy failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_psycopg_connection
|
||||
):
|
||||
inserted_at = datetime.now(UTC)
|
||||
row = {
|
||||
@@ -2401,6 +2483,10 @@ class TestComplianceRequirementCopy:
|
||||
|
||||
tenant_id = row["tenant_id"]
|
||||
|
||||
conn = MagicMock()
|
||||
mock_psycopg_connection.return_value.__enter__.return_value = conn
|
||||
mock_psycopg_connection.return_value.__exit__.return_value = False
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
@@ -2408,7 +2494,7 @@ class TestComplianceRequirementCopy:
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, [row])
|
||||
|
||||
mock_copy.assert_called_once_with(tenant_id, [row])
|
||||
mock_copy.assert_called_once_with(conn, tenant_id, [row])
|
||||
mock_rls_transaction.assert_called_once_with(tenant_id)
|
||||
mock_bulk_create.assert_called_once()
|
||||
|
||||
@@ -2431,26 +2517,14 @@ class TestComplianceRequirementCopy:
|
||||
mock_rls_transaction.assert_not_called()
|
||||
mock_bulk_create.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_multiple_rows(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
def test_copy_compliance_requirement_rows_multiple_rows(self):
|
||||
"""Test COPY with multiple rows to ensure batch processing works correctly."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -2516,9 +2590,8 @@ class TestComplianceRequirementCopy:
|
||||
]
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(tenant_id, rows)
|
||||
_copy_compliance_requirement_rows(connection, tenant_id, rows)
|
||||
|
||||
mock_psycopg_connection.assert_called_once_with("admin")
|
||||
connection.cursor.assert_called_once()
|
||||
cursor.execute.assert_called_once()
|
||||
cursor.copy_expert.assert_called_once()
|
||||
@@ -2550,26 +2623,14 @@ class TestComplianceRequirementCopy:
|
||||
assert csv_rows[2][5] == "2.0"
|
||||
assert csv_rows[2][9] == "MANUAL"
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_null_values(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
def test_copy_compliance_requirement_rows_null_values(self):
|
||||
"""Test COPY handles NULL/None values correctly in nullable fields."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -2597,7 +2658,7 @@ class TestComplianceRequirementCopy:
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(connection, str(row["tenant_id"]), [row])
|
||||
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
assert len(csv_rows) == 1
|
||||
@@ -2606,26 +2667,14 @@ class TestComplianceRequirementCopy:
|
||||
assert csv_rows[0][5] == "" # version
|
||||
assert csv_rows[0][6] == "" # description
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_special_characters(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
def test_copy_compliance_requirement_rows_special_characters(self):
|
||||
"""Test COPY correctly escapes special characters in CSV."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -2653,7 +2702,7 @@ class TestComplianceRequirementCopy:
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(connection, str(row["tenant_id"]), [row])
|
||||
|
||||
# Verify CSV was generated (csv module handles escaping automatically)
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
@@ -2665,26 +2714,14 @@ class TestComplianceRequirementCopy:
|
||||
assert "quotes" in csv_rows[0][6]
|
||||
assert "commas" in csv_rows[0][6]
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_missing_inserted_at(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
def test_copy_compliance_requirement_rows_missing_inserted_at(self):
|
||||
"""Test COPY uses current datetime when inserted_at is missing."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -2714,7 +2751,7 @@ class TestComplianceRequirementCopy:
|
||||
|
||||
before_call = datetime.now(UTC)
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(connection, str(row["tenant_id"]), [row])
|
||||
after_call = datetime.now(UTC)
|
||||
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
@@ -2725,26 +2762,14 @@ class TestComplianceRequirementCopy:
|
||||
inserted_at = datetime.fromisoformat(inserted_at_str)
|
||||
assert before_call <= inserted_at <= after_call
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_transaction_rollback_on_copy_error(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
def test_copy_compliance_requirement_rows_transaction_rollback_on_copy_error(self):
|
||||
"""Test transaction is rolled back when copy_expert fails."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
# Simulate copy_expert failure
|
||||
cursor.copy_expert.side_effect = Exception("COPY command failed")
|
||||
@@ -2767,32 +2792,24 @@ class TestComplianceRequirementCopy:
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
with pytest.raises(Exception, match="COPY command failed"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(
|
||||
connection, str(row["tenant_id"]), [row]
|
||||
)
|
||||
|
||||
# Verify rollback was called
|
||||
connection.rollback.assert_called_once()
|
||||
connection.commit.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_transaction_rollback_on_set_config_error(
|
||||
self, mock_psycopg_connection, settings
|
||||
self,
|
||||
):
|
||||
"""Test transaction is rolled back when SET_CONFIG fails."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
# Simulate cursor.execute failure
|
||||
cursor.execute.side_effect = Exception("SET prowler.tenant_id failed")
|
||||
@@ -2815,32 +2832,22 @@ class TestComplianceRequirementCopy:
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
with pytest.raises(Exception, match="SET prowler.tenant_id failed"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(
|
||||
connection, str(row["tenant_id"]), [row]
|
||||
)
|
||||
|
||||
# Verify rollback was called
|
||||
connection.rollback.assert_called_once()
|
||||
connection.commit.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_commit_on_success(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
def test_copy_compliance_requirement_rows_commit_on_success(self):
|
||||
"""Test transaction is committed on successful COPY."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
cursor.copy_expert.return_value = None # Success
|
||||
|
||||
@@ -2861,19 +2868,24 @@ class TestComplianceRequirementCopy:
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
_copy_compliance_requirement_rows(connection, str(row["tenant_id"]), [row])
|
||||
|
||||
# Verify commit was called and rollback was not
|
||||
connection.commit.assert_called_once()
|
||||
connection.rollback.assert_not_called()
|
||||
# Verify autocommit was disabled
|
||||
assert connection.autocommit is False
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
@patch("tasks.jobs.scan._copy_compliance_requirement_rows")
|
||||
def test_persist_compliance_requirement_rows_success(self, mock_copy):
|
||||
def test_persist_compliance_requirement_rows_success(
|
||||
self, mock_copy, mock_psycopg_connection
|
||||
):
|
||||
"""Test successful COPY path without fallback to ORM."""
|
||||
mock_copy.return_value = None # Success, no exception
|
||||
|
||||
conn = MagicMock()
|
||||
mock_psycopg_connection.return_value.__enter__.return_value = conn
|
||||
mock_psycopg_connection.return_value.__exit__.return_value = False
|
||||
|
||||
tenant_id = str(uuid.uuid4())
|
||||
rows = [
|
||||
{
|
||||
@@ -2897,8 +2909,9 @@ class TestComplianceRequirementCopy:
|
||||
_persist_compliance_requirement_rows(tenant_id, rows)
|
||||
|
||||
# Verify COPY was called
|
||||
mock_copy.assert_called_once_with(tenant_id, rows)
|
||||
mock_copy.assert_called_once_with(conn, tenant_id, rows)
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
@patch("tasks.jobs.scan.logger")
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@@ -2907,7 +2920,12 @@ class TestComplianceRequirementCopy:
|
||||
side_effect=Exception("COPY failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback_logging(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_logger
|
||||
self,
|
||||
mock_copy,
|
||||
mock_rls_transaction,
|
||||
mock_bulk_create,
|
||||
mock_logger,
|
||||
mock_psycopg_connection,
|
||||
):
|
||||
"""Test logger.exception is called when COPY fails and fallback occurs."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
@@ -2942,6 +2960,7 @@ class TestComplianceRequirementCopy:
|
||||
assert "falling back to ORM" in args[0]
|
||||
assert kwargs.get("exc_info") is not None
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
@@ -2949,7 +2968,7 @@ class TestComplianceRequirementCopy:
|
||||
side_effect=Exception("copy failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback_multiple_rows(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_psycopg_connection
|
||||
):
|
||||
"""Test ORM fallback with multiple rows."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
@@ -2991,6 +3010,10 @@ class TestComplianceRequirementCopy:
|
||||
},
|
||||
]
|
||||
|
||||
conn = MagicMock()
|
||||
mock_psycopg_connection.return_value.__enter__.return_value = conn
|
||||
mock_psycopg_connection.return_value.__exit__.return_value = False
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
@@ -2998,7 +3021,7 @@ class TestComplianceRequirementCopy:
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, rows)
|
||||
|
||||
mock_copy.assert_called_once_with(tenant_id, rows)
|
||||
mock_copy.assert_called_once_with(conn, tenant_id, rows)
|
||||
mock_rls_transaction.assert_called_once_with(tenant_id)
|
||||
mock_bulk_create.assert_called_once()
|
||||
|
||||
@@ -3023,6 +3046,7 @@ class TestComplianceRequirementCopy:
|
||||
assert objects[1].passed_checks == 2
|
||||
assert objects[1].failed_checks == 3
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
@@ -3030,7 +3054,7 @@ class TestComplianceRequirementCopy:
|
||||
side_effect=Exception("copy failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback_all_fields(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_psycopg_connection
|
||||
):
|
||||
"""Test ORM fallback correctly maps all fields from row dict to model."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
|
||||
@@ -14,6 +14,7 @@ from api.models import (
|
||||
Task,
|
||||
)
|
||||
from botocore.exceptions import ClientError
|
||||
from celery import states
|
||||
from django_celery_beat.models import IntervalSchedule, PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from tasks.jobs.lighthouse_providers import (
|
||||
@@ -2286,6 +2287,51 @@ class TestCleanupOrphanScheduledScans:
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
assert Scan.objects.filter(id=available_scan_other_task.id).exists()
|
||||
|
||||
def test_cleanup_keeps_db_queued_scheduled_scans(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""DB-queued scheduled scans have a task and must not be deleted as orphans."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status="QUEUED",
|
||||
)
|
||||
queued_task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
task_runner_task=task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
queued_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Queued scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
task=queued_task,
|
||||
)
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
assert deleted_count == 0
|
||||
assert Scan.objects.filter(id=queued_scan.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestPerformScheduledScanTask:
|
||||
@@ -2334,10 +2380,10 @@ class TestPerformScheduledScanTask:
|
||||
)
|
||||
return task_result
|
||||
|
||||
def test_skip_when_scheduled_scan_executing(
|
||||
def test_queues_scheduled_scan_when_scheduled_scan_is_executing(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Skip a scheduled run when another scheduled scan is already executing."""
|
||||
"""Queue a scheduled run when another scheduled scan is executing."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
@@ -2364,8 +2410,16 @@ class TestPerformScheduledScanTask:
|
||||
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
assert result["id"] == str(executing_scan.id)
|
||||
assert result["state"] == StateChoices.EXECUTING
|
||||
assert result["id"] != str(executing_scan.id)
|
||||
assert result["state"] == StateChoices.AVAILABLE
|
||||
queued_scheduled_scan = Scan.objects.get(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
assert result["id"] == str(queued_scheduled_scan.id)
|
||||
assert queued_scheduled_scan.task.task_runner_task.status == "QUEUED"
|
||||
assert (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant.id,
|
||||
@@ -2373,7 +2427,133 @@ class TestPerformScheduledScanTask:
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
).count()
|
||||
== 0
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_queues_scheduled_scan_when_manual_scan_is_pending(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Queue one scheduled run when a manual scan is already dispatched."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
self._create_periodic_task(provider.id, tenant.id)
|
||||
task_id = str(uuid.uuid4())
|
||||
self._create_task_result(tenant.id, task_id)
|
||||
manual_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status=states.PENDING,
|
||||
)
|
||||
manual_task = Task.objects.create(
|
||||
id=manual_task_result.task_id,
|
||||
task_runner_task=manual_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
manual_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task=manual_task,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan") as mock_scan,
|
||||
patch("tasks.tasks._perform_scan_complete_tasks") as mock_complete_tasks,
|
||||
self._override_task_request(perform_scheduled_scan_task, id=task_id),
|
||||
):
|
||||
result = perform_scheduled_scan_task.run(
|
||||
tenant_id=str(tenant.id), provider_id=str(provider.id)
|
||||
)
|
||||
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
assert result["id"] != str(manual_scan.id)
|
||||
assert result["state"] == StateChoices.AVAILABLE
|
||||
queued_scheduled_scan = Scan.objects.get(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
assert result["id"] == str(queued_scheduled_scan.id)
|
||||
assert queued_scheduled_scan.task.task_runner_task.status == "QUEUED"
|
||||
scheduled_scan = Scan.objects.get(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
assert scheduled_scan.scheduled_at > datetime.now(UTC)
|
||||
|
||||
def test_coalesces_scheduled_scan_when_one_is_already_queued(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Reuse the existing queued scheduled scan instead of adding another."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
task_id = str(uuid.uuid4())
|
||||
self._create_task_result(tenant.id, task_id)
|
||||
manual_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status=states.PENDING,
|
||||
)
|
||||
manual_task = Task.objects.create(
|
||||
id=manual_task_result.task_id,
|
||||
task_runner_task=manual_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task=manual_task,
|
||||
)
|
||||
queued_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status="QUEUED",
|
||||
)
|
||||
queued_task = Task.objects.create(
|
||||
id=queued_task_result.task_id,
|
||||
task_runner_task=queued_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
queued_scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
task=queued_task,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan") as mock_scan,
|
||||
patch("tasks.tasks._perform_scan_complete_tasks") as mock_complete_tasks,
|
||||
self._override_task_request(perform_scheduled_scan_task, id=task_id),
|
||||
):
|
||||
result = perform_scheduled_scan_task.run(
|
||||
tenant_id=str(tenant.id), provider_id=str(provider.id)
|
||||
)
|
||||
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
assert result["id"] == str(queued_scheduled_scan.id)
|
||||
assert (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_creates_next_scheduled_scan_after_completion(
|
||||
@@ -2435,6 +2615,41 @@ class TestPerformScheduledScanTask:
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_next_scheduled_scan_failure_does_not_mask_completed_scan(
|
||||
self, tenants_fixture, providers_fixture, caplog
|
||||
):
|
||||
"""Keep scheduled scan success when next-run creation fails."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
self._create_periodic_task(provider.id, tenant.id)
|
||||
task_id = str(uuid.uuid4())
|
||||
self._create_task_result(tenant.id, task_id)
|
||||
|
||||
def _complete_scan(tenant_id, scan_id, provider_id):
|
||||
scan_instance = Scan.objects.get(id=scan_id)
|
||||
scan_instance.state = StateChoices.COMPLETED
|
||||
scan_instance.save()
|
||||
return {"status": "ok"}
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan", side_effect=_complete_scan),
|
||||
patch("tasks.tasks._perform_scan_complete_tasks"),
|
||||
patch(
|
||||
"tasks.tasks._get_or_create_next_scheduled_scan",
|
||||
side_effect=RuntimeError("scheduler unavailable"),
|
||||
),
|
||||
patch("tasks.tasks._dispatch_next_queued_provider_scan") as mock_dispatch,
|
||||
self._override_task_request(perform_scheduled_scan_task, id=task_id),
|
||||
caplog.at_level("ERROR"),
|
||||
):
|
||||
result = perform_scheduled_scan_task.run(
|
||||
tenant_id=str(tenant.id), provider_id=str(provider.id)
|
||||
)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
mock_dispatch.assert_called_once_with(str(tenant.id), str(provider.id))
|
||||
assert "Failed to ensure next scheduled scan" in caplog.text
|
||||
|
||||
def test_dedupes_multiple_scheduled_scans_before_run(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
@@ -2549,6 +2764,104 @@ class TestPerformScanTask:
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
|
||||
def test_dispatches_next_queued_scan_after_completion(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Dispatch the next queued scan for the provider after completion."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
current_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Running scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
queued_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status="QUEUED",
|
||||
)
|
||||
queued_task = Task.objects.create(
|
||||
id=queued_task_result.task_id,
|
||||
task_runner_task=queued_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
queued_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Queued scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task=queued_task,
|
||||
)
|
||||
|
||||
def _complete_scan(tenant_id, scan_id, provider_id, checks_to_execute=None):
|
||||
scan_instance = Scan.objects.get(id=scan_id)
|
||||
scan_instance.state = StateChoices.COMPLETED
|
||||
scan_instance.save()
|
||||
return {"status": "ok"}
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan", side_effect=_complete_scan),
|
||||
patch("tasks.tasks._perform_scan_complete_tasks"),
|
||||
patch("tasks.tasks.perform_scan_task.apply_async") as mock_apply_async,
|
||||
):
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
result = perform_scan_task.run(
|
||||
tenant_id=str(tenant.id),
|
||||
scan_id=str(current_scan.id),
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
|
||||
queued_task_result.refresh_from_db()
|
||||
assert result == {"status": "ok"}
|
||||
assert queued_task_result.status == states.PENDING
|
||||
mock_apply_async.assert_called_once_with(
|
||||
kwargs={
|
||||
"tenant_id": str(tenant.id),
|
||||
"scan_id": str(queued_scan.id),
|
||||
"provider_id": str(provider.id),
|
||||
},
|
||||
task_id=str(queued_task.id),
|
||||
)
|
||||
|
||||
def test_dispatch_failure_does_not_mask_completed_scan(
|
||||
self, tenants_fixture, providers_fixture, caplog
|
||||
):
|
||||
"""Keep scan success when queued dispatch fails after completion."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
current_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Running scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan", return_value={"status": "ok"}),
|
||||
patch("tasks.tasks._perform_scan_complete_tasks"),
|
||||
patch(
|
||||
"tasks.tasks._dispatch_next_queued_provider_scan",
|
||||
side_effect=RuntimeError("dispatch unavailable"),
|
||||
) as mock_dispatch,
|
||||
caplog.at_level("ERROR"),
|
||||
):
|
||||
result = perform_scan_task.run(
|
||||
tenant_id=str(tenant.id),
|
||||
scan_id=str(current_scan.id),
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
mock_dispatch.assert_called_once_with(str(tenant.id), str(provider.id))
|
||||
assert "Failed to dispatch next queued scan" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestReaggregateAllFindingGroupSummaries:
|
||||
|
||||
@@ -103,6 +103,7 @@ def _get_or_create_scheduled_scan(
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state__in=(StateChoices.SCHEDULED, StateChoices.AVAILABLE),
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
task__isnull=True,
|
||||
).order_by("scheduled_at", "inserted_at")
|
||||
)
|
||||
|
||||
|
||||
@@ -4762,7 +4762,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.33.0"
|
||||
version = "1.34.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Build command
|
||||
# docker build --platform=linux/amd64 --no-cache -t prowler:latest .
|
||||
|
||||
ARG PROWLER_VERSION=latest@sha256:4b796c6df40a3350c7947747b59bdda230d0da6222287500e13b0a8e1574aad4
|
||||
ARG PROWLER_VERSION=latest@sha256:ebb4ab999f10cb7e7c256226c2873de9b3bf2f3d855f385e0164bcf34104bfba
|
||||
|
||||
FROM toniblyx/prowler:${PROWLER_VERSION}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:alpine@sha256:8b1e78743a03dbb2c95171cc58639fef29abc8816598e27fb910ed2e621e589a
|
||||
image: nginx:alpine@sha256:54f2a904c251d5a34adf545a72d32515a15e08418dae0266e23be2e18c66fefa
|
||||
container_name: prowler-nginx
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
|
||||
@@ -42,7 +42,7 @@ When adding a new configurable check to Prowler, update the following files:
|
||||
```
|
||||
- **Provider Schema:** Add the typed field to the provider's Pydantic schema in `prowler/config/schema/<provider>.py`. This is required: the loader validates user configs against these schemas and the shipped `config.yaml` must round-trip with zero warnings. See [Adding a Parameter to the Provider Schema](#adding-a-parameter-to-the-provider-schema) below.
|
||||
- **Test Fixtures:** If tests depend on this configuration, add the variable to `tests/config/fixtures/config.yaml`.
|
||||
- **Documentation:** Document the new variable in the list of configurable checks in `docs/tutorials/configuration_file.md`.
|
||||
- **Documentation:** Document the new variable in the list of configurable checks in [Configuration File](/user-guide/cli/tutorials/configuration_file) (`docs/user-guide/cli/tutorials/configuration_file.mdx`).
|
||||
|
||||
For a complete list of checks that already support configuration, see the [Configuration File Tutorial](/user-guide/cli/tutorials/configuration_file).
|
||||
|
||||
|
||||
@@ -39,6 +39,24 @@ The former build-time variables map to the new runtime variables as follows:
|
||||
|
||||
The build-time-only Sentry variables used for source-map upload — `SENTRY_ORG`, `SENTRY_PROJECT`, `SENTRY_AUTH_TOKEN`, and `SENTRY_RELEASE` — keep their names, as they are not part of the App's runtime configuration.
|
||||
|
||||
## Enabling Third-Party Integrations
|
||||
|
||||
Prowler App gates each optional third-party integration behind an explicit enable flag. When an integration is configured through its new `UI_*` variables, it loads only when its flag is set to the exact string `"true"`; any other value, including unset, leaves it off. This default-off behavior keeps a deployment free of third-party egress unless it opts in. Deployments still using the deprecated legacy variable names keep loading without the flag, for backward compatibility (see [Deprecated Names](#deprecated-names)).
|
||||
|
||||
| Integration | Enable flag | Required configuration when enabled |
|
||||
|-------------|-------------|-------------------------------------|
|
||||
| Sentry (error monitoring) | `UI_SENTRY_ENABLE` | `UI_SENTRY_DSN` |
|
||||
| Google Tag Manager | `UI_GOOGLE_TAG_MANAGER_ENABLE` | `UI_GOOGLE_TAG_MANAGER_ID` |
|
||||
| PostHog (product analytics) | `UI_POSTHOG_ENABLE` | `UI_POSTHOG_KEY` and `UI_POSTHOG_HOST` |
|
||||
|
||||
When an integration is enabled but its required configuration is missing, Prowler App fails fast at server startup with a clear error, so a misconfigured container never starts silently. A new `UI_*` value set while its enable flag is not `"true"` is ignored, and the server logs a one-time startup warning noting that the integration will not load. Legacy names follow the backward-compatible rule described in [Deprecated Names](#deprecated-names).
|
||||
|
||||
PostHog support is currently limited to configuration validation: Prowler App reads and validates the PostHog variables but does not yet load a PostHog client.
|
||||
|
||||
<Warning>
|
||||
Configuring an integration through the new `UI_*` variables now requires its enable flag. A deployment that adopted `UI_SENTRY_DSN` or `UI_GOOGLE_TAG_MANAGER_ID` must also set `UI_SENTRY_ENABLE=true` or `UI_GOOGLE_TAG_MANAGER_ENABLE=true` to keep the integration active. Deployments still using the legacy names (`NEXT_PUBLIC_*`, or `POSTHOG_KEY` and `POSTHOG_HOST`) keep working without the flag.
|
||||
</Warning>
|
||||
|
||||
## Upcoming Breaking Change
|
||||
|
||||
<Warning>
|
||||
@@ -49,5 +67,5 @@ Prowler API, Prowler SDK, and Prowler MCP Server have not yet adopted the conven
|
||||
|
||||
## Deprecated Names
|
||||
|
||||
- **Prowler App:** The bare server-side `SENTRY_DSN` and `SENTRY_ENVIRONMENT` are no longer read; the server and edge runtimes now read `UI_SENTRY_DSN` and `UI_SENTRY_ENVIRONMENT`. The former `NEXT_PUBLIC_*` build-time variables are deprecated but still read at runtime as a fallback when the matching `UI_*` variable is unset. This fallback will be removed in a future release, so set the `UI_*` runtime variables on the running container.
|
||||
- **Prowler App:** The bare server-side `SENTRY_DSN` and `SENTRY_ENVIRONMENT` are no longer read; the server and edge runtimes now read `UI_SENTRY_DSN` and `UI_SENTRY_ENVIRONMENT`. The former `NEXT_PUBLIC_*` names — and, for PostHog, the unprefixed `POSTHOG_KEY` and `POSTHOG_HOST` — are deprecated but stay backward compatible: they are read at runtime regardless of the enable flag, so an existing deployment keeps its integration active without opting in. The new `UI_*` names, by contrast, load only when the matching enable flag is set to `"true"`. These legacy names will be removed in a future release, so migrate to the `UI_*` runtime variables — and set the enable flag — on the running container.
|
||||
- **Prowler API, Prowler SDK, and Prowler MCP Server:** The current, unprefixed variable names are deprecated. They continue to work today and will be removed once the prefixed convention is adopted for each component, as described in [Upcoming Breaking Change](#upcoming-breaking-change).
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
{
|
||||
"group": "Prowler Lighthouse AI",
|
||||
"pages": [
|
||||
"getting-started/products/prowler-lighthouse-ai"
|
||||
"getting-started/products/prowler-cloud-lighthouse"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -128,6 +128,7 @@
|
||||
"user-guide/tutorials/prowler-scan-scheduling",
|
||||
"user-guide/tutorials/prowler-alerts",
|
||||
"user-guide/tutorials/prowler-app-scan-configuration",
|
||||
"user-guide/tutorials/prowler-app-findings-triage",
|
||||
{
|
||||
"group": "Mutelist",
|
||||
"expanded": true,
|
||||
@@ -153,8 +154,15 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Lighthouse AI",
|
||||
"group": "Lighthouse AI (Prowler Cloud)",
|
||||
"pages": [
|
||||
"user-guide/tutorials/prowler-cloud-lighthouse-multi-llm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Lighthouse AI (Open Source)",
|
||||
"pages": [
|
||||
"getting-started/products/prowler-lighthouse-ai",
|
||||
"user-guide/tutorials/prowler-app-lighthouse",
|
||||
"user-guide/tutorials/prowler-app-lighthouse-multi-llm"
|
||||
]
|
||||
@@ -237,6 +245,7 @@
|
||||
"user-guide/providers/azure/authentication",
|
||||
"user-guide/providers/azure/use-non-default-cloud",
|
||||
"user-guide/providers/azure/subscriptions",
|
||||
"user-guide/providers/azure/resource-groups",
|
||||
"user-guide/providers/azure/create-prowler-service-principal"
|
||||
]
|
||||
},
|
||||
@@ -359,7 +368,8 @@
|
||||
"group": "Okta",
|
||||
"pages": [
|
||||
"user-guide/providers/okta/getting-started-okta",
|
||||
"user-guide/providers/okta/authentication"
|
||||
"user-guide/providers/okta/authentication",
|
||||
"user-guide/providers/okta/retry-configuration"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -128,8 +128,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.31.0"
|
||||
PROWLER_API_VERSION="5.31.0"
|
||||
PROWLER_UI_VERSION="5.32.0"
|
||||
PROWLER_API_VERSION="5.32.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
title: 'Overview'
|
||||
---
|
||||
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
Prowler Cloud runs an enhanced version of Lighthouse AI in Open Source repository, the Agentic Cloud Defender that helps teams understand, prioritize, and remediate security findings across cloud environments.
|
||||
|
||||
<SubscriptionBanner />
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/chat-animation.gif" alt="Lighthouse AI on Prowler Cloud" />
|
||||
|
||||
## What's New
|
||||
|
||||
The Agentic Cloud Defender does more than answer questions, it helps teams **find and remediate what actually matters**, cutting through the noise to focus on the risk that counts. Prowler Cloud sharpens Lighthouse AI with the following improvements over the open-source version:
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Persistent Chat Sessions" icon="clock-rotate-left">
|
||||
Conversations are saved and can be revisited or resumed at any time.
|
||||
</Card>
|
||||
<Card title="GPT-5.5 by Default" icon="microchip">
|
||||
An upgraded default model delivers stronger reasoning and tool calling.
|
||||
</Card>
|
||||
<Card title="Normal and Agentic Views" icon="table-columns">
|
||||
Switch between the standard interface and a chat-first agentic view.
|
||||
</Card>
|
||||
<Card title="Provider Connection Checks" icon="plug-circle-check">
|
||||
Credentials are validated automatically when a provider is configured.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
## Chat View
|
||||
|
||||
Lighthouse AI is no longer a separate section in the left navigation. Prowler Cloud now offers two application views: a normal view for browsing dashboards, findings, and configuration, and an agentic chat view, powered by Lighthouse AI, for conversational, multi-step security analysis. Conversations are saved automatically, so earlier sessions can be reopened and resumed at any time.
|
||||
|
||||
Promoting the chat to a top-level view gives Lighthouse AI the room it needs for a fully agentic workflow and makes the Agentic Cloud Defender a primary way to work in Prowler Cloud.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/main-chat-page.png" alt="Lighthouse AI chat view in Prowler Cloud" />
|
||||
|
||||
### Tool Usage
|
||||
|
||||
Lighthouse AI on Prowler Cloud renders the agent's work as it happens, so responses are easier to follow and to trust. Tool calls and reasoning steps appear in the order they occur within the conversation.
|
||||
|
||||
- **Ordered steps:** Tool calls and reasoning are shown in sequence, reflecting how the agent reached its answer.
|
||||
- **Tool visibility:** The data tools invoked to retrieve findings and other Prowler information are displayed as the agent uses them.
|
||||
- **Thought process:** The agent's reasoning is presented alongside its actions, and can be expanded to review the full chain of steps.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/tool-usage.png" alt="Ordered tool usage and reasoning in the Lighthouse AI chat" />
|
||||
|
||||
## Configuration
|
||||
|
||||
Configure Lighthouse AI on Prowler Cloud from **Configuration** → **Lighthouse AI**:
|
||||
|
||||
1. Click the desired provider (OpenAI, Amazon Bedrock, or OpenAI Compatible).
|
||||
2. Enter the required credentials.
|
||||
3. Click **Save**. The connection is validated automatically before the provider becomes available.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/config-page.png" alt="Lighthouse AI configuration page in Prowler Cloud" />
|
||||
|
||||
### Business Context
|
||||
|
||||
At the top of the configuration page, the optional **Business Context** field lets teams add environment priorities, compliance requirements, and ownership details, so responses align with organizational needs.
|
||||
|
||||
Lighthouse AI on Prowler Cloud supports OpenAI, Amazon Bedrock, and OpenAI-compatible providers, with GPT-5.5 as the default. For per-provider setup and how to switch the default provider or model, see [Using Multiple LLM Providers](/user-guide/tutorials/prowler-cloud-lighthouse-multi-llm).
|
||||
|
||||
## FAQ
|
||||
|
||||
**Which LLM providers are supported?**
|
||||
|
||||
OpenAI (GPT models, including the default GPT-5.5), Amazon Bedrock (Claude, Llama, Titan, and others), and any OpenAI-compatible service such as OpenRouter.
|
||||
|
||||
**Can Lighthouse AI change my cloud environment?**
|
||||
|
||||
No. Lighthouse AI has read-only access to security data and no tools to modify resources, even when the connected cloud credentials would allow changes.
|
||||
|
||||
## Looking for the Open Source Version?
|
||||
|
||||
Lighthouse AI is also available in the self-hosted, open-source Prowler App. For its capabilities, FAQs, and limitations, see the open-source documentation.
|
||||
|
||||
<Card title="Lighthouse AI (Open Source)" icon="github" href="/getting-started/products/prowler-lighthouse-ai">
|
||||
Capabilities, FAQs, and limitations for Lighthouse AI in the open-source Prowler App
|
||||
</Card>
|
||||
|
||||
## Getting Help
|
||||
|
||||
For issues or suggestions with Lighthouse AI on Prowler Cloud, request support at [support.prowler.com](https://support.prowler.com) or [reach out through our Slack channel](https://goto.prowler.com/slack).
|
||||
@@ -8,11 +8,16 @@ import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
Prowler Lighthouse AI is a Cloud Security Analyst chatbot that helps you understand, prioritize, and remediate security findings in your cloud environments. It's designed to provide security expertise for teams without dedicated resources, acting as your 24/7 virtual cloud security analyst.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-intro.png" alt="Prowler Lighthouse" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/intro.png" alt="Prowler Lighthouse" />
|
||||
|
||||
<Card title="Set Up Lighthouse AI" icon="rocket" href="/user-guide/tutorials/prowler-app-lighthouse#set-up">
|
||||
Learn how to configure Lighthouse AI with your preferred LLM provider
|
||||
</Card>
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Lighthouse AI on Prowler Cloud" icon="cloud" href="/getting-started/products/prowler-cloud-lighthouse">
|
||||
Discover the enhanced Cloud experience: persistent chat sessions, GPT-5.5 by default, a dedicated agentic view, and transparent reasoning
|
||||
</Card>
|
||||
<Card title="Set Up Lighthouse AI" icon="rocket" href="/user-guide/tutorials/prowler-app-lighthouse#set-up">
|
||||
Learn how to configure Lighthouse AI with your preferred LLM provider
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
## Capabilities
|
||||
|
||||
@@ -26,7 +31,7 @@ Ask questions in plain English about your security findings. Examples:
|
||||
- "Show me all S3 buckets with public access."
|
||||
- "What security issues were found in my production accounts?"
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-feature1.png" alt="Natural language querying" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/feature1.png" alt="Natural language querying" />
|
||||
|
||||
### Detailed Remediation Guidance
|
||||
|
||||
@@ -36,7 +41,7 @@ Get tailored step-by-step instructions for fixing security issues:
|
||||
- Commands or console steps to implement fixes
|
||||
- Alternative approaches with different solutions
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-feature2.png" alt="Detailed Remediation" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/feature2.png" alt="Detailed Remediation" />
|
||||
|
||||
### Enhanced Context and Analysis
|
||||
|
||||
@@ -46,9 +51,9 @@ Lighthouse AI can provide additional context to help you understand the findings
|
||||
- Provide risk assessments based on your environment and context
|
||||
- Connect related findings to show broader security patterns
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-config.png" alt="Business Context" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/config.png" alt="Business Context" />
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-feature3.png" alt="Contextual Responses" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/feature3.png" alt="Contextual Responses" />
|
||||
|
||||
## Important Notes
|
||||
|
||||
|
||||
|
Before Width: | Height: | Size: 115 KiB After Width: | Height: | Size: 123 KiB |
|
Before Width: | Height: | Size: 316 KiB After Width: | Height: | Size: 111 KiB |
|
Before Width: | Height: | Size: 304 KiB After Width: | Height: | Size: 97 KiB |
|
Before Width: | Height: | Size: 477 KiB After Width: | Height: | Size: 136 KiB |
|
Before Width: | Height: | Size: 575 KiB After Width: | Height: | Size: 107 KiB |
|
Before Width: | Height: | Size: 324 KiB After Width: | Height: | Size: 136 KiB |
|
Before Width: | Height: | Size: 115 KiB After Width: | Height: | Size: 123 KiB |
|
Before Width: | Height: | Size: 423 KiB After Width: | Height: | Size: 120 KiB |
|
Before Width: | Height: | Size: 275 KiB After Width: | Height: | Size: 121 KiB |
|
After Width: | Height: | Size: 81 KiB |
|
After Width: | Height: | Size: 610 KiB |
|
After Width: | Height: | Size: 593 KiB |
|
Before Width: | Height: | Size: 96 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 540 KiB After Width: | Height: | Size: 540 KiB |
|
Before Width: | Height: | Size: 136 KiB After Width: | Height: | Size: 136 KiB |
|
Before Width: | Height: | Size: 147 KiB After Width: | Height: | Size: 147 KiB |
|
Before Width: | Height: | Size: 180 KiB After Width: | Height: | Size: 180 KiB |
|
Before Width: | Height: | Size: 165 KiB After Width: | Height: | Size: 165 KiB |
|
Before Width: | Height: | Size: 347 KiB After Width: | Height: | Size: 347 KiB |
|
Before Width: | Height: | Size: 173 KiB After Width: | Height: | Size: 173 KiB |
|
After Width: | Height: | Size: 236 KiB |
|
After Width: | Height: | Size: 472 KiB |
|
After Width: | Height: | Size: 324 KiB |
|
After Width: | Height: | Size: 505 KiB |
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate docs/snippets/provider-cards.mdx from provider getting-started pages.
|
||||
|
||||
Scans docs/user-guide/providers/<name>/getting-started-*.mdx, keeps only the
|
||||
providers that Prowler App/Cloud actually supports (source of truth: the
|
||||
`ProviderChoices` enum in api/src/backend/api/models.py — CLI-only providers
|
||||
such as Linode/LLM/Scaleway/StackIT are excluded), reads the frontmatter
|
||||
`title`, derives a display name, and emits a snippet exporting a
|
||||
`ProviderCards` component. Wired into pre-commit so the snippet stays in sync
|
||||
whenever a provider page or the API enum changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PROVIDERS_DIR = REPO_ROOT / "docs" / "user-guide" / "providers"
|
||||
SNIPPET_PATH = REPO_ROOT / "docs" / "snippets" / "provider-cards.mdx"
|
||||
API_MODELS_PATH = REPO_ROOT / "api" / "src" / "backend" / "api" / "models.py"
|
||||
|
||||
# Docs folder names that don't match the API enum key. Keep tiny — only rename
|
||||
# entries when the docs folder disagrees with the API-side identifier.
|
||||
DOCS_DIR_TO_API_KEY = {
|
||||
"microsoft365": "m365",
|
||||
"oci": "oraclecloud",
|
||||
}
|
||||
|
||||
# Folder-name → Mintlify icon override. Providers not listed fall back to
|
||||
# DEFAULT_ICON. Add an entry only when the default looks wrong for a provider.
|
||||
ICON_OVERRIDES = {
|
||||
"alibabacloud": "cloud",
|
||||
"aws": "aws",
|
||||
"azure": "microsoft",
|
||||
"cloudflare": "cloudflare",
|
||||
"gcp": "google",
|
||||
"github": "github",
|
||||
"googleworkspace": "users",
|
||||
"iac": "code",
|
||||
"image": "docker",
|
||||
"kubernetes": "dharmachakra",
|
||||
"microsoft365": "briefcase",
|
||||
"mongodbatlas": "leaf",
|
||||
"oci": "database",
|
||||
"okta": "key",
|
||||
"openstack": "cubes",
|
||||
"vercel": "triangle",
|
||||
}
|
||||
DEFAULT_ICON = "cloud"
|
||||
|
||||
TITLE_RE = re.compile(r"^\s*title\s*:\s*['\"](?P<title>.+?)['\"]\s*$", re.MULTILINE)
|
||||
NAME_CLEANUP_RE = re.compile(
|
||||
r"^Getting Started [Ww]ith (?:the )?(?P<name>.+?)(?: on Prowler)?(?: Provider)?$"
|
||||
)
|
||||
|
||||
|
||||
def app_supported_provider_keys() -> set[str]:
|
||||
"""Return the set of provider keys declared in the API's ProviderChoices enum.
|
||||
|
||||
Uses ast rather than regex so formatting changes, decorators, comments, or
|
||||
multi-line values in the enum body don't silently drop or invent providers.
|
||||
"""
|
||||
tree = ast.parse(API_MODELS_PATH.read_text(encoding="utf-8"))
|
||||
for node in ast.walk(tree):
|
||||
if not (isinstance(node, ast.ClassDef) and node.name == "ProviderChoices"):
|
||||
continue
|
||||
keys: set[str] = set()
|
||||
for item in node.body:
|
||||
if not isinstance(item, ast.Assign):
|
||||
continue
|
||||
value = item.value
|
||||
# Django TextChoices members look like: NAME = "key", _("Label")
|
||||
# which parses as an ast.Tuple whose first element is the key.
|
||||
if isinstance(value, ast.Tuple) and value.elts:
|
||||
first = value.elts[0]
|
||||
if isinstance(first, ast.Constant) and isinstance(first.value, str):
|
||||
keys.add(first.value)
|
||||
return keys
|
||||
raise RuntimeError(
|
||||
f"Could not locate ProviderChoices class in {API_MODELS_PATH.relative_to(REPO_ROOT)}"
|
||||
)
|
||||
|
||||
|
||||
def extract_title(mdx_path: Path) -> str:
|
||||
text = mdx_path.read_text(encoding="utf-8")
|
||||
match = TITLE_RE.search(text)
|
||||
if not match:
|
||||
raise ValueError(f"No frontmatter title in {mdx_path}")
|
||||
return match.group("title")
|
||||
|
||||
|
||||
def display_name(title: str) -> str:
|
||||
match = NAME_CLEANUP_RE.match(title)
|
||||
return match.group("name") if match else title
|
||||
|
||||
|
||||
def collect_providers() -> list[dict]:
|
||||
supported = app_supported_provider_keys()
|
||||
providers = []
|
||||
for provider_dir in sorted(PROVIDERS_DIR.iterdir()):
|
||||
if not provider_dir.is_dir():
|
||||
continue
|
||||
api_key = DOCS_DIR_TO_API_KEY.get(provider_dir.name, provider_dir.name)
|
||||
if api_key not in supported:
|
||||
continue
|
||||
pages = sorted(provider_dir.glob("getting-started-*.mdx"))
|
||||
if not pages:
|
||||
continue
|
||||
page = pages[0]
|
||||
name = display_name(extract_title(page))
|
||||
href = f"/user-guide/providers/{provider_dir.name}/{page.stem}"
|
||||
icon = ICON_OVERRIDES.get(provider_dir.name, DEFAULT_ICON)
|
||||
providers.append({"name": name, "href": href, "icon": icon})
|
||||
providers.sort(key=lambda p: p["name"].lower())
|
||||
return providers
|
||||
|
||||
|
||||
def render_snippet(providers: list[dict]) -> str:
|
||||
cards = "\n".join(
|
||||
f' <Card title="{p["name"]}" icon="{p["icon"]}" href="{p["href"]}" />'
|
||||
for p in providers
|
||||
)
|
||||
return (
|
||||
"{/* AUTO-GENERATED by docs/scripts/generate_provider_cards.py — do not edit by hand. */}\n"
|
||||
"{/* Regenerated on pre-commit whenever any provider getting-started page changes. */}\n"
|
||||
"\n"
|
||||
"export const ProviderCards = () => (\n"
|
||||
" <Columns cols={3}>\n"
|
||||
f"{cards}\n"
|
||||
" </Columns>\n"
|
||||
");\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
providers = collect_providers()
|
||||
if not providers:
|
||||
print("No provider getting-started pages found", file=sys.stderr)
|
||||
return 1
|
||||
new_content = render_snippet(providers)
|
||||
current = SNIPPET_PATH.read_text(encoding="utf-8") if SNIPPET_PATH.exists() else ""
|
||||
if new_content == current:
|
||||
return 0
|
||||
SNIPPET_PATH.write_text(new_content, encoding="utf-8")
|
||||
print(
|
||||
f"Regenerated {SNIPPET_PATH.relative_to(REPO_ROOT)} ({len(providers)} providers)"
|
||||
)
|
||||
return 1 # signal pre-commit that the file changed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,23 @@
|
||||
{/* AUTO-GENERATED by docs/scripts/generate_provider_cards.py — do not edit by hand. */}
|
||||
{/* Regenerated on pre-commit whenever any provider getting-started page changes. */}
|
||||
|
||||
export const ProviderCards = () => (
|
||||
<Columns cols={3}>
|
||||
<Card title="Alibaba Cloud" icon="cloud" href="/user-guide/providers/alibabacloud/getting-started-alibabacloud" />
|
||||
<Card title="AWS" icon="aws" href="/user-guide/providers/aws/getting-started-aws" />
|
||||
<Card title="Azure" icon="microsoft" href="/user-guide/providers/azure/getting-started-azure" />
|
||||
<Card title="Cloudflare" icon="cloudflare" href="/user-guide/providers/cloudflare/getting-started-cloudflare" />
|
||||
<Card title="GCP" icon="google" href="/user-guide/providers/gcp/getting-started-gcp" />
|
||||
<Card title="GitHub" icon="github" href="/user-guide/providers/github/getting-started-github" />
|
||||
<Card title="Google Workspace" icon="users" href="/user-guide/providers/googleworkspace/getting-started-googleworkspace" />
|
||||
<Card title="IaC" icon="code" href="/user-guide/providers/iac/getting-started-iac" />
|
||||
<Card title="Image" icon="docker" href="/user-guide/providers/image/getting-started-image" />
|
||||
<Card title="Kubernetes" icon="dharmachakra" href="/user-guide/providers/kubernetes/getting-started-k8s" />
|
||||
<Card title="Microsoft 365" icon="briefcase" href="/user-guide/providers/microsoft365/getting-started-m365" />
|
||||
<Card title="MongoDB Atlas" icon="leaf" href="/user-guide/providers/mongodbatlas/getting-started-mongodbatlas" />
|
||||
<Card title="Okta" icon="key" href="/user-guide/providers/okta/getting-started-okta" />
|
||||
<Card title="OpenStack" icon="cubes" href="/user-guide/providers/openstack/getting-started-openstack" />
|
||||
<Card title="Oracle Cloud Infrastructure (OCI)" icon="database" href="/user-guide/providers/oci/getting-started-oci" />
|
||||
<Card title="Vercel" icon="triangle" href="/user-guide/providers/vercel/getting-started-vercel" />
|
||||
</Columns>
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
export const SubscriptionBanner = ({ children }) => {
|
||||
return (
|
||||
<Note>
|
||||
This feature is available exclusively in <b>Prowler Cloud</b> and <b>Prowler Enterprise</b> with a <a href="https://prowler.com/pricing">subscription</a>.
|
||||
{children}
|
||||
</Note>
|
||||
);
|
||||
};
|
||||
@@ -22,71 +22,95 @@ Numeric thresholds enforce hard limits. A value outside the accepted range is dr
|
||||
|
||||
The following list includes all the AWS checks with configurable variables that can be changed in the configuration yaml file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|--------------------------------------------------|-----------------|
|
||||
| `acm_certificates_expiration_check` | `days_to_expire_threshold` | Integer |
|
||||
| `acmpca_certificate_authority_pqc_key_algorithm` | `acmpca_pqc_key_algorithms` | List of Strings |
|
||||
| `appstream_fleet_maximum_session_duration` | `max_session_duration_seconds` | Integer |
|
||||
| `appstream_fleet_session_disconnect_timeout` | `max_disconnect_timeout_in_seconds` | Integer |
|
||||
| `appstream_fleet_session_idle_disconnect_timeout` | `max_idle_disconnect_timeout_in_seconds` | Integer |
|
||||
| `autoscaling_find_secrets_ec2_launch_configuration` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `awslambda_function_no_secrets_in_code` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `awslambda_function_no_secrets_in_variables` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `awslambda_function_using_supported_runtimes` | `obsolete_lambda_runtimes` | Integer |
|
||||
| `awslambda_function_vpc_is_in_multi_azs` | `lambda_min_azs` | Integer |
|
||||
| `cloudformation_stack_outputs_find_secrets` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `cloudtrail_threat_detection_enumeration` | `threat_detection_enumeration_actions` | List of Strings |
|
||||
| `cloudtrail_threat_detection_enumeration` | `threat_detection_enumeration_entropy` | Integer |
|
||||
| `cloudtrail_threat_detection_enumeration` | `threat_detection_enumeration_minutes` | Integer |
|
||||
| `cloudtrail_threat_detection_privilege_escalation` | `threat_detection_privilege_escalation_actions` | List of Strings |
|
||||
| `cloudtrail_threat_detection_privilege_escalation` | `threat_detection_privilege_escalation_entropy` | Integer |
|
||||
| `cloudtrail_threat_detection_privilege_escalation` | `threat_detection_privilege_escalation_minutes` | Integer |
|
||||
| `cloudwatch_log_group_no_secrets_in_logs` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `cloudwatch_log_group_retention_policy_specific_days_enabled` | `log_group_retention_days` | Integer |
|
||||
| `codebuild_github_allowed_organizations` | `github_allowed_organizations` | List of Strings |
|
||||
| `codebuild_project_no_secrets_in_variables` | `excluded_sensitive_environment_variables` | List of Strings |
|
||||
| `codebuild_project_no_secrets_in_variables` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `config_recorder_all_regions_enabled` | `mute_non_default_regions` | Boolean |
|
||||
| `drs_job_exist` | `mute_non_default_regions` | Boolean |
|
||||
| `ec2_elastic_ip_shodan` | `shodan_api_key` | String |
|
||||
| `ec2_instance_older_than_specific_days` | `max_ec2_instance_age_in_days` | Integer |
|
||||
| `ec2_instance_secrets_user_data` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `ec2_launch_template_no_secrets` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `ec2_securitygroup_allow_ingress_from_internet_to_any_port` | `ec2_allowed_instance_owners` | List of Strings |
|
||||
| `ec2_securitygroup_allow_ingress_from_internet_to_any_port` | `ec2_allowed_interface_types` | List of Strings |
|
||||
| `ec2_securitygroup_allow_ingress_from_internet_to_high_risk_tcp_ports`| `ec2_high_risk_ports` | List of Integer |
|
||||
| `ec2_securitygroup_with_many_ingress_egress_rules` | `max_security_group_rules` | Integer |
|
||||
| `ecs_task_definitions_no_environment_secrets` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `ecr_repositories_scan_vulnerabilities_in_latest_image` | `ecr_repository_vulnerability_minimum_severity` | String |
|
||||
| `eks_cluster_uses_a_supported_version` | `eks_cluster_oldest_version_supported` | String |
|
||||
| `eks_control_plane_logging_all_types_enabled` | `eks_required_log_types` | List of Strings |
|
||||
| `elasticache_redis_cluster_backup_enabled` | `minimum_snapshot_retention_period` | Integer |
|
||||
| `elb_is_in_multiple_az` | `elb_min_azs` | Integer |
|
||||
| `elbv2_is_in_multiple_az` | `elbv2_min_azs` | Integer |
|
||||
| `rolesanywhere_trust_anchor_pqc_pki` | `rolesanywhere_pqc_pca_key_algorithms` | List of Strings |
|
||||
| `cloudfront_distributions_pqc_tls_enabled` | `cloudfront_pqc_min_protocol_versions` | List of Strings |
|
||||
| `apigateway_domain_name_pqc_tls_enabled` | `apigateway_pqc_tls_allowed_policies` | List of Strings |
|
||||
| `guardduty_is_enabled` | `mute_non_default_regions` | Boolean |
|
||||
| `iam_user_access_not_stale_to_sagemaker` | `max_unused_sagemaker_access_days` | Integer |
|
||||
| `iam_user_accesskey_unused` | `max_unused_access_keys_days` | Integer |
|
||||
| `iam_user_console_access_unused` | `max_console_access_days` | Integer |
|
||||
| `organizations_delegated_administrators` | `organizations_trusted_delegated_administrators` | List of Strings |
|
||||
| `organizations_scp_check_deny_regions` | `organizations_enabled_regions` | List of Strings |
|
||||
| `rds_instance_backup_enabled` | `check_rds_instance_replicas` | Boolean |
|
||||
| `securityhub_enabled` | `mute_non_default_regions` | Boolean |
|
||||
| `secretsmanager_secret_unused` | `max_days_secret_unused` | Integer |
|
||||
| `secretsmanager_secret_rotated_periodically` | `max_days_secret_unrotated` | Integer |
|
||||
| `ssm_document_secrets` | `secrets_ignore_patterns` | List of Strings |
|
||||
| `trustedadvisor_premium_support_plan_subscribed` | `verify_premium_support_plans` | Boolean |
|
||||
| `transfer_server_pqc_ssh_kex_enabled` | `transfer_pqc_ssh_allowed_policies` | List of Strings |
|
||||
| `dynamodb_table_cross_account_access` | `trusted_account_ids` | List of Strings |
|
||||
| `eventbridge_bus_cross_account_access` | `trusted_account_ids` | List of Strings |
|
||||
| `eventbridge_schema_registry_cross_account_access` | `trusted_account_ids` | List of Strings |
|
||||
| `s3_bucket_cross_account_access` | `trusted_account_ids` | List of Strings |
|
||||
| `ssm_documents_set_as_public` | `trusted_account_ids` | List of Strings |
|
||||
| `vpc_endpoint_connections_trust_boundaries` | `trusted_account_ids` | List of Strings |
|
||||
| `vpc_endpoint_services_allowed_principals_trust_boundaries` | `trusted_account_ids` | List of Strings |
|
||||
| `opensearch_service_domains_not_publicly_accessible` | `trusted_ips` | List of Strings |
|
||||
| Check Name | Value | Type | Default |
|
||||
|------------------------------------------------------------------------|---------------------------------------------------|-----------------|--------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `accessanalyzer_enabled` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `acm_certificates_expiration_check` | `days_to_expire_threshold` | Integer | `7` |
|
||||
| `acm_certificates_with_secure_key_algorithms` | `insecure_key_algorithms` | List of Strings | `["RSA-1024", "P-192"]` |
|
||||
| `acmpca_certificate_authority_pqc_key_algorithm` | `acmpca_pqc_key_algorithms` | List of Strings | `["ML_DSA_44", "ML_DSA_65", "ML_DSA_87"]` |
|
||||
| `apigateway_domain_name_pqc_tls_enabled` | `apigateway_pqc_tls_allowed_policies` | List of Strings | `["SecurityPolicy_TLS13_1_2_FIPS_PFS_PQ_2025_09", "SecurityPolicy_TLS13_1_2_PFS_PQ_2025_09", "SecurityPolicy_TLS13_1_2_PQ_2025_09"]` |
|
||||
| `apigateway_restapi_no_secrets_in_stage_variables` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `appstream_fleet_maximum_session_duration` | `max_session_duration_seconds` | Integer | `36000` |
|
||||
| `appstream_fleet_session_disconnect_timeout` | `max_disconnect_timeout_in_seconds` | Integer | `300` |
|
||||
| `appstream_fleet_session_idle_disconnect_timeout` | `max_idle_disconnect_timeout_in_seconds` | Integer | `600` |
|
||||
| `autoscaling_find_secrets_ec2_launch_configuration` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `awslambda_function_no_secrets_in_code` | `secrets_ignore_files` | List of Strings | `[]` |
|
||||
| `awslambda_function_no_secrets_in_code` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `awslambda_function_no_secrets_in_variables` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `awslambda_function_using_supported_runtimes` | `obsolete_lambda_runtimes` | List of Strings | See `config.yaml` |
|
||||
| `awslambda_function_vpc_multi_az` | `lambda_min_azs` | Integer | `2` |
|
||||
| `cloudformation_stack_cdktoolkit_bootstrap_version` | `recommended_cdk_bootstrap_version` | Integer | `21` |
|
||||
| `cloudformation_stack_outputs_find_secrets` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `cloudfront_distributions_pqc_tls_enabled` | `cloudfront_pqc_min_protocol_versions` | List of Strings | `["TLSv1.3_2025"]` |
|
||||
| `cloudtrail_threat_detection_enumeration` | `threat_detection_enumeration_actions` | List of Strings | See `config.yaml` |
|
||||
| `cloudtrail_threat_detection_enumeration` | `threat_detection_enumeration_minutes` | Integer | `1440` |
|
||||
| `cloudtrail_threat_detection_enumeration` | `threat_detection_enumeration_threshold` | Float | `0.3` |
|
||||
| `cloudtrail_threat_detection_llm_jacking` | `threat_detection_llm_jacking_actions` | List of Strings | See `config.yaml` |
|
||||
| `cloudtrail_threat_detection_llm_jacking` | `threat_detection_llm_jacking_minutes` | Integer | `1440` |
|
||||
| `cloudtrail_threat_detection_llm_jacking` | `threat_detection_llm_jacking_threshold` | Float | `0.4` |
|
||||
| `cloudtrail_threat_detection_privilege_escalation` | `threat_detection_privilege_escalation_actions` | List of Strings | See `config.yaml` |
|
||||
| `cloudtrail_threat_detection_privilege_escalation` | `threat_detection_privilege_escalation_minutes` | Integer | `1440` |
|
||||
| `cloudtrail_threat_detection_privilege_escalation` | `threat_detection_privilege_escalation_threshold` | Float | `0.2` |
|
||||
| `cloudwatch_log_group_no_secrets_in_logs` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `cloudwatch_log_group_retention_policy_specific_days_enabled` | `log_group_retention_days` | Integer | `365` |
|
||||
| `codebuild_project_no_secrets_in_variables` | `excluded_sensitive_environment_variables` | List of Strings | `[]` |
|
||||
| `codebuild_project_no_secrets_in_variables` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `codebuild_project_uses_allowed_github_organizations` | `codebuild_github_allowed_organizations` | List of Strings | `[]` |
|
||||
| `config_delegated_admin_and_org_aggregator_all_regions` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `config_recorder_all_regions_enabled` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `documentdb_cluster_backup_enabled` | `minimum_backup_retention_period` | Integer | `7` |
|
||||
| `drs_job_exist` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `dynamodb_table_cross_account_access` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
| `ec2_elastic_ip_shodan` | `shodan_api_key` | String | `null` |
|
||||
| `ec2_instance_older_than_specific_days` | `max_ec2_instance_age_in_days` | Integer | `180` |
|
||||
| `ec2_instance_secrets_user_data` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `ec2_launch_template_no_secrets` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `ec2_securitygroup_allow_ingress_from_internet_to_any_port` | `ec2_allowed_instance_owners` | List of Strings | `["amazon-elb"]` |
|
||||
| `ec2_securitygroup_allow_ingress_from_internet_to_any_port` | `ec2_allowed_interface_types` | List of Strings | `["api_gateway_managed", "vpc_endpoint"]` |
|
||||
| `ec2_securitygroup_allow_ingress_from_internet_to_high_risk_tcp_ports` | `ec2_high_risk_ports` | List of Integer | `[25, 110, 135, 143, 445, 3000, 4333, 5000, 5500, 8080, 8088]` |
|
||||
| `ec2_securitygroup_with_many_ingress_egress_rules` | `max_security_group_rules` | Integer | `50` |
|
||||
| `ecr_repositories_scan_vulnerabilities_in_latest_image` | `ecr_repository_vulnerability_minimum_severity` | String | `"MEDIUM"` |
|
||||
| `ecs_service_fargate_latest_platform_version` | `fargate_linux_latest_version` | String | `"1.4.0"` |
|
||||
| `ecs_service_fargate_latest_platform_version` | `fargate_windows_latest_version` | String | `"1.0.0"` |
|
||||
| `ecs_task_definitions_no_environment_secrets` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `eks_cluster_uses_a_supported_version` | `eks_cluster_oldest_version_supported` | String | `"1.28"` |
|
||||
| `eks_control_plane_logging_all_types_enabled` | `eks_required_log_types` | List of Strings | `["api", "audit", "authenticator", "controllerManager", "scheduler"]` |
|
||||
| `elasticache_redis_cluster_backup_enabled` | `minimum_snapshot_retention_period` | Integer | `7` |
|
||||
| `elb_is_in_multiple_az` | `elb_min_azs` | Integer | `2` |
|
||||
| `elbv2_is_in_multiple_az` | `elbv2_min_azs` | Integer | `2` |
|
||||
| `eventbridge_bus_cross_account_access` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
| `eventbridge_schema_registry_cross_account_access` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
| `glue_etl_jobs_no_secrets_in_arguments` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `guardduty_delegated_admin_enabled_all_regions` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `guardduty_is_enabled` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `iam_role_access_not_stale_to_bedrock` | `max_unused_bedrock_access_days` | Integer | `60` |
|
||||
| `iam_user_access_not_stale_to_bedrock` | `max_unused_bedrock_access_days` | Integer | `60` |
|
||||
| `iam_user_access_not_stale_to_sagemaker` | `max_unused_sagemaker_access_days` | Integer | `90` |
|
||||
| `iam_user_accesskey_unused` | `max_unused_access_keys_days` | Integer | `45` |
|
||||
| `iam_user_console_access_unused` | `max_console_access_days` | Integer | `45` |
|
||||
| `kinesis_stream_data_retention_period` | `min_kinesis_stream_retention_hours` | Integer | `168` |
|
||||
| `neptune_cluster_backup_enabled` | `minimum_backup_retention_period` | Integer | `7` |
|
||||
| `opensearch_service_domains_not_publicly_accessible` | `trusted_ips` | List of Strings | `[]` |
|
||||
| `organizations_delegated_administrators` | `organizations_trusted_delegated_administrators` | List of Strings | `[]` |
|
||||
| `organizations_scp_check_deny_regions` | `organizations_enabled_regions` | List of Strings | `[]` |
|
||||
| `rds_instance_backup_enabled` | `check_rds_instance_replicas` | Boolean | `False` |
|
||||
| `rolesanywhere_trust_anchor_pqc_pki` | `rolesanywhere_pqc_pca_key_algorithms` | List of Strings | `["ML_DSA_44", "ML_DSA_65", "ML_DSA_87"]` |
|
||||
| `s3_bucket_cross_account_access` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
| `s3_bucket_object_public` | `s3_bucket_object_public_enabled` | Boolean | `False` |
|
||||
| `s3_bucket_object_public` | `s3_bucket_object_public_max_objects` | Integer | `100` |
|
||||
| `s3_bucket_object_public` | `s3_bucket_object_public_sample_size` | Integer | `3` |
|
||||
| `secretsmanager_has_restrictive_resource_policy` | `organizations_trusted_ids` | List of Strings | `[]` |
|
||||
| `secretsmanager_secret_rotated_periodically` | `max_days_secret_unrotated` | Integer | `90` |
|
||||
| `secretsmanager_secret_unused` | `max_days_secret_unused` | Integer | `90` |
|
||||
| `securityhub_delegated_admin_enabled_all_regions` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `securityhub_enabled` | `mute_non_default_regions` | Boolean | `False` |
|
||||
| `ssm_document_secrets` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `ssm_documents_set_as_public` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
| `stepfunctions_statemachine_no_secrets_in_definition` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `transfer_server_pqc_ssh_kex_enabled` | `transfer_pqc_ssh_allowed_policies` | List of Strings | `["TransferSecurityPolicy-2025-03", "TransferSecurityPolicy-FIPS-2025-03", "TransferSecurityPolicy-AS2Restricted-2025-07"]` |
|
||||
| `trustedadvisor_premium_support_plan_subscribed` | `verify_premium_support_plans` | Boolean | `True` |
|
||||
| `vpc_endpoint_connections_trust_boundaries` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
| `vpc_endpoint_services_allowed_principals_trust_boundaries` | `trusted_account_ids` | List of Strings | `[]` |
|
||||
|
||||
### Resource Scan Limit
|
||||
|
||||
@@ -179,20 +203,20 @@ Secret validation makes outbound network calls that authenticate with each disco
|
||||
### Configurable Checks
|
||||
The following list includes all the Azure checks with configurable variables that can be changed in the configuration yaml file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|--------------------------------------------------|-----------------|
|
||||
| `network_public_ip_shodan` | `shodan_api_key` | String |
|
||||
| `app_ensure_php_version_is_latest` | `php_latest_version` | String |
|
||||
| `app_ensure_python_version_is_latest` | `python_latest_version` | String |
|
||||
| `app_ensure_java_version_is_latest` | `java_latest_version` | String |
|
||||
| `sqlserver_recommended_minimal_tls_version` | `recommended_minimal_tls_versions` | List of Strings |
|
||||
| `vm_sufficient_daily_backup_retention_period` | `vm_backup_min_daily_retention_days` | Integer |
|
||||
| `vm_desired_sku_size` | `desired_vm_sku_sizes` | List of Strings |
|
||||
| `storage_smb_channel_encryption_with_secure_algorithm` | `recommended_smb_channel_encryption_algorithms` | List of Strings |
|
||||
| `defender_attack_path_notifications_properly_configured` | `defender_attack_path_minimal_risk_level` | String |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_threshold` | Float |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_minutes` | Integer |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_actions` | List of Strings |
|
||||
| Check Name | Value | Type | Default |
|
||||
|----------------------------------------------------------|-------------------------------------------------|-----------------|------------------------------------------------------------|
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_actions` | List of Strings | See `config.yaml` |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_minutes` | Integer | `1440` |
|
||||
| `apim_threat_detection_llm_jacking` | `apim_threat_detection_llm_jacking_threshold` | Float | `0.1` |
|
||||
| `app_ensure_java_version_is_latest` | `java_latest_version` | String | `"17"` |
|
||||
| `app_ensure_php_version_is_latest` | `php_latest_version` | String | `"8.2"` |
|
||||
| `app_ensure_python_version_is_latest` | `python_latest_version` | String | `"3.12"` |
|
||||
| `defender_attack_path_notifications_properly_configured` | `defender_attack_path_minimal_risk_level` | String | `"High"` |
|
||||
| `network_public_ip_shodan` | `shodan_api_key` | String | `null` |
|
||||
| `sqlserver_recommended_minimal_tls_version` | `recommended_minimal_tls_versions` | List of Strings | `["1.2", "1.3"]` |
|
||||
| `storage_smb_channel_encryption_with_secure_algorithm` | `recommended_smb_channel_encryption_algorithms` | List of Strings | `["AES-256-GCM"]` |
|
||||
| `vm_desired_sku_size` | `desired_vm_sku_sizes` | List of Strings | `["Standard_A8_v2", "Standard_DS3_v2", "Standard_D4s_v3"]` |
|
||||
| `vm_sufficient_daily_backup_retention_period` | `vm_backup_min_daily_retention_days` | Integer | `7` |
|
||||
|
||||
|
||||
## GCP
|
||||
@@ -200,23 +224,28 @@ The following list includes all the Azure checks with configurable variables tha
|
||||
### Configurable Checks
|
||||
The following list includes all the GCP checks with configurable variables that can be changed in the configuration yaml file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|--------------------------------------------------|-----------------|
|
||||
| `compute_configuration_changes` | `compute_audit_log_lookback_days` | Integer |
|
||||
| `compute_instance_group_multiple_zones` | `mig_min_zones` | Integer |
|
||||
| Check Name | Value | Type | Default |
|
||||
|---------------------------------------------------|-----------------------------------|---------|---------|
|
||||
| `cloudstorage_bucket_sufficient_retention_period` | `storage_min_retention_days` | Integer | `90` |
|
||||
| `compute_instance_group_multiple_zones` | `mig_min_zones` | Integer | `2` |
|
||||
| `compute_public_address_shodan` | `shodan_api_key` | String | `null` |
|
||||
| `compute_snapshot_not_outdated` | `max_snapshot_age_days` | Integer | `90` |
|
||||
| `iam_sa_user_managed_key_unused` | `max_unused_account_days` | Integer | `180` |
|
||||
| `iam_service_account_unused` | `max_unused_account_days` | Integer | `180` |
|
||||
| `secretmanager_secret_rotation_enabled` | `secretmanager_max_rotation_days` | Integer | `90` |
|
||||
|
||||
## Kubernetes
|
||||
|
||||
### Configurable Checks
|
||||
The following list includes all the Kubernetes checks with configurable variables that can be changed in the configuration yaml file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|--------------------------------------------------|-----------------|
|
||||
| `audit_log_maxbackup` | `audit_log_maxbackup` | String |
|
||||
| `audit_log_maxsize` | `audit_log_maxsize` | String |
|
||||
| `audit_log_maxage` | `audit_log_maxage` | String |
|
||||
| `apiserver_strong_ciphers` | `apiserver_strong_ciphers` | String |
|
||||
| `kubelet_strong_ciphers_only` | `kubelet_strong_ciphers` | String |
|
||||
| Check Name | Value | Type | Default |
|
||||
|-------------------------------------|----------------------------|-----------------|----------------------------------------------------------------------------------------|
|
||||
| `apiserver_audit_log_maxage_set` | `audit_log_maxage` | Integer | `30` |
|
||||
| `apiserver_audit_log_maxbackup_set` | `audit_log_maxbackup` | Integer | `10` |
|
||||
| `apiserver_audit_log_maxsize_set` | `audit_log_maxsize` | Integer | `100` |
|
||||
| `apiserver_strong_ciphers_only` | `apiserver_strong_ciphers` | List of Strings | `["TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256"]` |
|
||||
| `kubelet_strong_ciphers_only` | `kubelet_strong_ciphers` | List of Strings | See `config.yaml` |
|
||||
|
||||
|
||||
## M365
|
||||
@@ -224,11 +253,13 @@ The following list includes all the Kubernetes checks with configurable variable
|
||||
### Configurable Checks
|
||||
The following list includes all the Microsoft 365 checks with configurable variables that can be changed in the configuration yaml file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|--------------------------------------------------|-----------------|
|
||||
| `entra_admin_users_sign_in_frequency_enabled` | `sign_in_frequency` | Integer |
|
||||
| `teams_external_file_sharing_restricted` | `allowed_cloud_storage_services` | List of Strings |
|
||||
| `exchange_organization_mailtips_enabled` | `recommended_mailtips_large_audience_threshold` | Integer |
|
||||
| Check Name | Value | Type | Default |
|
||||
|--------------------------------------------------------------------|-------------------------------------------------|-----------------|--------------------|
|
||||
| `defender_malware_policy_comprehensive_attachments_filter_applied` | `recommended_blocked_file_types` | List of Strings | See check defaults |
|
||||
| `entra_admin_users_sign_in_frequency_enabled` | `sign_in_frequency` | Integer | `4` |
|
||||
| `exchange_organization_mailtips_enabled` | `recommended_mailtips_large_audience_threshold` | Integer | `25` |
|
||||
| `exchange_user_mailbox_auditing_enabled` | `audit_log_age` | Integer | `90` |
|
||||
| `teams_external_file_sharing_restricted` | `allowed_cloud_storage_services` | List of Strings | `[]` |
|
||||
|
||||
|
||||
## GitHub
|
||||
@@ -236,35 +267,76 @@ The following list includes all the Microsoft 365 checks with configurable varia
|
||||
### Configurable Checks
|
||||
The following list includes all the GitHub checks with configurable variables that can be changed in the configuration yaml file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|--------------------------------------------|---------------------------------------------|---------|
|
||||
| `repository_inactive_not_archived` | `inactive_not_archived_days_threshold` | Integer |
|
||||
| Check Name | Value | Type | Default |
|
||||
|------------------------------------|----------------------------------------|---------|---------|
|
||||
| `repository_inactive_not_archived` | `inactive_not_archived_days_threshold` | Integer | `180` |
|
||||
|
||||
## Vercel
|
||||
|
||||
### Configurable Checks
|
||||
The following list includes all the Vercel checks with configurable variables that can be changed in the configuration YAML file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|-----------------------------------------------------|------------------------------------|-----------------|
|
||||
| `authentication_no_stale_tokens` | `stale_token_threshold_days` | Integer |
|
||||
| `authentication_token_not_expired` | `days_to_expire_threshold` | Integer |
|
||||
| `deployment_production_uses_stable_target` | `stable_branches` | List of Strings |
|
||||
| `domain_ssl_certificate_valid` | `days_to_expire_threshold` | Integer |
|
||||
| `project_environment_no_secrets_in_plain_type` | `secret_suffixes` | List of Strings |
|
||||
| `team_member_role_least_privilege` | `max_owner_percentage` | Integer |
|
||||
| `team_member_role_least_privilege` | `max_owners` | Integer |
|
||||
| `team_no_stale_invitations` | `stale_invitation_threshold_days` | Integer |
|
||||
| Check Name | Value | Type | Default |
|
||||
|------------------------------------------------|-----------------------------------|-----------------|--------------------------------------------------------------------------|
|
||||
| `authentication_no_stale_tokens` | `stale_token_threshold_days` | Integer | `90` |
|
||||
| `authentication_token_not_expired` | `days_to_expire_threshold` | Integer | `7` |
|
||||
| `deployment_production_uses_stable_target` | `stable_branches` | List of Strings | `["main", "master"]` |
|
||||
| `domain_ssl_certificate_valid` | `days_to_expire_threshold` | Integer | `7` |
|
||||
| `project_environment_no_secrets_in_plain_type` | `secret_suffixes` | List of Strings | `["_KEY", "_SECRET", "_TOKEN", "_PASSWORD", "_API_KEY", "_PRIVATE_KEY"]` |
|
||||
| `team_member_role_least_privilege` | `max_owner_percentage` | Integer | `20` |
|
||||
| `team_member_role_least_privilege` | `max_owners` | Integer | `3` |
|
||||
| `team_no_stale_invitations` | `stale_invitation_threshold_days` | Integer | `30` |
|
||||
|
||||
## Okta
|
||||
|
||||
### Configurable Checks
|
||||
The following list includes all the Okta checks with configurable variables that can be changed in the configuration YAML file:
|
||||
|
||||
| Check Name | Value | Type |
|
||||
|---------------------------------------------------------------|------------------------------------|---------|
|
||||
| `application_admin_console_session_idle_timeout_15min` | `okta_admin_console_idle_timeout_max_minutes` | Integer |
|
||||
| `signon_global_session_idle_timeout_15min` | `okta_max_session_idle_minutes` | Integer |
|
||||
| Check Name | Value | Type | Default |
|
||||
|--------------------------------------------------------|-----------------------------------------------|-----------------|---------|
|
||||
| `application_admin_console_session_idle_timeout_15min` | `okta_admin_console_idle_timeout_max_minutes` | Integer | `15` |
|
||||
| `idp_smart_card_dod_approved_ca` | `okta_dod_approved_ca_issuer_patterns` | List of Strings | `[]` |
|
||||
| `signon_global_session_idle_timeout_15min` | `okta_max_session_idle_minutes` | Integer | `15` |
|
||||
| `signon_global_session_lifetime_18h` | `okta_max_session_lifetime_minutes` | Integer | `1080` |
|
||||
| `user_inactivity_automation_35d_enabled` | `okta_user_inactivity_max_days` | Integer | `35` |
|
||||
|
||||
## Alibaba Cloud
|
||||
|
||||
### Configurable Checks
|
||||
The following list includes all the Alibaba Cloud checks with configurable variables that can be changed in the configuration YAML file:
|
||||
|
||||
| Check Name | Value | Type | Default |
|
||||
|--------------------------------------|--------------------------------|---------|---------|
|
||||
| `cs_kubernetes_cluster_check_recent` | `max_cluster_check_days` | Integer | `7` |
|
||||
| `cs_kubernetes_cluster_check_weekly` | `max_cluster_check_days` | Integer | `7` |
|
||||
| `ram_user_console_access_unused` | `max_console_access_days` | Integer | `90` |
|
||||
| `rds_instance_sql_audit_retention` | `min_rds_audit_retention_days` | Integer | `180` |
|
||||
| `sls_logstore_retention_period` | `min_log_retention_days` | Integer | `365` |
|
||||
|
||||
|
||||
## MongoDB Atlas
|
||||
|
||||
### Configurable Checks
|
||||
The following list includes all the MongoDB Atlas checks with configurable variables that can be changed in the configuration YAML file:
|
||||
|
||||
| Check Name | Value | Type | Default |
|
||||
|----------------------------------------------------|---------------------------------------------|---------|---------|
|
||||
| `organizations_service_account_secrets_expiration` | `max_service_account_secret_validity_hours` | Integer | `8` |
|
||||
|
||||
|
||||
## OpenStack
|
||||
|
||||
### Configurable Checks
|
||||
The following list includes all the OpenStack checks with configurable variables that can be changed in the configuration YAML file:
|
||||
|
||||
| Check Name | Value | Type | Default |
|
||||
|---------------------------------------------------|---------------------------|-----------------|---------|
|
||||
| `blockstorage_snapshot_metadata_sensitive_data` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `blockstorage_volume_metadata_sensitive_data` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `compute_instance_metadata_sensitive_data` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
| `image_not_shared_with_multiple_projects` | `image_sharing_threshold` | Integer | `5` |
|
||||
| `objectstorage_container_metadata_sensitive_data` | `secrets_ignore_patterns` | List of Strings | `[]` |
|
||||
|
||||
|
||||
## Config YAML File Structure
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ The checks with this functionality are the following.
|
||||
|
||||
AWS:
|
||||
|
||||
- apigateway\_restapi\_no\_secrets\_in\_stage\_variables
|
||||
- autoscaling\_find\_secrets\_ec2\_launch\_configuration
|
||||
- awslambda\_function\_no\_secrets\_in\_code
|
||||
- awslambda\_function\_no\_secrets\_in\_variables
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
title: 'Azure Resource Group Scope'
|
||||
---
|
||||
|
||||
Prowler supports narrowing security scans to specific resource groups within Azure subscriptions. This is useful when you want to audit only a subset of resources rather than scanning an entire subscription.
|
||||
|
||||
By default, Prowler scans all resource groups it has permission to access. Passing `--azure-resource-group` limits the scan to only the specified resource groups across all accessible subscriptions.
|
||||
|
||||
## Configuring Resource Group Scoped Scans
|
||||
|
||||
To restrict a scan to one or more resource groups, pass them as arguments using the `--azure-resource-group` flag:
|
||||
|
||||
```console
|
||||
prowler azure --az-cli-auth --azure-resource-group <resource-group-1> <resource-group-2> ... <resource-group-N>
|
||||
```
|
||||
|
||||
For example, to scan only `rg-production` and `rg-staging`:
|
||||
|
||||
```console
|
||||
prowler azure --az-cli-auth --azure-resource-group rg-production rg-staging
|
||||
```
|
||||
|
||||
This works with all supported authentication methods:
|
||||
|
||||
```console
|
||||
# Service Principal
|
||||
prowler azure --sp-env-auth --azure-resource-group rg-production
|
||||
|
||||
# Browser
|
||||
prowler azure --browser-auth --tenant-id <tenant-id> --azure-resource-group rg-production
|
||||
|
||||
# Managed Identity
|
||||
prowler azure --managed-identity-auth --azure-resource-group rg-production
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
When `--azure-resource-group` is provided, Prowler validates each specified resource group against all accessible subscriptions. A resource group is included in the scan if it exists in **at least one** subscription.
|
||||
|
||||
- If a resource group is found in one or more subscriptions, it will be scanned in those subscriptions only.
|
||||
- If a resource group is **not found in any** subscription, Prowler logs a warning and skips it.
|
||||
- If **none** of the provided resource groups are found across any subscription, Prowler logs a warning and no resource group scoped checks will run.
|
||||
- Resource group names are matched case-insensitively, so `MyGroup` and `mygroup` are treated as the same group, mirroring Azure's own behavior.
|
||||
|
||||
<Warning>
|
||||
If `--azure-resource-group` is used, checks that apply to specific resources are limited to the relevant resource groups. But if checks that apply to tenant or subscription scope (identity, policy, or subscription-level configuration checks) are involved, then these checks will run in their natural scope.
|
||||
</Warning>
|
||||
@@ -23,6 +23,10 @@ title: 'Getting Started with Kubernetes'
|
||||
|
||||
For Kubernetes, Prowler App uses a `kubeconfig` file to authenticate. Paste the contents of your `kubeconfig` file into the `Kubeconfig content` field.
|
||||
|
||||
<Note>
|
||||
Kubeconfigs that use `users[].user.exec` authentication are not supported in Prowler Cloud/App. For security reasons, Prowler Cloud does not run commands declared by uploaded kubeconfigs. Use kubeconfig credentials that do not rely on `exec` authentication, such as the ServiceAccount token flow documented below.
|
||||
</Note>
|
||||
|
||||
By default, the `kubeconfig` file is located at `~/.kube/config`.
|
||||
|
||||

|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
---
|
||||
title: "Okta Rate Limit Configuration in Prowler"
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
Prowler's Okta Provider manages API rate limits with two complementary controls:
|
||||
|
||||
- **Request throttling (proactive):** Prowler paces outbound requests through a shared limiter so scans stay under Okta's rate limits and rarely trigger a rate-limit response in the first place.
|
||||
- **Retries (reactive):** When Okta still returns a rate-limit response (HTTP 429), the official Okta Python SDK reads the `X-Rate-Limit-Reset` header and waits until the window resets before retrying. This acts as a safety net for occasional bursts.
|
||||
|
||||
Both controls are configurable through the configuration file or command line flags.
|
||||
|
||||
## Request Throttling (Requests per Second)
|
||||
|
||||
Throttling is the primary control for avoiding rate limits. Prowler limits the aggregate number of Okta API requests per second across every service in a scan.
|
||||
|
||||
### Using the Command Line Flag
|
||||
|
||||
```bash
|
||||
prowler okta --okta-requests-per-second 4
|
||||
```
|
||||
|
||||
Set the value to `0` to disable throttling.
|
||||
|
||||
### Using the Configuration File
|
||||
|
||||
```yaml
|
||||
okta:
|
||||
# Maximum aggregate Okta API requests per second. Default: 4. Set to 0 to disable.
|
||||
okta_requests_per_second: 4
|
||||
```
|
||||
|
||||
Okta enforces rate limits per endpoint, so this single global cap is a deliberately simple control. Lower the value if scans still hit limits on large organizations; raise it to scan faster when the organization has generous limits.
|
||||
|
||||
## Retries
|
||||
|
||||
Retries cover the cases throttling does not prevent, such as short bursts or per-endpoint limits lower than the global cap.
|
||||
|
||||
### Using the Command Line Flag
|
||||
|
||||
```bash
|
||||
prowler okta --okta-retries-max-attempts 8
|
||||
```
|
||||
|
||||
### Using the Configuration File
|
||||
|
||||
```yaml
|
||||
okta:
|
||||
# Maximum retries on HTTP 429. Default: 5.
|
||||
okta_max_retries: 8
|
||||
# Per-request timeout in seconds. Default: 300.
|
||||
okta_request_timeout: 300
|
||||
```
|
||||
|
||||
The command line flags override the configuration file values.
|
||||
|
||||
## How It Works
|
||||
|
||||
- **Automatic detection:** The Okta SDK retries the retryable statuses 429, 503, and 504.
|
||||
- **Reset-aware backoff:** On a 429 response the SDK sleeps until the `X-Rate-Limit-Reset` window before each retry, rather than using a fixed delay.
|
||||
- **Bounded attempts:** `okta_max_retries` caps how many times a single request is retried. The Okta SDK default is 2, which is often too low for large organizations, so Prowler defaults to 5.
|
||||
|
||||
## Request Timeout
|
||||
|
||||
The `okta_request_timeout` setting plays a dual role in the Okta SDK:
|
||||
|
||||
- It is the per-request socket timeout, bounding how long a single HTTP call can hang.
|
||||
- It is also the total wall-clock budget for the whole retry-and-backoff loop of one request.
|
||||
|
||||
For this reason, the value defaults to 300 seconds rather than 0 (no timeout). A value of 0 leaves hung connections unbounded, while a value that is too low cuts the rate-limit waits short and reintroduces the errors. As a guideline, keep `okta_request_timeout` greater than or equal to `okta_max_retries` multiplied by 60 when raising the retry count, because Okta reset windows are typically up to one minute.
|
||||
|
||||
## Error Example Handled
|
||||
|
||||
```
|
||||
Okta HTTP 429: Too Many Requests. Hit rate limit. Retry request in 42 seconds.
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
### Debug Logging
|
||||
|
||||
To confirm that throttling and retries are active, run a scan with debug logging:
|
||||
|
||||
```bash
|
||||
prowler okta --okta-requests-per-second 4 --log-level DEBUG --log-file debuglogs.txt
|
||||
```
|
||||
|
||||
### Check the Messages
|
||||
|
||||
```bash
|
||||
grep -i "throttling\|rate limit\|retry" debuglogs.txt
|
||||
```
|
||||
|
||||
### Expected Output
|
||||
|
||||
When throttling is enabled, Prowler logs the configured rate at startup:
|
||||
|
||||
```
|
||||
Okta request throttling enabled at 4 req/s
|
||||
```
|
||||
|
||||
If a rate limit is still hit, the SDK logs the backoff:
|
||||
|
||||
```
|
||||
Hit rate limit. Retry request in 42 seconds.
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If scans continue to hit rate limits:
|
||||
|
||||
1. Lower `--okta-requests-per-second` so requests are paced more conservatively.
|
||||
2. Raise `--okta-retries-max-attempts` (and keep `okta_request_timeout` proportionally large) so the safety net absorbs more bursts.
|
||||
3. Review the rate-limit allocation for the Okta organization and request an increase if needed.
|
||||
4. Verify throttling and retry behavior with debug logging.
|
||||
|
||||
## Official References
|
||||
|
||||
- [Okta Rate Limits](https://developer.okta.com/docs/reference/rate-limits/)
|
||||
- [Okta SDK for Python](https://github.com/okta/okta-sdk-python)
|
||||
@@ -4,14 +4,13 @@ description: 'Create email alerts from Prowler Cloud findings to monitor relevan
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.26.0" />
|
||||
|
||||
Alerts notify recipients by email when security findings match saved filter conditions. Use Alerts to track high-priority findings, monitor specific providers or services, and keep teams informed about scan results that match defined criteria.
|
||||
|
||||
<Note>
|
||||
This feature is available exclusively in **Prowler Cloud** and **Prowler Enterprise** with a [subscription](https://prowler.com/pricing).
|
||||
</Note>
|
||||
<SubscriptionBanner />
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
@@ -21,13 +21,13 @@ By mapping these relationships as a graph, Attack Paths reveals risks that indiv
|
||||
The following prerequisites are required for Attack Paths:
|
||||
|
||||
- **An AWS provider is configured** with valid credentials in Prowler App. For setup instructions, see [Getting Started with AWS](/user-guide/providers/aws/getting-started-aws).
|
||||
- **At least one scan has completed** on the configured AWS provider. Attack Paths scans run automatically alongside regular security scans, no separate configuration is required.
|
||||
- **At least one scan has completed** on the configured AWS provider and produced graph data. Attack Paths scans run automatically alongside regular security scans, no separate configuration is required.
|
||||
|
||||
## How Attack Paths Scans Work
|
||||
|
||||
Attack Paths scans are generated automatically when a security scan runs on an AWS provider. Each completed scan produces graph data that maps relationships between IAM principals, policies, trust configurations, and other resources.
|
||||
|
||||
Once the scan finishes and the graph data is ready, the scan appears in the Attack Paths scan table with a **Completed** status. Scans that are still processing display as **Executing** or **Scheduled**.
|
||||
Once the scan finishes and graph data is ready, the scan appears in the Attack Paths scan table with a **Completed** status and a check in the **Graph** column. Scans that are still queued or running remain visible, but they cannot be selected until graph data is ready.
|
||||
|
||||
<Note>
|
||||
Since Prowler scans all configured providers every **24 hours** by default,
|
||||
@@ -41,25 +41,28 @@ To open Attack Paths, click **Attack Paths** in the left navigation menu.
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/navigation.png"
|
||||
alt="Attack Paths navigation menu entry"
|
||||
width="700"
|
||||
width="320"
|
||||
/>
|
||||
|
||||
The main interface is divided into two areas:
|
||||
The Attack Paths page guides you through the workflow on one page:
|
||||
|
||||
- **Left panel:** A table listing all available Attack Paths scans
|
||||
- **Right panel:** The query selector, parameter form, and execute controls
|
||||
- Select a scan with graph data.
|
||||
- Choose a built-in query or a custom openCypher query.
|
||||
- Add parameters when the selected query requires them.
|
||||
- Execute the query and explore the resulting graph.
|
||||
|
||||
## Selecting a Scan
|
||||
|
||||
The scans table displays all Attack Paths scans with the following columns:
|
||||
|
||||
- **Provider / Account:** The AWS provider alias and account identifier
|
||||
- **Last scan date:** When the scan completed
|
||||
- **Status:** Current state of the scan (Completed, Executing, Scheduled, or Failed)
|
||||
- **Progress:** Completion percentage for in-progress scans
|
||||
- **Duration:** Total scan time
|
||||
- **Select:** A radio button used to choose a scan. The radio button is disabled when graph data is not available.
|
||||
- **Provider:** The AWS provider alias and account identifier.
|
||||
- **Last Scan Date:** When the scan completed.
|
||||
- **Status:** Current state of the scan, such as **Completed**, **Executing**, **Scheduled**, or **Failed**.
|
||||
- **Graph:** Whether Attack Paths graph data is available for the scan.
|
||||
- **Duration:** Total scan time.
|
||||
|
||||
To select a scan for analysis, click **Select** on any row with a **Completed** status.
|
||||
To select a scan for analysis, click the radio button on any row with a **Completed** status and available graph data.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/scan-list-table.png"
|
||||
@@ -68,19 +71,18 @@ To select a scan for analysis, click **Select** on any row with a **Completed**
|
||||
/>
|
||||
|
||||
<Note>
|
||||
Only scans with a **Completed** status and ready graph data can be selected.
|
||||
Scans that are still executing or have failed appear with disabled action
|
||||
buttons.
|
||||
Only scans with graph data can be selected. Disabled rows include a tooltip
|
||||
that explains why the graph is not available yet.
|
||||
</Note>
|
||||
|
||||
## Choosing a Query
|
||||
|
||||
After selecting a scan, the right panel activates a query dropdown. Each query targets a specific type of privilege escalation or misconfiguration pattern.
|
||||
After selecting a scan, the query selector becomes available. Each query targets a specific privilege escalation, exposure, inventory, or misconfiguration pattern.
|
||||
|
||||
To choose a query, click the dropdown and select from the available options. Each option displays:
|
||||
|
||||
- **Query name:** A descriptive title (e.g., "IAM Privilege Escalation via AssumeRole")
|
||||
- **Short description:** A brief summary of what the query detects
|
||||
- **Query name:** A descriptive title, such as **Internet-Exposed EC2 with Sensitive S3 Access**.
|
||||
- **Short description:** A brief summary of what the query detects.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/query-selector.png"
|
||||
@@ -88,16 +90,17 @@ To choose a query, click the dropdown and select from the available options. Eac
|
||||
width="700"
|
||||
/>
|
||||
|
||||
Once selected, a description card appears below the dropdown with additional context about the query, including attribution links to external references when available.
|
||||
Once selected, a description panel appears below the dropdown with more context about the query.
|
||||
|
||||
## Configuring Query Parameters
|
||||
|
||||
Some queries accept optional or required parameters to narrow the scope of the analysis. When a query has parameters, a dynamic form appears below the query description.
|
||||
Some queries accept optional or required parameters to narrow the scope of the analysis. When a query has parameters, a form appears below the query description.
|
||||
|
||||
- **Required fields** are marked with an asterisk (\*) and must be filled before executing
|
||||
- **Optional fields** refine the query results but are not mandatory
|
||||
- **Required fields** are marked with an asterisk (\*) and must be filled before executing.
|
||||
- **Optional fields** refine the query results but are not mandatory.
|
||||
- Queries without parameters show no parameter form.
|
||||
|
||||
If a query requires no parameters, the form displays a message confirming that the query is ready to execute.
|
||||
For example, **Internet-Exposed EC2 with Sensitive S3 Access** uses **Tag key** and **Tag value** fields to identify sensitive S3 buckets.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/query-parameters.png"
|
||||
@@ -117,7 +120,7 @@ Custom queries are sandboxed to keep the graph database safe and responsive:
|
||||
|
||||
- **Read-only:** Only read operations are allowed. Statements that mutate the graph (`CREATE`, `MERGE`, `SET`, `DELETE`, `REMOVE`, `DROP`, `LOAD CSV`, `CALL { ... }` writes, etc.) are rejected before execution.
|
||||
- **Length limit:** Each query is capped at **10,000 characters**.
|
||||
- **Scoped to the selected scan:** Results are automatically scoped to the provider and scan selected on the left panel. There is no need to filter by tenant or scan identifier in the query body.
|
||||
- **Scoped to the selected scan:** Results are automatically scoped to the provider and scan selected in the scan table. There is no need to filter by tenant or scan identifier in the query body.
|
||||
|
||||
### Example Queries
|
||||
|
||||
@@ -284,53 +287,75 @@ In addition to the upstream schema, Prowler enriches the graph with:
|
||||
|
||||
## Executing a Query
|
||||
|
||||
To run the selected query against the scan data, click **Execute Query**. The button displays a loading state while the query processes.
|
||||
To run the selected query against the scan data, click **Execute Query**. The button is disabled until a query is selected and all required parameters are valid.
|
||||
|
||||
The button displays a loading state while the query runs. After the query completes, the graph appears below the query builder.
|
||||
|
||||
If the query returns no results, an informational message appears. Common reasons include:
|
||||
|
||||
- **No matching patterns found:** The scanned environment does not contain the privilege escalation chain the query targets
|
||||
- **Insufficient permissions:** The scan credentials may not have captured all the data the query needs
|
||||
- **No matching patterns found:** The scanned environment does not contain the pattern the query targets.
|
||||
- **Not enough permissions:** The scan credentials may not have captured all the data the query needs.
|
||||
- **Server unavailable:** The graph service may be temporarily unavailable.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/execute-query.png"
|
||||
alt="Attack Paths right panel with query selected and execute button"
|
||||
alt="Attack Paths query builder with query selected and execute button"
|
||||
width="700"
|
||||
/>
|
||||
|
||||
## Exploring the Graph
|
||||
|
||||
After a successful execution, the graph visualization renders below the query builder in a full-width panel. The graph maps relationships between cloud resources, IAM entities, and security findings.
|
||||
After a successful execution, the graph visualization renders below the query builder. The graph maps relationships between cloud resources, IAM entities, public exposure, and security findings.
|
||||
|
||||
### Node Types
|
||||
|
||||
- **Resource nodes** (rounded pills): Represent cloud resources such as IAM roles, policies, EC2 instances, and S3 buckets. Each resource type has a distinct color.
|
||||
- **Finding nodes** (hexagons): Represent Prowler security findings linked to resources in the graph. Colors indicate severity level (critical, high, medium, low).
|
||||
- **Provider root nodes:** Represent the AWS account or provider root for the selected scan.
|
||||
- **Resource nodes:** Represent cloud resources such as IAM roles, policies, EC2 instances, security groups, and S3 buckets.
|
||||
- **Internet nodes:** Represent exposure from the public internet.
|
||||
- **Finding nodes:** Represent Prowler findings linked to resources. Finding colors indicate risk level, such as critical, high, medium, or low.
|
||||
|
||||
### Edge Types
|
||||
|
||||
- **Solid lines:** Direct relationships between resources (e.g., a role attached to a policy)
|
||||
- **Dashed lines:** Connections between resources and their associated findings
|
||||
- **Normal edges:** Direct relationships between graph nodes, such as role-to-policy or resource-to-security-group relationships.
|
||||
- **Finding edges:** Dashed relationships between resources and their associated findings.
|
||||
- **Highlighted paths:** Green edges that show the active path when you hover a node or focus a finding.
|
||||
|
||||
A **legend** at the bottom of the graph lists all node types and edge types present in the current view.
|
||||
The standard graph view includes a minimap and a legend below the canvas. The legend shows the provider roots, visible node types, finding risk levels, node states, and edge types present in the current view.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/graph-visualization.png"
|
||||
alt="Attack Paths graph showing nodes, edges, and legend"
|
||||
alt="Attack Paths graph showing nodes and edges"
|
||||
width="700"
|
||||
/>
|
||||
|
||||
## Interacting with the Graph
|
||||
|
||||
### Filtering by Node
|
||||
The graph banner describes the main interactions:
|
||||
|
||||
Click any node in the graph to filter the view and display only paths that pass through that node. When a filter is active:
|
||||
- Click a finding to focus its connected path.
|
||||
- Click a resource with findings to show or hide its related findings.
|
||||
- Hover a node to highlight its connected path.
|
||||
|
||||
- An information banner shows which node is selected
|
||||
- Click **Back to Full View** to restore the complete graph
|
||||
### Showing Related Findings
|
||||
|
||||
Resource nodes with related findings are clickable. Click one of these resources to show its finding nodes. Click the resource again to hide them.
|
||||
|
||||
The graph automatically fits the selected resource and its related findings when the findings are shown.
|
||||
|
||||
### Focusing a Finding Path
|
||||
|
||||
Click a finding node to focus the graph on the path connected to that finding. When the graph is focused:
|
||||
|
||||
- The graph shows **Back to Full View**.
|
||||
- The status banner shows the selected finding.
|
||||
- The graph keeps only the connected path in view.
|
||||
- The finding detail drawer opens.
|
||||
|
||||
After you close the drawer, the graph remains focused on the selected path.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/graph-filtered.png"
|
||||
alt="Attack Paths graph filtered to show paths through a selected node"
|
||||
alt="Attack Paths graph focused on a selected finding path"
|
||||
width="700"
|
||||
/>
|
||||
|
||||
@@ -339,29 +364,29 @@ Click any node in the graph to filter the view and display only paths that pass
|
||||
The toolbar in the top-right corner of the graph provides:
|
||||
|
||||
- **Zoom in / Zoom out:** Adjust the zoom level
|
||||
- **Fit to screen:** Reset the view to fit all nodes
|
||||
- **Export:** Download the current graph as an SVG file
|
||||
- **Fullscreen:** Open the graph in a full-screen modal with a side-by-side node detail panel
|
||||
- **Fit graph to view:** Reset the view to fit the visible graph
|
||||
- **Export graph:** Download the current graph as a PNG file
|
||||
- **Fullscreen:** Open the graph in a full-size modal
|
||||
|
||||
<Note>
|
||||
Use **Ctrl + Scroll** (or **Cmd + Scroll** on macOS) to zoom directly within
|
||||
the graph area.
|
||||
</Note>
|
||||
|
||||
## Viewing Node Details
|
||||
## Viewing Finding Details
|
||||
|
||||
Click any node to open the **Node Details** panel below the graph. This panel displays:
|
||||
Click a finding node to open the finding detail drawer. The drawer uses the same finding detail layout as the Findings page and includes:
|
||||
|
||||
- **Node type:** The resource category (e.g., "IAM Role," "EC2 Instance")
|
||||
- **Properties:** All attributes of the selected node, including identifiers, timestamps, and configuration details
|
||||
- **Related findings** (for resource nodes): A list of Prowler findings linked to the resource, with severity, title, and status
|
||||
- **Affected resources** (for finding nodes): A list of resources associated with the finding
|
||||
- The finding title, status, and severity.
|
||||
- The affected resource summary.
|
||||
- Overview, remediation, evidence, related findings, scans, and events tabs when data is available.
|
||||
- A Lighthouse AI action when the account has access to Lighthouse AI.
|
||||
|
||||
For finding nodes, a "View Finding" button links directly to the finding detail page for further investigation.
|
||||
Resource nodes do not open a node detail panel. When a resource has related findings, clicking it expands or collapses those finding nodes in the graph.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/node-details.png"
|
||||
alt="Attack Paths node detail panel showing properties and related findings"
|
||||
alt="Attack Paths finding detail drawer"
|
||||
width="700"
|
||||
/>
|
||||
|
||||
@@ -369,16 +394,28 @@ For finding nodes, a "View Finding" button links directly to the finding detail
|
||||
|
||||
To expand the graph for detailed exploration, click the fullscreen icon in the graph toolbar. The fullscreen modal provides:
|
||||
|
||||
- The full graph visualization with all zoom and export controls
|
||||
- A side panel for node details that appears when a node is selected
|
||||
- All filtering and interaction capabilities available in the standard view
|
||||
- The graph in a full-size modal.
|
||||
- The same zoom, fit, and export controls.
|
||||
- The same node expansion, finding focus, hover highlight, and minimap interactions available in the standard view.
|
||||
|
||||
<img
|
||||
src="/images/prowler-app/attack-paths/fullscreen-mode.png"
|
||||
alt="Attack Paths fullscreen mode with graph and node detail side panel"
|
||||
alt="Attack Paths fullscreen graph mode"
|
||||
width="700"
|
||||
/>
|
||||
|
||||
## Available Queries
|
||||
|
||||
The query selector includes custom openCypher and built-in AWS queries for common security investigation workflows. Available queries are loaded from the selected scan and may change as new query packs are added.
|
||||
|
||||
Available queries include:
|
||||
|
||||
- **Custom openCypher query:** Write and run a read-only graph query.
|
||||
- **Exposure queries:** Find internet-exposed EC2 instances, load balancers, open security groups, and resources by public IP.
|
||||
- **Inventory queries:** List resources such as RDS instances.
|
||||
- **Misconfiguration queries:** Find unencrypted RDS instances, public S3 buckets, and wildcard IAM statements.
|
||||
- **Privilege escalation queries:** Detect IAM and AWS service paths based on known attack techniques, including queries based on [pathfinding.cloud](https://pathfinding.cloud) research by Datadog.
|
||||
|
||||
## Using Attack Paths with the MCP Server and Lighthouse AI
|
||||
|
||||
Attack Paths capabilities are also available through the [Prowler MCP Server](/getting-started/products/prowler-mcp), enabling interaction with Attack Paths data via AI assistants like Claude Desktop, Cursor, and other MCP clients.
|
||||
@@ -387,10 +424,10 @@ Attack Paths capabilities are also available through the [Prowler MCP Server](/g
|
||||
|
||||
The following MCP tools are available for Attack Paths:
|
||||
|
||||
- **`prowler_app_list_attack_paths_scans`** - List and filter Attack Paths scans
|
||||
- **`prowler_app_list_attack_paths_queries`** - Discover available queries for a completed scan
|
||||
- **`prowler_app_run_attack_paths_query`** - Execute a query and retrieve graph results with nodes and relationships
|
||||
- **`prowler_app_get_attack_paths_cartography_schema`** - Retrieve the Cartography graph schema for custom openCypher queries
|
||||
- **`prowler_app_list_attack_paths_scans`** - List and filter Attack Paths scans.
|
||||
- **`prowler_app_list_attack_paths_queries`** - Discover available queries for a completed scan.
|
||||
- **`prowler_app_run_attack_paths_query`** - Execute a query and retrieve graph results with nodes and relationships.
|
||||
- **`prowler_app_get_attack_paths_cartography_schema`** - Retrieve the Cartography graph schema for custom openCypher queries.
|
||||
|
||||
### Example Questions
|
||||
|
||||
@@ -405,110 +442,6 @@ Ask through the MCP Server or Lighthouse AI:
|
||||
- "Are there any CloudFormation stacks that could be hijacked for privilege escalation?"
|
||||
- "Show me all roles that can be assumed for lateral movement"
|
||||
|
||||
### Supported Queries
|
||||
|
||||
Attack Paths currently supports the following built-in queries for AWS:
|
||||
|
||||
#### Custom Attack Path Queries
|
||||
|
||||
| Query | Description |
|
||||
| ------------------------------------------------- | ---------------------------------------------------------------------------------------- |
|
||||
| **Internet-Exposed EC2 with Sensitive S3 Access** | Find SSH-exposed EC2 instances that can assume roles to read tagged sensitive S3 buckets |
|
||||
|
||||
#### Basic Resource Queries
|
||||
|
||||
| Query | Description |
|
||||
| ------------------------------------------- | ------------------------------------------------------------------- |
|
||||
| **RDS Instances Inventory** | List all provisioned RDS database instances in the account |
|
||||
| **Unencrypted RDS Instances** | Find RDS instances with storage encryption disabled |
|
||||
| **S3 Buckets with Anonymous Access** | Find S3 buckets that allow anonymous access |
|
||||
| **IAM Statements Allowing All Actions** | Find IAM policy statements that allow all actions via wildcard (\*) |
|
||||
| **IAM Statements Allowing Policy Deletion** | Find IAM policy statements that allow iam:DeletePolicy |
|
||||
| **IAM Statements Allowing Create Actions** | Find IAM policy statements that allow any create action |
|
||||
|
||||
#### Network Exposure Queries
|
||||
|
||||
| Query | Description |
|
||||
| ----------------------------------------------------- | ----------------------------------------------------------------------------------- |
|
||||
| **Internet-Exposed EC2 Instances** | Find EC2 instances flagged as exposed to the internet |
|
||||
| **Open Security Groups on Internet-Facing Resources** | Find internet-facing resources with security groups allowing inbound from 0.0.0.0/0 |
|
||||
| **Internet-Exposed Classic Load Balancers** | Find Classic Load Balancers exposed to the internet with their listeners |
|
||||
| **Internet-Exposed ALB/NLB Load Balancers** | Find ELBv2 (ALB/NLB) load balancers exposed to the internet with their listeners |
|
||||
| **Resource Lookup by Public IP** | Find the AWS resource associated with a given public IP address |
|
||||
|
||||
#### Privilege Escalation Queries
|
||||
|
||||
These queries are based on research from [pathfinding.cloud](https://pathfinding.cloud) by Datadog.
|
||||
|
||||
| Query | Description |
|
||||
| -------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **App Runner Service Creation with Privileged Role (APPRUNNER-001)** | Create an App Runner service with a privileged IAM role to gain its permissions |
|
||||
| **App Runner Service Update for Role Access (APPRUNNER-002)** | Update an existing App Runner service to leverage its already-attached privileged role |
|
||||
| **Bedrock Code Interpreter with Privileged Role (BEDROCK-001)** | Create a Bedrock AgentCore Code Interpreter with a privileged role attached |
|
||||
| **Bedrock Code Interpreter Session Hijacking (BEDROCK-002)** | Start a session on an existing Bedrock code interpreter to exfiltrate its privileged role credentials |
|
||||
| **CloudFormation Stack Creation with Privileged Role (CLOUDFORMATION-001)** | Create a CloudFormation stack with a privileged role to provision arbitrary AWS resources |
|
||||
| **CloudFormation Stack Update for Role Access (CLOUDFORMATION-002)** | Update an existing CloudFormation stack to leverage its already-attached privileged service role |
|
||||
| **CloudFormation StackSet Creation with Privileged Role (CLOUDFORMATION-003)** | Create a CloudFormation StackSet with a privileged execution role to provision arbitrary resources across accounts |
|
||||
| **CloudFormation StackSet Update with Privileged Role (CLOUDFORMATION-004)** | Update an existing CloudFormation StackSet to inject malicious resources using a privileged execution role |
|
||||
| **CloudFormation Change Set Privilege Escalation (CLOUDFORMATION-005)** | Create and execute a change set on an existing stack to leverage its privileged service role |
|
||||
| **CodeBuild Project Creation with Privileged Role (CODEBUILD-001)** | Create a CodeBuild project with a privileged role to execute arbitrary code via a malicious buildspec |
|
||||
| **CodeBuild Buildspec Override for Role Access (CODEBUILD-002)** | Start a build on an existing CodeBuild project with a buildspec override to execute code with its privileged role |
|
||||
| **CodeBuild Batch Buildspec Override for Role Access (CODEBUILD-003)** | Start a batch build on an existing CodeBuild project with a buildspec override to execute code with its privileged role |
|
||||
| **CodeBuild Batch Project Creation with Privileged Role (CODEBUILD-004)** | Create a CodeBuild project configured for batch builds with a privileged role to execute arbitrary code via a malicious buildspec |
|
||||
| **Data Pipeline Creation with Privileged Role (DATAPIPELINE-001)** | Create a Data Pipeline with a privileged role to execute arbitrary commands on provisioned infrastructure |
|
||||
| **EC2 Instance Launch with Privileged Role (EC2-001)** | Launch EC2 instances with privileged IAM roles to gain their permissions via IMDS |
|
||||
| **EC2 Role Hijacking via UserData Injection (EC2-002)** | Inject malicious scripts into EC2 instance userData to gain the attached role's permissions |
|
||||
| **Spot Instance Launch with Privileged Role (EC2-003)** | Launch EC2 Spot Instances with privileged IAM roles to gain their permissions via IMDS |
|
||||
| **Launch Template Poisoning for Role Access (EC2-004)** | Inject malicious userData into launch templates that reference privileged roles, no PassRole needed |
|
||||
| **EC2 Instance Connect SSH Access for Role Credentials (EC2INSTANCECONNECT-003)** | Push a temporary SSH key to an EC2 instance via Instance Connect to access its attached role credentials through IMDS |
|
||||
| **ECS Service Creation with Privileged Role (ECS-001 - New Cluster)** | Create an ECS cluster and service with a privileged Fargate task role to execute arbitrary code |
|
||||
| **ECS Task Execution with Privileged Role (ECS-002 - New Cluster)** | Create an ECS cluster and run a one-off Fargate task with a privileged role to execute arbitrary code |
|
||||
| **ECS Service Creation with Privileged Role (ECS-003 - Existing Cluster)** | Deploy a Fargate service with a privileged role on an existing ECS cluster |
|
||||
| **ECS Task Execution with Privileged Role (ECS-004 - Existing Cluster)** | Run a one-off Fargate task with a privileged role on an existing ECS cluster |
|
||||
| **ECS Task Start with Privileged Role on EC2 (ECS-005 - Existing Cluster)** | Register a task definition with a privileged role and start it on an EC2 container instance to execute arbitrary code |
|
||||
| **ECS Exec Container Hijacking for Role Credentials (ECS-006)** | Shell into a running ECS container via ECS Exec to steal the attached task role's credentials |
|
||||
| **Glue Dev Endpoint with Privileged Role (GLUE-001)** | Create a Glue development endpoint with a privileged role attached to gain its permissions |
|
||||
| **Glue Dev Endpoint SSH Hijacking via Update (GLUE-002)** | Update an existing Glue development endpoint to inject an SSH public key and access its attached role credentials |
|
||||
| **Glue Job Creation with Privileged Role (GLUE-003)** | Create a Glue job with a privileged role and start it to execute arbitrary code with that role's permissions |
|
||||
| **Glue Job Creation with Scheduled Trigger and Privileged Role (GLUE-004)** | Create a Glue job with a privileged role and a scheduled trigger to persistently execute arbitrary code |
|
||||
| **Glue Job Hijacking via Update with Privileged Role (GLUE-005)** | Update an existing Glue job to attach a privileged role and inject malicious code, then start it to gain that role's permissions |
|
||||
| **Glue Job Hijacking with Scheduled Trigger and Privileged Role (GLUE-006)** | Update an existing Glue job to attach a privileged role and inject malicious code, then create a scheduled trigger for persistent automated execution |
|
||||
| **Policy Version Override for Self-Escalation (IAM-001)** | Create a new version of an attached policy with administrative permissions, instantly escalating the principal's own privileges |
|
||||
| **Access Key Creation for Lateral Movement (IAM-002)** | Create access keys for other IAM users to gain their permissions and move laterally across the account |
|
||||
| **Access Key Rotation Attack for Lateral Movement (IAM-003)** | Delete and recreate access keys for other IAM users to bypass the two-key limit and gain their permissions |
|
||||
| **Console Login Profile Creation for Lateral Movement (IAM-004)** | Create console login profiles for other IAM users to access the AWS Console with their permissions |
|
||||
| **Inline Policy Injection for Self-Escalation (IAM-005)** | Attach an inline policy with administrative permissions to your own role, instantly escalating privileges |
|
||||
| **Console Password Override for Lateral Movement (IAM-006)** | Change the console password of other IAM users to log in as them and gain their permissions |
|
||||
| **Inline Policy Injection on User for Self-Escalation (IAM-007)** | Attach an inline policy with administrative permissions to your own IAM user, instantly escalating privileges |
|
||||
| **Managed Policy Attachment on User for Self-Escalation (IAM-008)** | Attach existing managed policies with administrative permissions to your own IAM user, instantly escalating privileges |
|
||||
| **Managed Policy Attachment on Role for Self-Escalation (IAM-009)** | Attach existing managed policies with administrative permissions to your own IAM role, instantly escalating privileges |
|
||||
| **Managed Policy Attachment on Group for Self-Escalation (IAM-010)** | Attach existing managed policies with administrative permissions to a group you belong to, escalating privileges for all group members |
|
||||
| **Inline Policy Injection on Group for Self-Escalation (IAM-011)** | Attach an inline policy with administrative permissions to a group you belong to, escalating privileges for all group members |
|
||||
| **Trust Policy Hijacking for Role Assumption (IAM-012)** | Modify a role's trust policy to allow yourself to assume it, gaining the role's permissions |
|
||||
| **Group Membership Hijacking for Privilege Escalation (IAM-013)** | Add yourself to a privileged IAM group to inherit its permissions, gaining access to all policies attached to the group |
|
||||
| **Managed Policy Attachment with Role Assumption for Lateral Movement (IAM-014)** | Attach administrative managed policies to another role you can assume, then assume it to gain elevated privileges |
|
||||
| **Managed Policy Attachment with Access Key Creation for Lateral Movement (IAM-015)** | Attach administrative managed policies to another IAM user and create access keys for them to gain programmatic access with elevated privileges |
|
||||
| **Policy Version Override with Role Assumption for Lateral Movement (IAM-016)** | Create a new version of a customer-managed policy attached to another role with administrative permissions, then assume that role to gain elevated access |
|
||||
| **Inline Policy Injection with Role Assumption for Lateral Movement (IAM-017)** | Attach an inline policy with administrative permissions to another role you can assume, then assume it to gain elevated privileges |
|
||||
| **Inline Policy Injection with Access Key Creation for Lateral Movement (IAM-018)** | Attach an inline policy with administrative permissions to another IAM user and create access keys for them to gain programmatic access with elevated privileges |
|
||||
| **Managed Policy Attachment with Trust Policy Hijacking for Privilege Escalation (IAM-019)** | Attach administrative managed policies to a role and modify its trust policy to allow yourself to assume it, gaining elevated privileges without prior assume-role access |
|
||||
| **Policy Version Override with Trust Policy Hijacking for Privilege Escalation (IAM-020)** | Create a new version of a customer-managed policy attached to a role with administrative permissions and modify its trust policy to assume it, without prior assume-role access |
|
||||
| **Inline Policy Injection with Trust Policy Hijacking for Privilege Escalation (IAM-021)** | Add an inline policy with administrative permissions to a role and modify its trust policy to allow yourself to assume it, gaining elevated privileges without prior assume-role access |
|
||||
| **Lambda Function Creation with Privileged Role (LAMBDA-001)** | Create a Lambda function with a privileged IAM role and invoke it to execute code with that role's permissions |
|
||||
| **Lambda Function Creation with Event Source Trigger (LAMBDA-002)** | Create a Lambda function with a privileged IAM role and an event source mapping to trigger it automatically, executing code with the role's permissions |
|
||||
| **Lambda Function Code Injection (LAMBDA-003)** | Modify the code of an existing Lambda function to execute arbitrary commands with the function's execution role permissions |
|
||||
| **Lambda Function Code Injection with Direct Invocation (LAMBDA-004)** | Modify the code of an existing Lambda function and invoke it directly to execute arbitrary commands with the function's execution role permissions |
|
||||
| **Lambda Function Code Injection with Resource Policy Grant (LAMBDA-005)** | Modify the code of an existing Lambda function and grant yourself invocation permission via its resource-based policy to execute code with the function's execution role |
|
||||
| **Lambda Function Creation with Resource Policy Invocation (LAMBDA-006)** | Create a Lambda function with a privileged IAM role and grant yourself invocation permission via its resource-based policy to execute code with the role's permissions |
|
||||
| **SageMaker Notebook Creation with Privileged Role (SAGEMAKER-001)** | Create a SageMaker notebook instance with a privileged IAM role to execute arbitrary code with the role's permissions via the Jupyter environment |
|
||||
| **SageMaker Training Job Creation with Privileged Role (SAGEMAKER-002)** | Create a SageMaker training job with a privileged IAM role to execute arbitrary container code with the role's permissions |
|
||||
| **SageMaker Processing Job Creation with Privileged Role (SAGEMAKER-003)** | Create a SageMaker processing job with a privileged IAM role to execute arbitrary container code with the role's permissions |
|
||||
| **SageMaker Presigned Notebook URL for Privilege Escalation (SAGEMAKER-004)** | Generate a presigned URL to access an existing SageMaker notebook instance and execute code with its execution role's permissions |
|
||||
| **SageMaker Notebook Lifecycle Config Injection (SAGEMAKER-005)** | Inject a malicious lifecycle configuration into an existing SageMaker notebook to execute code with the notebook's execution role during startup |
|
||||
| **SSM Session Access for EC2 Role Credentials (SSM-001)** | Start an SSM session on an EC2 instance to access its attached role credentials through IMDS |
|
||||
| **SSM Send Command for EC2 Role Credentials (SSM-002)** | Execute commands on an EC2 instance via SSM Run Command to access its attached role credentials through IMDS |
|
||||
| **Role Assumption for Privilege Escalation (STS-001)** | Assume IAM roles with elevated permissions by exploiting bidirectional trust between the starting principal and the target role |
|
||||
|
||||
These tools enable workflows such as:
|
||||
|
||||
- Asking an AI assistant to identify privilege escalation paths in a specific AWS account
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
---
|
||||
title: "Findings Triage"
|
||||
description: "Track finding review status and team notes in Prowler Cloud."
|
||||
---
|
||||
|
||||
import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
<VersionBadge version="5.32.0" />
|
||||
|
||||
Findings Triage lets teams track review status and notes for individual findings in Prowler Cloud. Use it to record investigation state, remediation work, accepted risk, or false positive decisions without leaving the Findings workflow.
|
||||
|
||||
<SubscriptionBanner />
|
||||
|
||||
## What Is Findings Triage?
|
||||
|
||||
Findings Triage adds a **Triage** status and team note workflow to individual finding rows. It is available from:
|
||||
|
||||
- Expanded rows in **Finding Groups**
|
||||
- Standalone finding tables
|
||||
- Finding and resource detail drawers, including related findings tables
|
||||
|
||||
Finding Groups rows do not show triage controls because a group row represents several findings. Expand a group to work with each affected resource.
|
||||
|
||||

|
||||
|
||||
## Required Permissions
|
||||
|
||||
To update triage statuses and notes, the user role must have the **Manage Scans** permission. For more information, see [Role-Based Access Control (RBAC)](/user-guide/tutorials/prowler-app-rbac).
|
||||
|
||||
Users without this permission can still see existing triage context when it is available, but cannot change statuses or save notes.
|
||||
|
||||
## Triage Statuses
|
||||
|
||||
The status selector includes manual statuses. Prowler also sets automatic statuses after scans.
|
||||
|
||||
| Status | Type | Use It When |
|
||||
| --- | --- | --- |
|
||||
| **Open** | Manual | A failed finding has not been reviewed yet. A failed finding with no saved triage state also appears as **Open**. |
|
||||
| **Under Review** | Manual | A team is investigating the finding. |
|
||||
| **Remediating** | Manual | Work is in progress to fix the finding. |
|
||||
| **Risk Accepted** | Manual | The team accepts the risk and wants to mute the finding. |
|
||||
| **False Positive** | Manual | The finding does not apply and should be muted. |
|
||||
| **Resolved** | Automatic | A finding changed from `FAIL` to `PASS` in a later scan. A passed finding with no saved triage state also appears as **Resolved**. |
|
||||
| **Reopened** | Automatic | A finding changed from `PASS` to `FAIL` in a later scan. |
|
||||
|
||||

|
||||
|
||||
Resolved and Reopened are not manual selector options.
|
||||
|
||||
These automatic states keep triage tied to the finding UID across scans, even when each scan creates a new finding snapshot.
|
||||
|
||||
## Change a Triage Status
|
||||
|
||||
<Steps>
|
||||
<Step title="Open Findings">
|
||||
Go to **Findings** in Prowler Cloud.
|
||||
</Step>
|
||||
<Step title="Select an individual finding">
|
||||
Expand a Finding Group, open a resource findings table, or use a standalone finding row.
|
||||
</Step>
|
||||
<Step title="Open the triage selector">
|
||||
In the **Triage** column, click the current status.
|
||||
</Step>
|
||||
<Step title="Choose a status">
|
||||
Select **Open**, **Under Review**, **Remediating**, **Risk Accepted**, or **False Positive**.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
Changing a finding to **Risk Accepted** or **False Positive** will mute the finding. Prowler asks for confirmation and creates a mute rule for the finding.
|
||||
|
||||
## Add or Edit a Triage Note
|
||||
|
||||
Triage notes are visible only to the team in the current organization. Each note supports up to 500 characters.
|
||||
|
||||
<Steps>
|
||||
<Step title="Open the finding actions menu">
|
||||
On an individual finding row, click the actions menu.
|
||||
</Step>
|
||||
<Step title="Open the note modal">
|
||||
Click **Add Triage Note**. If a note already exists, click **Open note**.
|
||||
</Step>
|
||||
<Step title="Set status and note text">
|
||||
Optionally change the status, then write the note.
|
||||
</Step>
|
||||
<Step title="Save changes">
|
||||
Click **Save changes**.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||

|
||||
|
||||
To remove an existing note, clear the note text and save the change.
|
||||
|
||||
## Mutelist Behavior
|
||||
|
||||
Findings Triage uses Mutelist when a status means the finding should be muted:
|
||||
|
||||
- **Risk Accepted** creates a mute rule because the team accepts the finding as a known risk.
|
||||
- **False Positive** creates a mute rule because the finding should not count as an active issue.
|
||||
|
||||
Use [Simple Mutelist](/user-guide/tutorials/prowler-app-simple-mutelist) to review, disable, or delete mute rules created through this workflow. For pattern-based muting, use [Advanced Mutelist](/user-guide/tutorials/prowler-app-mute-findings).
|
||||
|
||||
<Warning>
|
||||
Muting a finding does not fix the underlying configuration. Review the finding before using **Risk Accepted** or **False Positive**.
|
||||
</Warning>
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Triage controls do not appear
|
||||
|
||||
Make sure the row is an individual finding row. Finding Groups rows do not show triage controls. Expand a group to see affected resources and their triage controls.
|
||||
|
||||
### Changes cannot be saved
|
||||
|
||||
Confirm that the user role has **Manage Scans** permission. Self-hosted Prowler App does not support Findings Triage writes.
|
||||
|
||||
### Resolved or Reopened is missing from the selector
|
||||
|
||||
This is expected. Prowler sets **Resolved** and **Reopened** automatically from scan result changes.
|
||||
|
||||
### Risk Accepted or False Positive muted a finding
|
||||
|
||||
This is expected. Those statuses create a mute rule through Mutelist.
|
||||
@@ -32,13 +32,13 @@ All three providers can be configured for a tenant, but only one can be set as t
|
||||
|
||||
When visiting Lighthouse AI chat, the default provider's default model loads automatically. Users can switch to any available LLM model (including those from non-default providers) using the dropdown in chat.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-switch-models.png" alt="Switch models in Lighthouse AI chat interface" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/switch-models.png" alt="Switch models in Lighthouse AI chat interface" />
|
||||
|
||||
## Configuring Providers
|
||||
|
||||
Navigate to **Configuration** → **Lighthouse AI** to see all three provider options with a **Connect** button under each.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-configuration.png" alt="Prowler Lighthouse Configuration" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/configuration.png" alt="Prowler Lighthouse Configuration" />
|
||||
|
||||
### Connecting a Provider
|
||||
|
||||
@@ -139,7 +139,7 @@ To set a different provider as default:
|
||||
2. Click **Configure** under the desired provider to set as default
|
||||
3. Click **Set as Default**
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-set-default-provider.png" alt="Set default LLM provider" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/set-default-provider.png" alt="Set default LLM provider" />
|
||||
|
||||
## Updating Provider Credentials
|
||||
|
||||
|
||||
@@ -8,6 +8,10 @@ import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
Prowler Lighthouse AI integrates Large Language Models (LLMs) with Prowler security findings data.
|
||||
|
||||
<Info>
|
||||
Using Prowler Cloud? Lighthouse AI on Prowler Cloud adds persistent chat sessions, GPT-5.5 as the default model, a dedicated agentic (chat) view, and transparent reasoning. See [Lighthouse AI on Prowler Cloud](/getting-started/products/prowler-cloud-lighthouse).
|
||||
</Info>
|
||||
|
||||
Behind the scenes, Lighthouse AI works as follows:
|
||||
|
||||
- Lighthouse AI runs as a [Langchain agent](https://docs.langchain.com/oss/javascript/langchain/agents) in NextJS
|
||||
@@ -47,7 +51,7 @@ Getting started with Prowler Lighthouse AI is easy:
|
||||
For detailed configuration instructions for each provider, see [Using Multiple LLM Providers with Lighthouse](/user-guide/tutorials/prowler-app-lighthouse-multi-llm).
|
||||
</Note>
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-configuration.png" alt="Lighthouse AI Configuration" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/configuration.png" alt="Lighthouse AI Configuration" />
|
||||
|
||||
### Adding Business Context
|
||||
|
||||
|
||||
@@ -40,6 +40,11 @@ Follow these steps to edit a user of your account:
|
||||
|
||||
<img src="/images/prowler-app/rbac/user_edit_details.png" alt="Edit User Details" width="700" />
|
||||
|
||||
<Note>
|
||||
Users can edit their own account details. Editing another user's account details requires the **Invite and Manage Users** or **admin** permission.
|
||||
|
||||
</Note>
|
||||
|
||||
#### Removing a User
|
||||
|
||||
Follow these steps to remove a user of your account:
|
||||
@@ -226,8 +231,8 @@ Assign administrative permissions by selecting from the following options:
|
||||
|------------|-------|-------------|
|
||||
| Invite and Manage Users | All | Invite new users and manage existing ones. |
|
||||
| Manage Account | All | Adjust account settings, delete users and read/manage users permissions. |
|
||||
| Manage Scans | All | Run and review scans. |
|
||||
| Manage Providers | All | Add or modify connected providers. |
|
||||
| Manage Scans | All | Run and review scans, and manage [Scan Configuration](/user-guide/tutorials/prowler-app-scan-configuration) settings. |
|
||||
| Manage Providers | All | Add or modify connected providers, and attach or detach providers from a [Scan Configuration](/user-guide/tutorials/prowler-app-scan-configuration) (in addition to Manage Scans). |
|
||||
| Manage Integrations | All | Add or modify the Prowler Integrations. |
|
||||
| Manage Ingestions | Prowler Cloud | Allow or deny the ability to submit findings ingestion batches via the API. |
|
||||
| Manage Billing | Prowler Cloud | Access and manage billing settings and subscription information. |
|
||||
|
||||